diff --git a/README.md b/README.md index 832d0e62..3b780114 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ wp static-site-importer materialize-wordpress-site-plan --plan=/path/to/plan.jso Static Site Importer is the WordPress materialization layer for static website inputs. It accepts two related shapes: -- Static source imports: an HTML entry file, pasted HTML document, public HTML URL, direct HTML upload, or ZIP source tree. +- Static source imports: an HTML entry file, pasted HTML document, public HTML URL, bounded public static-site collection, direct HTML upload, or ZIP source tree. - Generated website artifacts: a `blocks-engine/php-transformer/site-artifact/v1` bundle emitted by website generation or browser runtimes. The conversion stack is split by responsibility: @@ -167,7 +167,13 @@ PHP consumers can build the same blueprint with `static_site_importer_playground URL intake rules: -- Fetches one URL only; this is not a crawler and does not execute JavaScript. +- Fetches one URL by default and does not execute JavaScript. +- `provider_args.collect_site=true` or CLI `--collect-site` enables bounded collection. It reads the origin's `/sitemap.xml`, follows same-origin HTML links, collects directly referenced page assets and nested CSS assets, and emits one canonical website artifact. +- Collection defaults to 20 pages, 200 assets, 50 MiB total, 5 MiB per response, and 100 ms between requests. Limits can be configured with `max_pages`, `max_assets`, `max_total_bytes`, `max_bytes`, and `request_delay_ms` up to the collector's hard caps. +- Directly referenced scripts are collected by default so conversion starts from the complete source behavior. Set `include_scripts=false` or CLI `--skip-scripts` only when the caller has verified that script behavior is intentionally excluded or replaced. +- Registered source-exclusion rules remove non-authored platform chrome before asset discovery and compilation. Each removal records selector, provider, reason, and before/after hashes under `source_metadata.collection.source_exclusions`; set `exclude_platform_chrome=false` to preserve the raw source. +- Extensions can add or replace source-exclusion rules with the `static_site_importer_source_exclusion_rules` filter. Rules use stable ID selectors and reason-coded categories so removals remain explicit and auditable. +- External assets must be directly referenced by fetched HTML or CSS and pass the same public-IP and redirect validation as page URLs. - Only `http` and `https` URLs are accepted. - Localhost, loopback, link-local, private, and otherwise reserved IP targets are rejected before connecting. - Redirect targets are revalidated with the same policy and capped. @@ -264,6 +270,14 @@ wp static-site-importer import-url https://example.com/ \ --keep-source \ --report=report.json +wp static-site-importer import-url https://example.com/ \ + --collect-site \ + --max-pages=20 \ + --max-assets=100 \ + --slug=example-site \ + --activate \ + --overwrite + # Commerce-bearing import on a host without WooCommerce: skip seeding and continue. wp static-site-importer import-theme /path/to/store/index.html \ --slug=store-no-woo \ @@ -448,10 +462,10 @@ This repo is Homeboy-managed: ## Current Boundaries And Limitations - The importer is intentionally static-site/artifact-to-block-theme glue. Blocks Engine PHP transformer owns generic artifact compilation, format conversion, and conversion reports; SSI owns WordPress uploads, import workflows, media, route rewriting, page/product materialization, and theme assembly. -- The importer currently discovers flat sibling `*.html` files beside the selected entry file and recursive Markdown content documents; it does not crawl arbitrary nested HTML routes. +- Local source imports discover flat sibling `*.html` files beside the selected entry file and recursive Markdown content documents. Bounded URL collection discovers sitemap and same-origin linked HTML routes but does not execute JavaScript or perform platform-specific API extraction. - Admin imports accept pasted HTML, one public URL, a direct `.html` / `.htm` file, or a ZIP with a root `index.html` or exactly one nested `index.html`; CLI imports take a direct HTML entry path or one public URL. - MDX, Astro, Eleventy, Hugo, and other runtime/build orchestration is out of scope. Build those projects to static HTML first, or provide plain `.md` / `.markdown` source content alongside the HTML shell. -- Linked local stylesheets and inline styles are copied into `style.css`; inline scripts are copied into `assets/site.js`. Other asset copying is not a general-purpose crawler yet. +- Linked local stylesheets and inline styles are copied into `style.css`; inline scripts are copied into `assets/site.js`. Bounded URL collection packages directly referenced HTML/CSS assets, while local source intake does not independently crawl missing assets. - Navigation persistence is limited to supported header/footer shapes that can be converted into deterministic `wp_navigation` entities without guessing. - External live triage has exercised additional static sites; committed first-party fixtures include `tests/fixtures/wordpress-is-dead/` and `tests/fixtures/mixed-source-site/`. diff --git a/homeboy-test-manifest.json b/homeboy-test-manifest.json index a96c4111..aa37b989 100644 --- a/homeboy-test-manifest.json +++ b/homeboy-test-manifest.json @@ -30,6 +30,10 @@ "tests/smoke-visual-repair-css.php": { "environment": "standalone-php" }, "tests/smoke-webfont-producer-consumer.php": { "environment": "standalone-php" }, "tests/smoke-website-artifact-import-input.php": { "environment": "standalone-php" }, - "tests/smoke-wordpress-site-plan-materializer.php": { "environment": "standalone-php" } + "tests/smoke-wordpress-site-plan-materializer.php": { "environment": "standalone-php" }, + "tests/smoke-artifact-run-primitives.php": { "environment": "standalone-php" }, + "tests/smoke-figma-workspace-lifecycle.php": { "environment": "standalone-php" }, + "tests/smoke-url-batch-import.php": { "environment": "standalone-php" }, + "tests/smoke-url-site-collector.php": { "environment": "standalone-php" } } } diff --git a/includes/abilities.php b/includes/abilities.php index 608895d5..43ddc01a 100644 --- a/includes/abilities.php +++ b/includes/abilities.php @@ -156,16 +156,16 @@ function static_site_importer_register_abilities(): void { 'static-site-importer/import-url', array( 'label' => __( 'Import URL', 'static-site-importer' ), - 'description' => __( 'Import a source URL through a URL extraction provider and return a Static Site Importer report.', 'static-site-importer' ), + 'description' => __( 'Import one public HTML URL or collect a bounded public static site through a URL extraction provider.', 'static-site-importer' ), 'category' => STATIC_SITE_IMPORTER_ABILITY_CATEGORY, 'input_schema' => array( 'type' => 'object', 'properties' => array_merge( array( - 'url' => array( 'type' => 'string' ), - 'provider' => array( 'type' => 'string' ), - 'provider_args' => array( 'type' => 'object' ), - 'work_dir' => array( 'type' => 'string' ), + 'url' => array( 'type' => 'string' ), + 'provider' => array( 'type' => 'string' ), + 'provider_args' => array( 'type' => 'object' ), + 'work_dir' => array( 'type' => 'string' ), ), $import_properties ), diff --git a/includes/class-static-site-importer-artifact-run.php b/includes/class-static-site-importer-artifact-run.php new file mode 100644 index 00000000..04216614 --- /dev/null +++ b/includes/class-static-site-importer-artifact-run.php @@ -0,0 +1,75 @@ +root = $resolved; $token = preg_replace( '/[^A-Za-z0-9_-]/', '-', $purpose ); $this->directory = $this->root . '/.ssi-artifact-run-' . $token; + if ( is_link( $this->directory ) ) { throw new RuntimeException( 'Artifact workspace directory cannot be a symlink.' ); } + if ( ! is_dir( $this->directory ) && ! mkdir( $this->directory, 0700 ) ) { throw new RuntimeException( 'Artifact workspace could not be created.' ); } + $existing = $this->read_raw( 'workspace.json' ); $record = is_string( $existing ) ? json_decode( $existing, true ) : null; + $this->record = is_array( $record ) ? $record : array( 'schema' => 'static-site-importer/artifact-workspace/v1', 'purpose' => $token, 'created_at' => gmdate( 'c' ), 'retention' => $retention ); + if ( ! is_array( $record ) && is_wp_error( $this->publish_json( 'workspace.json', $this->record ) ) ) { throw new RuntimeException( 'Artifact workspace ownership record could not be published.' ); } + } + public function path( string $relative ): string|WP_Error { + if ( '' === $relative || str_contains( $relative, '\\' ) || str_starts_with( $relative, '/' ) || preg_match( '#(^|/)\.{1,2}(/|$)#', $relative ) ) { return new WP_Error( 'static_site_importer_artifact_workspace_path_invalid', 'Workspace paths must be safe relative paths.' ); } + $parts = explode( '/', $relative ); $current = $this->directory; foreach ( array_slice( $parts, 0, -1 ) as $part ) { $current .= '/' . $part; if ( is_link( $current ) ) { return new WP_Error( 'static_site_importer_artifact_workspace_symlink', 'Workspace paths cannot traverse symlinks.' ); } } + return $this->directory . '/' . $relative; + } + public function publish_raw( string $relative, string $bytes ) { + $path = $this->path( $relative ); if ( is_wp_error( $path ) ) { return $path; } $parent = dirname( $path ); + if ( ! is_dir( $parent ) && ! mkdir( $parent, 0700, true ) ) { return new WP_Error( 'static_site_importer_artifact_workspace_unavailable', 'Workspace directory is unavailable.' ); } + if ( is_link( $parent ) || ! str_starts_with( (string) realpath( $parent ) . '/', $this->directory . '/' ) ) { return new WP_Error( 'static_site_importer_artifact_workspace_symlink', 'Workspace writes must remain in owned directories.' ); } + $temp = tempnam( $parent, '.ssi-artifact-' ); if ( false === $temp || strlen( $bytes ) !== file_put_contents( $temp, $bytes ) || ! rename( $temp, $path ) ) { if ( is_string( $temp ) && is_file( $temp ) ) { unlink( $temp ); } return new WP_Error( 'static_site_importer_artifact_workspace_write_failed', 'Unable to atomically publish workspace data.', array( 'path' => $path ) ); } return $path; + } + public function publish_json( string $relative, array $data ) { $json = wp_json_encode( $data, JSON_PRETTY_PRINT ); return is_string( $json ) ? $this->publish_raw( $relative, $json ) : new WP_Error( 'static_site_importer_artifact_workspace_json_invalid', 'Workspace JSON could not be encoded.' ); } + public function read_raw( string $relative ): ?string { $path = $this->path( $relative ); return is_string( $path ) && ! is_link( $path ) && is_file( $path ) ? file_get_contents( $path ) : null; } // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reads owned bytes. + public function delete( string $relative ): bool { $path = $this->path( $relative ); return is_string( $path ) && ! is_link( $path ) && is_file( $path ) ? unlink( $path ) : false; } + public function directory(): string { return $this->directory; } public function retention(): array { return $this->record['retention'] ?? array(); } + public function is_expired(): bool { $expires = $this->retention()['expires_at'] ?? ''; return is_string( $expires ) && '' !== $expires && strtotime( $expires ) <= time(); } + public function purge_expired(): array { return $this->is_expired() ? $this->purge() : array( 'status' => 'retained', 'workspace' => $this->directory, 'reason' => 'not_expired', 'deleted' => array() ); } + public function cleanup( string $outcome ): array { $policy = $this->retention()[ 'on_' . $outcome ] ?? 'retain'; return 'purge_on_success' === $policy || 'purge' === $policy ? $this->purge() : array( 'status' => 'retained', 'workspace' => $this->directory, 'expires_at' => $this->retention()['expires_at'] ?? null, 'deleted' => array() ); } + public function purge(): array { + $removed=array();$skipped=array();$failed=array();if(is_link($this->directory)||!is_dir($this->directory)){return array('status'=>'failed','workspace'=>$this->directory,'removed'=>$removed,'skipped'=>array($this->directory),'failed'=>$failed);}$iterator=new RecursiveIteratorIterator(new RecursiveDirectoryIterator($this->directory,FilesystemIterator::SKIP_DOTS),RecursiveIteratorIterator::CHILD_FIRST);foreach($iterator as $item){$path=$item->getPathname();if(is_link($path)){$skipped[]=$path;continue;}$ok=$item->isDir()?@rmdir($path):@unlink($path);if($ok){$removed[]=$path;}else{$failed[]=$path;}}if(!@rmdir($this->directory)){$failed[]=$this->directory;}$status=empty($failed)&&empty($skipped)?'purged':(empty($removed)?'failed':'partial');return array('status'=>$status,'workspace'=>$this->directory,'removed'=>$removed,'skipped'=>$skipped,'failed'=>$failed); + } + public static function purge_expired_in( string $parent ): array { if(is_link($parent)||false===($parent=realpath($parent))){return array();}$receipts=array();foreach(glob($parent.'/.ssi-artifact-run-*')?:array()as$path){if(is_link($path)||!is_dir($path)){continue;}$raw=@file_get_contents($path.'/workspace.json');$record=is_string($raw)?json_decode($raw,true):null;$expires=is_array($record)?($record['retention']['expires_at']??''):'';if(is_string($expires)&&''!==$expires&&strtotime($expires)<=time()){$workspace=new self($parent,substr(basename($path),strlen('.ssi-artifact-run-')));$receipts[]=$workspace->purge();}}return $receipts; } +} + +final class Static_Site_Importer_Artifact_Run_Manifest { + private string $path; private string $identity; private array $contract; private array $data = array(); + public function __construct( string $path, string $identity, string $schema, array $contract ) { $this->path=$path; $this->identity=$identity; $this->contract=$contract; $this->data=array( 'schema'=>$schema, 'version'=>1, 'source'=>array( 'identity'=>$identity ), 'contract'=>$contract, 'state'=>'running', 'diagnostics'=>array() ); } + public function load() { if(!is_file($this->path)){return array();}$raw=file_get_contents($this->path);$data=is_string($raw)?json_decode($raw,true):null; if(!is_array($data)||empty($data['source']['identity'])){return new WP_Error('static_site_importer_batch_manifest_invalid','The batch run manifest is invalid.');}if($this->identity!==$data['source']['identity']||($data['contract']??null)!==$this->contract){return new WP_Error('static_site_importer_batch_contract_mismatch','The existing batch run targets a different import contract.',array('run_manifest'=>$this->path));}$this->data=$data;return $data; } // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reads public checkpoint. + public function save( array $data ) { $this->data=$data;$temp=tempnam(dirname($this->path),'.ssi-manifest-');$json=wp_json_encode($data,JSON_PRETTY_PRINT);if(false===$temp||!is_string($json)||strlen($json)!==file_put_contents($temp,$json)||!rename($temp,$this->path)){if(is_string($temp)&&is_file($temp)){unlink($temp);}return new WP_Error('static_site_importer_batch_checkpoint_write_failed','Unable to atomically write run state.',array('path'=>$this->path));}return true; } + public function replay(): ?array{return 'completed'===($this->data['state']??'')&&is_array($this->data['final_result']??null)?$this->data['final_result']:null;} + public function path(): string{return $this->path;} +} + +final class Static_Site_Importer_Artifact_Byte_Cache { + private Static_Site_Importer_Artifact_Run_Workspace $workspace; private string $namespace; private int $max_entries; private int $max_bytes; private ?int $entry_count=null; private ?int $used_bytes=null; private $reject_when=null; private array $counts=array('hits'=>0,'misses'=>0,'bytes_read'=>0,'bytes_written'=>0,'corrupt_entries'=>0,'bypassed'=>0,'negative_hits'=>0,'negative_writes'=>0,'negative_expired'=>0,'network_requests_avoided'=>0); private array $adopted=array(); + public function __construct( Static_Site_Importer_Artifact_Run_Workspace $workspace, string $namespace, int $max_entries=25000, int $max_bytes=4294967296 ){$this->workspace=$workspace;$this->namespace=preg_replace('/[^A-Za-z0-9_-]/','-',$namespace);$this->max_entries=$max_entries;$this->max_bytes=$max_bytes;} + private function name(string $key):string{return 'cache/'.$this->namespace.'/'.(preg_match('/^[a-f0-9]{64}$/',$key)?$key:hash('sha256',$key)).'.entry';} + public function get(string $key):?array{$name=$this->name($key);$raw=$this->workspace->read_raw($name);if(!is_string($raw)){return null;}$line=strpos($raw,"\n");$meta=false===$line?null:json_decode(substr($raw,0,$line),true);$bytes=false===$line?false:substr($raw,$line+1);if(!is_array($meta)||!is_string($bytes)||strlen($bytes)!==(int)($meta['bytes']??-1)||hash('sha256',$bytes)!==($meta['sha256']??'')||!is_array($meta['value']??null)){$this->counts['corrupt_entries']++;if($this->workspace->delete($name)){$this->removed(strlen($raw));}return null;}if($this->rejected($bytes,$meta['value'])){if($this->workspace->delete($name)){$this->removed(strlen($raw));}return null;}$this->counts['bytes_read']+=strlen($raw);return array('bytes'=>$bytes,'value'=>$meta['value']);} + public function get_failure(string $key,int $now):?array{$name=$this->name($key);$raw=$this->workspace->read_raw($name);if(!is_string($raw)){return null;}$meta=json_decode($raw,true);if(!is_array($meta)||'failure'!==($meta['type']??'')||!is_array($meta['error']??null))return null;if(isset($meta['retry_after'])&&(int)$meta['retry_after']<=$now){$this->counts['negative_expired']++;if($this->workspace->delete($name)){$this->removed(strlen($raw));}return null;}$this->counts['negative_hits']++;$this->counts['network_requests_avoided']++;$error=$meta['error'];$error['data']=is_array($error['data']??null)?$error['data']:array();$error['data']['_static_site_importer_negative_cache_hit']=true;return $error;} + public function put_failure(string $key,array $error,?int $retry_after=null):void{$raw=wp_json_encode(array('type'=>'failure','error'=>$error,'retry_after'=>$retry_after));if(!is_string($raw)){return;}$name=$this->name($key);$previous=$this->admit($name,strlen($raw));if(false===$previous){$this->counts['bypassed']++;return;}if(!is_wp_error($this->workspace->publish_raw($name,$raw))){$this->stored($previous,strlen($raw));$this->counts['negative_writes']++;$this->counts['bytes_written']+=strlen($raw);}} + public function put(string $key,string $bytes,array $value):void{if($this->rejected($bytes,$value)){return;}$meta=wp_json_encode(array('bytes'=>strlen($bytes),'sha256'=>hash('sha256',$bytes),'value'=>$value));$raw=is_string($meta)?$meta."\n".$bytes:false;if(!is_string($raw)){$this->counts['bypassed']++;return;}$name=$this->name($key);$previous=$this->admit($name,strlen($raw));if(false===$previous){$this->counts['bypassed']++;return;}if(!is_wp_error($this->workspace->publish_raw($name,$raw))){$this->stored($previous,strlen($raw));$this->counts['bytes_written']+=strlen($raw);}} + public function reject_when(callable $predicate):void{$this->reject_when=$predicate;} + private function rejected(string $bytes,array $value):bool{return is_callable($this->reject_when)&&(bool)call_user_func($this->reject_when,$bytes,$value);} + private function admit(string $name,int $bytes):int|false{$this->occupancy();$path=$this->workspace->path($name);$previous=is_string($path)&&is_file($path)?(int)filesize($path):0;$entries=(int)$this->entry_count+(0===$previous?1:0);$used=(int)$this->used_bytes-$previous+$bytes;return $entries>$this->max_entries||$used>$this->max_bytes?false:$previous;} + private function stored(int $previous,int $bytes):void{$this->entry_count=(int)$this->entry_count+(0===$previous?1:0);$this->used_bytes=(int)$this->used_bytes-$previous+$bytes;} + private function removed(int $bytes):void{if(null!==$this->entry_count&&null!==$this->used_bytes){$this->entry_count=max(0,$this->entry_count-1);$this->used_bytes=max(0,$this->used_bytes-$bytes);}} + private function occupancy():void{if(null!==$this->entry_count&&null!==$this->used_bytes){return;}$dir=$this->workspace->directory().'/cache/'.$this->namespace;$files=glob($dir.'/*.entry')?:array();$this->entry_count=count($files);$this->used_bytes=array_sum(array_map('filesize',$files));} + public function adopt_legacy(string $directory):void{if(is_link($directory)||!is_dir($directory)){return;}foreach(glob(rtrim($directory,'/').'/*.entry')?:array() as $path){if(is_link($path)){continue;}$raw=file_get_contents($path);$line=is_string($raw)?strpos($raw,"\n"):false;$meta=false===$line?null:json_decode(substr((string)$raw,0,$line),true);$bytes=false===$line?false:substr((string)$raw,$line+1);$value=is_array($meta)?($meta['metadata']??$meta['value']??null):null;if(is_array($meta)&&is_string($bytes)&&strlen($bytes)===(int)($meta['bytes']??-1)&&hash('sha256',$bytes)===($meta['sha256']??'')&&is_array($value)){$key=basename($path,'.entry');$this->put($key,$bytes,$value);$verified=$this->get($key);if(is_array($verified)&&$verified['bytes']===$bytes&&$verified['value']===$value){$this->adopted[$directory][]=$path;}}}} + public function cleanup_adopted(): array {$removed=array();$failed=array();foreach($this->adopted as $directory=>$paths){foreach($paths as $path){if(!is_link($path)&&is_file($path)&&@unlink($path)){$removed[]=$path;}elseif(is_file($path)){$failed[]=$path;}}if(!is_link($directory)&&is_dir($directory)&&!(glob($directory.'/*')?:array())){@rmdir($directory);}}return array('removed'=>$removed,'failed'=>$failed);} + public function hit():void{$this->counts['hits']++;}public function miss():void{$this->counts['misses']++;}public function network_avoided():void{$this->counts['network_requests_avoided']++;}public function evidence():array{return $this->counts;}public function consume():array{$delta=$this->counts;foreach($this->counts as $key=>$value){$this->counts[$key]=0;}return $delta;} +} + +final class Static_Site_Importer_Artifact_Batch_Cursor { + private static function id(array $units):string{return 'batch-'.substr(hash('sha256',(string)json_encode(array_values($units))),0,16);} + public static function create(array $units,int $size):array{$rows=array();foreach(array_chunk(array_values($units),$size)as $index=>$chunk){$rows[]=array('index'=>$index,'batch_id'=>self::id($chunk),'units'=>$chunk,'state'=>'pending','completed_units'=>0);}return $rows;} + public static function hydrate(array $rows,string $units='route_indexes',string $completed='completed_routes'):array{foreach($rows as $index=>&$row){$values=array_values($row[$units]??array());$row=array('index'=>$index,'batch_id'=>$row['batch_id']??self::id($values),'units'=>$values,'state'=>$row['state']??'pending','completed_units'=>(int)($row[$completed]??0),'result'=>$row['result']??null,'split_from'=>$row['split_from']??null,'effective_batch_size'=>$row['effective_batch_size']??null);}unset($row);return $rows;} + public static function next(array $rows):?int{foreach($rows as $index=>$row){if('completed'!==($row['state']??'')){return $index;}}return null;}public static function complete(array $rows,int $index):array{$rows[$index]['state']='completed';$rows[$index]['completed_units']=count($rows[$index]['units']??array());return $rows;}public static function fail(array $rows,int $index):array{$rows[$index]['state']='failed';return $rows;} + public static function split(array $rows,int $index):array{$row=$rows[$index];$units=$row['units']??array();$middle=(int)ceil(count($units)/2);$children=array(array('units'=>array_slice($units,0,$middle)),array('units'=>array_slice($units,$middle)));foreach($children as &$child){$child+=array('batch_id'=>self::id($child['units']),'state'=>'pending','completed_units'=>0,'split_from'=>$row['batch_id']??self::id($units),'effective_batch_size'=>count($child['units']));}unset($child);array_splice($rows,$index,1,$children);foreach($rows as $position=>&$row){$row['index']=$position;}unset($row);return $rows;} +} diff --git a/includes/class-static-site-importer-figma-import.php b/includes/class-static-site-importer-figma-import.php index 9f883e83..352437ab 100644 --- a/includes/class-static-site-importer-figma-import.php +++ b/includes/class-static-site-importer-figma-import.php @@ -8,6 +8,9 @@ if ( ! defined( 'ABSPATH' ) ) { exit; } +if ( ! class_exists( 'Static_Site_Importer_Artifact_Run_Workspace' ) ) { + require_once __DIR__ . '/class-static-site-importer-artifact-run.php'; +} if ( ! class_exists( 'Static_Site_Importer_Website_Artifact_Import_Input' ) ) { require_once __DIR__ . '/class-static-site-importer-website-artifact-import-input.php'; @@ -356,18 +359,21 @@ private static function website_artifact_from_figma_file( array $figma_file, arr return new WP_Error( 'static_site_importer_figma_file_content_invalid', 'Uploaded .fig content could not be decoded.', array( 'status' => 400 ) ); } - $tmp = tempnam( sys_get_temp_dir(), 'ssi-fig-' ); - // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_file_put_contents -- Transformer requires a local file path for .fig archive inspection. - if ( false === $tmp || false === file_put_contents( $tmp, $content ) ) { + $retention = ! empty( $input['retain_workspace'] ) ? array( 'on_success' => 'retain', 'on_failure' => 'retain', 'expires_at' => gmdate( 'c', time() + 604800 ) ) : array( 'on_success' => 'purge_on_success' ); + $workspace = new Static_Site_Importer_Artifact_Run_Workspace( sys_get_temp_dir(), 'fig-' . bin2hex( random_bytes( 8 ) ), $retention ); + $tmp = $workspace->publish_raw( 'input.fig', $content ); + if ( is_wp_error( $tmp ) ) { return new WP_Error( 'static_site_importer_figma_file_tempfile_failed', 'Uploaded .fig file could not be staged for transformation.', array( 'status' => 500 ) ); } try { - return self::website_artifact_from_figma_file_path( $tmp, $name, $input ); - } finally { - if ( file_exists( $tmp ) ) { - wp_delete_file( $tmp ); + $result = self::website_artifact_from_figma_file_path( $tmp, $name, $input ); + if ( ! is_wp_error( $result ) && ! empty( $input['retain_workspace'] ) ) { + $result['provenance']['artifact_workspace'] = array( 'path' => $workspace->directory(), 'expires_at' => $workspace->retention()['expires_at'] ?? null, 'cleanup' => $workspace->cleanup( 'failure' ) ); } + return $result; + } finally { + if ( empty( $input['retain_workspace'] ) ) { $workspace->cleanup( 'success' ); } } } diff --git a/includes/class-static-site-importer-source-normalizer.php b/includes/class-static-site-importer-source-normalizer.php new file mode 100644 index 00000000..c50e43f6 --- /dev/null +++ b/includes/class-static-site-importer-source-normalizer.php @@ -0,0 +1,154 @@ + $args Normalization options. + * @return array{html:string,exclusions:array>,diagnostics:array>} + */ + public static function normalize_html( string $html, string $source_url, array $args = array() ): array { + $cloudflare_email_links = 0; + $html = self::normalize_cloudflare_email_links( $html, $cloudflare_email_links ); + $diagnostics = array(); + if ( $cloudflare_email_links > 0 ) { + $diagnostics[] = array( + 'type' => 'source_normalization', + 'severity' => 'info', + 'reason_code' => 'cloudflare_email_link_decoded', + 'source_path' => $source_url, + 'count' => (string) $cloudflare_email_links, + ); + } + + if ( array_key_exists( 'exclude_platform_chrome', $args ) && ! $args['exclude_platform_chrome'] ) { + return array( 'html' => $html, 'exclusions' => array(), 'diagnostics' => $diagnostics ); + } + + $rules = self::rules(); + if ( function_exists( 'apply_filters' ) ) { + $rules = apply_filters( 'static_site_importer_source_exclusion_rules', $rules, $source_url, $args ); + } + if ( ! is_array( $rules ) ) { + $rules = array(); + } + + $original = $html; + $exclusions = array(); + foreach ( $rules as $rule ) { + if ( ! is_array( $rule ) || ! str_starts_with( (string) ( $rule['selector'] ?? '' ), '#' ) ) { + continue; + } + $selector = (string) $rule['selector']; + $removed = self::remove_element_by_id( $html, substr( $selector, 1 ) ); + if ( null === $removed ) { + continue; + } + $html = $removed['html']; + $receipt = array( + 'schema' => 'static-site-importer/source-exclusion/v1', + 'action' => 'removed', + 'category' => (string) ( $rule['category'] ?? 'source_chrome' ), + 'provider' => (string) ( $rule['provider'] ?? '' ), + 'rule_id' => (string) ( $rule['id'] ?? '' ), + 'selector' => $selector, + 'source_path' => $source_url, + 'reason_code' => (string) ( $rule['reason_code'] ?? 'source_chrome_removed' ), + 'removed_sha256' => hash( 'sha256', $removed['element'] ), + ); + $exclusions[] = $receipt; + $diagnostics[] = array( + 'type' => 'source_exclusion', + 'severity' => 'info', + 'reason_code' => $receipt['reason_code'], + 'source_path' => $source_url, + 'selector' => $selector, + 'provider' => $receipt['provider'], + ); + } + + foreach ( $exclusions as &$exclusion ) { + $exclusion['source_sha256'] = hash( 'sha256', $original ); + $exclusion['normalized_sha256'] = hash( 'sha256', $html ); + } + unset( $exclusion ); + + return array( 'html' => $html, 'exclusions' => $exclusions, 'diagnostics' => $diagnostics ); + } + + private static function normalize_cloudflare_email_links( string $html, int &$count ): string { + return (string) preg_replace_callback( + '~\bhref\s*=\s*(["\'])(?:https?://[^/"\']+)?/cdn-cgi/l/email-protection#([a-f0-9]+)\1~i', + static function ( array $match ) use ( &$count ): string { + $bytes = hex2bin( $match[2] ); + if ( false === $bytes || strlen( $bytes ) < 2 ) { + return $match[0]; + } + $key = ord( $bytes[0] ); + $email = ''; + for ( $index = 1, $length = strlen( $bytes ); $index < $length; ++$index ) { + $email .= chr( ord( $bytes[ $index ] ) ^ $key ); + } + if ( false === filter_var( $email, FILTER_VALIDATE_EMAIL ) ) { + return $match[0]; + } + ++$count; + return 'href=' . $match[1] . 'mailto:' . htmlspecialchars( $email, ENT_QUOTES | ENT_HTML5, 'UTF-8' ) . $match[1]; + }, + $html + ); + } + + /** @return array> */ + private static function rules(): array { + $path = __DIR__ . '/source-exclusion-rules.json'; + $json = is_readable( $path ) ? file_get_contents( $path ) : false; // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reads an importer-owned static policy file. + $data = is_string( $json ) ? json_decode( $json, true ) : null; + return is_array( $data ) && is_array( $data['rules'] ?? null ) ? $data['rules'] : array(); + } + + /** @return null|array{html:string,element:string} */ + private static function remove_element_by_id( string $html, string $id ): ?array { + if ( '' === $id ) { + return null; + } + $quoted_id = preg_quote( $id, '#' ); + $pattern = '#<([a-z][a-z0-9:-]*)\b[^>]*\bid\s*=\s*(?:"' . $quoted_id . '"|\'' . $quoted_id . '\'|' . $quoted_id . ')(?:\s|/?>)#is'; + if ( ! preg_match( $pattern, $html, $opening, PREG_OFFSET_CAPTURE ) ) { + return null; + } + $tag = strtolower( (string) $opening[1][0] ); + $start = (int) $opening[0][1]; + $remainder = substr( $html, $start ); + if ( ! preg_match_all( '#]*>#is', $remainder, $tags, PREG_OFFSET_CAPTURE ) ) { + return null; + } + $depth = 0; + foreach ( $tags[0] as $match ) { + $token = (string) $match[0]; + if ( str_starts_with( $token, '' ) ) { + ++$depth; + } + if ( 0 === $depth ) { + $length = (int) $match[1] + strlen( $token ); + $element = substr( $remainder, 0, $length ); + return array( 'html' => substr_replace( $html, '', $start, $length ), 'element' => $element ); + } + } + return null; + } +} diff --git a/includes/class-static-site-importer-theme-generator.php b/includes/class-static-site-importer-theme-generator.php index 2dcd41a8..5ddaa374 100644 --- a/includes/class-static-site-importer-theme-generator.php +++ b/includes/class-static-site-importer-theme-generator.php @@ -367,6 +367,23 @@ private static function public_result_from_wordpress_site_plan_receipt( array $r 'provenance_meta_key' => ! empty( $match['protected'] ) ? '' : '_static_site_importer_provenance', ); } + if ( ! empty( $args['batch_import'] ) ) { + $previous = self::read_source_of_truth_manifest( $theme['dir'] . '/static-site-importer-manifest.json' ); + if ( is_array( $previous['desired'] ?? null ) ) { + foreach ( array( 'pages', 'files', 'assets' ) as $kind ) { + $keys = array(); + foreach ( $manifest['desired'][ $kind ] as $item ) { + $keys[ (string) ( $item['source_path'] ?? $item['path'] ?? $item['theme_path'] ?? '' ) ] = true; + } + foreach ( $previous['desired'][ $kind ] ?? array() as $item ) { + $key = (string) ( $item['source_path'] ?? $item['path'] ?? $item['theme_path'] ?? '' ); + if ( '' !== $key && ! isset( $keys[ $key ] ) ) { + $manifest['desired'][ $kind ][] = $item; + } + } + } + } + } $manifest['existing_matches'] = $receipt['existing_matches'] ?? array( 'pages' => array() ); $cleanup = self::cleanup_stale_generated_theme_files( $theme['dir'], $manifest, $args ); if ( is_wp_error( $cleanup ) ) { @@ -431,6 +448,15 @@ private static function public_result_from_wordpress_site_plan_receipt( array $r ); } + /** @return array */ + private static function read_source_of_truth_manifest( string $path ): array { + if ( ! is_file( $path ) ) { + return array(); + } + $manifest = json_decode( (string) file_get_contents( $path ), true ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reads the prior importer-owned source-of-truth manifest for a resumable batch. + return is_array( $manifest ) && 'static-site-importer/source-of-truth-manifest/v1' === ( $manifest['schema'] ?? '' ) ? $manifest : array(); + } + /** Retain source diagnostics unless a persisted provider replacement explicitly covers them. */ private static function diagnostics_after_completed_entity_bindings( array $diagnostics, array $receipt ): array { $superseded_runtime_selectors = array(); @@ -749,7 +775,13 @@ private static function document_metadata_from_plan_receipt( array $plan ): arra /** @param array $payload */ private static function write_plan_projection( string $path, array $payload ): void { $json = wp_json_encode( $payload, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES ); - if ( false === $json || false === file_put_contents( $path, $json . "\n" ) ) { // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_file_put_contents -- Writes preflighted public import artifacts. + $data = false === $json ? false : $json . "\n"; + $temp = is_string( $data ) ? tempnam( dirname( $path ), '.ssi-projection-' ) : false; + $written = is_string( $data ) && false !== $temp ? file_put_contents( $temp, $data ) : false; // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_file_put_contents -- Atomically writes preflighted public import artifacts. + if ( false === $data || false === $temp || strlen( $data ) !== $written || ! rename( $temp, $path ) ) { + if ( is_string( $temp ) && file_exists( $temp ) ) { + unlink( $temp ); // phpcs:ignore WordPress.WP.AlternativeFunctions.unlink_unlink -- Removes a failed atomic projection temporary file. + } throw new RuntimeException( 'Failed to write a preflighted import artifact.' ); } } diff --git a/includes/class-static-site-importer-url-batch-import.php b/includes/class-static-site-importer-url-batch-import.php new file mode 100644 index 00000000..d7208cad --- /dev/null +++ b/includes/class-static-site-importer-url-batch-import.php @@ -0,0 +1,39 @@ +'purge_on_success','on_failure'=>'retain','expires_at'=>gmdate('c',time()+604800)));}catch(RuntimeException $error){return new WP_Error('static_site_importer_batch_work_dir_unavailable',$error->getMessage());} + if($workspace->is_expired()){$cleanup=$workspace->purge();$expired=$manifest_path??(trailingslashit($work_dir).'url-site-batch-manifest-'.hash('sha256',self::VERSION."\n".$url).'.json');$archive=$expired.'.expired-'.gmdate('YmdHis');$archived=is_file($expired)&&!is_link($expired)?@rename($expired,$archive):false;return new WP_Error('static_site_importer_batch_run_expired','The retained URL batch run expired and must be restarted.',array('cleanup'=>$cleanup,'expired_manifest'=>$expired,'archived_manifest'=>$archived?$archive:null,'restart_required'=>true));}$cache=new Static_Site_Importer_Artifact_Byte_Cache($workspace,'http-response');$cache->reject_when(static function(string $bytes,array $metadata):bool{$type=strtolower((string)($metadata['content_type']??''));return (str_starts_with($type,'text/html')||str_starts_with($type,'application/xhtml+xml'))&&'error'===(Static_Site_Importer_URL_Fetcher::html_source_diagnostic($bytes)['severity']??'');});$cache->adopt_legacy(trailingslashit($work_dir).'url-response-cache-'.$identity);$cache->adopt_legacy($workspace->directory().'/responses'); + $source_fetcher=$fetcher;$fetcher=self::cached_fetcher($cache,$source_fetcher);$run_manifest=new Static_Site_Importer_Artifact_Run_Manifest($manifest_path,$identity,'static-site-importer/url-site-batch-run/v1',$contract);$manifest=$run_manifest->load();if(is_wp_error($manifest)){return $manifest;}if(!empty($manifest)){$manifest['fetch_cache']=self::cache_counters($manifest['fetch_cache']??array());} + if(empty($manifest)){$routes=Static_Site_Importer_URL_Site_Collector::discover_routes($url,$args,$fetcher);if(is_wp_error($routes)){return $routes;}$routes=self::ordered_routes($url,$routes);if(empty($routes)){$routes=array($url);}$cursor=Static_Site_Importer_Artifact_Batch_Cursor::create(array_keys($routes),min(self::MAX_BATCH_PAGES,$batch_pages));$manifest=array('schema'=>'static-site-importer/url-site-batch-run/v1','version'=>self::VERSION,'source'=>array('url'=>$url,'identity'=>$identity),'contract'=>$contract,'discovery_limits'=>Static_Site_Importer_URL_Site_Collector::discovery_limits(),'per_batch_limits'=>array('max_pages'=>min(self::MAX_BATCH_PAGES,$batch_pages),'max_assets'=>min(2000,max(0,(int)$args['max_assets'])),'max_total_bytes'=>min(268435456,max(1,(int)$args['max_total_bytes'])),'max_response_bytes'=>10485760),'total_routes'=>count($routes),'routes'=>$routes,'batch_pages'=>min(self::MAX_BATCH_PAGES,$batch_pages),'batches'=>self::legacy_batches($cursor),'failures'=>array(),'diagnostics'=>array(),'external_asset_retained'=>array('count'=>0,'samples'=>array()),'fetch_cache'=>$cache->consume(),'state'=>'running');if(is_wp_error($run_manifest->save($manifest))){return $run_manifest->save($manifest);}} + if('completed'===($manifest['state']??'')&&is_array($manifest['final_result']??null)){$manifest['final_result']['url_batch_run']['fetch_cache']=$manifest['fetch_cache'];$cache->cleanup_adopted();return $manifest['final_result'];}$importer=$importer??static fn(array $artifact,array $import_args)=>Static_Site_Importer_Theme_Generator::import_website_artifact($artifact,$import_args);$cursor=Static_Site_Importer_Artifact_Batch_Cursor::hydrate($manifest['batches']); + while(null!==($index=Static_Site_Importer_Artifact_Batch_Cursor::next($cursor))){$batch=$cursor[$index];$routes=array_values(array_intersect_key($manifest['routes'],array_flip($batch['units'])));$batch_entry=in_array($url,$routes,true)?$url:($routes[0]??$url);$cache_name='batches/'.$batch['batch_id'].'.json';$old_cache=trailingslashit($work_dir).'url-site-batch-cache-'.$identity.'-'.$index.'.json';$raw=self::retained_runtime($workspace,$cache_name,'batches/'.$index.'.json',$old_cache,$routes);$runtime=is_string($raw)?(json_decode($raw,true)?:array()):array(); + if(empty($runtime)){$collect_args=$args;$collect_args['_route_set']=array_values(array_unique($routes));$collect_args['max_pages']=min(self::MAX_BATCH_PAGES+1,count($collect_args['_route_set'])+1);$collect_args['require_complete_collection']=true;$collect_args['asset_failure_policy']=count($routes)>1?'preserve_failed_external_assets':'preserve_external';$runtime=Static_Site_Importer_URL_Site_Collector::collect($batch_entry,$collect_args,$fetcher);if(is_wp_error($runtime)){if(count($routes)>1&&self::splittable_collection_error($runtime)){$cursor=Static_Site_Importer_Artifact_Batch_Cursor::split($cursor,$index);$manifest['batches']=self::legacy_batches($cursor);self::checkpoint_cache($manifest,$cache);$manifest['diagnostics'][]=array('code'=>'batch_subdivided','parent_batch'=>$batch['batch_id'],'children'=>array_column(array_slice($cursor,$index,2),'batch_id'));$run_manifest->save($manifest);return self::import($request,$input,$source_fetcher,$importer);}return self::failed($run_manifest,$workspace,$manifest,$cursor,$index,$runtime,$cache);}$write=$workspace->publish_json($cache_name,$runtime);if(is_wp_error($write)){return self::failed($run_manifest,$workspace,$manifest,$cursor,$index,$write,$cache);}} + $manifest['external_asset_retained']=self::merge_external_assets($manifest['external_asset_retained']??array(),$runtime['source_metadata']['collection']['external_asset_retained']??array(),$index);self::checkpoint_cache($manifest,$cache);$manifest['batches']=self::legacy_batches($cursor);if(is_wp_error($run_manifest->save($manifest))){return $run_manifest->save($manifest);}$import_args=Static_Site_Importer_URL_Import_Runtime::batch_import_args($input,$runtime);$import_args['activate']=$index===array_key_last($cursor)&&!empty($input['activate']);$import_args['batch_import']=true;$import_args['preserve_existing_theme_bootstrap']=$index>0;$import_args['import_run_id']=$identity;$result=$importer($runtime['artifact'],$import_args);if(is_wp_error($result)){return self::failed($run_manifest,$workspace,$manifest,$cursor,$index,$result,$cache);}$cursor=Static_Site_Importer_Artifact_Batch_Cursor::complete($cursor,$index);$cursor[$index]['result']=self::result_evidence($result,$runtime);$manifest['batches']=self::legacy_batches($cursor);$manifest['diagnostics']=array_slice(array_merge($manifest['diagnostics'],$result['import_validation_result']['diagnostics']??array()),-100);if(is_wp_error($run_manifest->save($manifest))){return $run_manifest->save($manifest);}$workspace->delete($cache_name);if(is_file($old_cache)){unlink($old_cache);}$final=$result;unset($result,$runtime,$raw);} + $manifest['batches']=self::legacy_batches($cursor);$aggregate=self::aggregate_result($manifest,$manifest_path,$final??array());$manifest['state']='completed';$manifest['completed_at']=gmdate('c');self::checkpoint_cache($manifest,$cache);$legacy_cleanup=$cache->cleanup_adopted();$aggregate['url_batch_run']['cleanup']=$workspace->cleanup('success');$aggregate['url_batch_run']['legacy_cache_cleanup']=$legacy_cleanup;$manifest['final_result']=$aggregate;if(is_wp_error($run_manifest->save($manifest))){return $run_manifest->save($manifest);}return $aggregate; + } + private static function cached_fetcher(Static_Site_Importer_Artifact_Byte_Cache $cache,?callable $fetcher):callable{$fetcher=$fetcher??static fn(string $url,array $args)=>Static_Site_Importer_URL_Fetcher::fetch($url,$args);return static function(string $url,array $args)use($cache,$fetcher){$types=isset($args['content_types'])&&is_array($args['content_types'])?array_values($args['content_types']):null;if(is_array($types)){sort($types);}$key=hash('sha256',$url."\n".wp_json_encode(array('max_bytes'=>$args['max_bytes']??null,'content_types'=>$types,'timeout'=>$args['timeout']??null)));$now=isset($args['_static_site_importer_negative_cache_now'])&&is_callable($args['_static_site_importer_negative_cache_now'])?(int)call_user_func($args['_static_site_importer_negative_cache_now']):time();if(isset($args['_static_site_importer_cache_failure'])&&$args['_static_site_importer_cache_failure'] instanceof WP_Error){$error=$args['_static_site_importer_cache_failure'];if(self::cacheable_failure($error)){$data=$error->get_error_data();$transient=self::transient_failure($error);$cache->put_failure($key,array('code'=>$error->get_error_code(),'message'=>$error->get_error_message(),'data'=>$data),$transient?$now+30:null);}return $error;}$failure=$cache->get_failure($key,$now);if(is_array($failure)){return new WP_Error((string)$failure['code'],(string)$failure['message'],$failure['data']??null);}$cached=$cache->get($key);if(is_array($cached)){$cache->hit();$cache->network_avoided();return array('body'=>$cached['bytes'],'metadata'=>$cached['value']+array('_static_site_importer_cache_hit'=>true));}$cache->miss();$response=$fetcher($url,$args);if(is_wp_error($response)){$data=is_array($response->get_error_data())?$response->get_error_data():array();$data['_static_site_importer_cache_aware']=true;return new WP_Error($response->get_error_code(),$response->get_error_message(),$data);}if(is_array($response)&&is_string($response['body']??null)&&is_array($response['metadata']??null)){$cache->put($key,$response['body'],$response['metadata']);}return $response;};} + private static function cacheable_failure(WP_Error $error):bool{$code=$error->get_error_code();if(str_contains($code,'invalid')||str_contains($code,'private')||str_contains($code,'credential')||str_contains($code,'scheme')){return false;}$status=is_array($error->get_error_data())?(int)($error->get_error_data()['status']??0):0;return self::transient_failure($error)||in_array($code,array('static_site_importer_url_unexpected_content_type','static_site_importer_url_empty_body','static_site_importer_url_too_large'),true)||('static_site_importer_url_http_status'===$code&&in_array($status,array(404,410),true));} + private static function transient_failure(WP_Error $error):bool{$code=strtolower($error->get_error_code());return str_contains($code,'timeout')||str_contains($code,'connect')||str_contains($code,'tls')||str_contains($code,'dns');} + private static function legacy_batches(array $cursor):array{return array_map(static fn(array $row):array=>array_filter(array('index'=>$row['index'],'batch_id'=>$row['batch_id'],'route_indexes'=>$row['units'],'state'=>$row['state'],'completed_routes'=>$row['completed_units'],'result'=>$row['result']??null,'split_from'=>$row['split_from']??null,'effective_batch_size'=>$row['effective_batch_size']??null),static fn($value):bool=>null!==$value),$cursor);} + private static function failed(Static_Site_Importer_Artifact_Run_Manifest $run_manifest,Static_Site_Importer_Artifact_Run_Workspace $workspace,array $manifest,array $cursor,int $index,WP_Error $error,Static_Site_Importer_Artifact_Byte_Cache $cache):WP_Error{$cursor=Static_Site_Importer_Artifact_Batch_Cursor::fail($cursor,$index);$manifest['state']='failed';$manifest['batches']=self::legacy_batches($cursor);self::checkpoint_cache($manifest,$cache);$manifest['failures'][]=array('batch'=>$index,'code'=>$error->get_error_code(),'message'=>$error->get_error_message(),'at'=>gmdate('c'));$write=$run_manifest->save($manifest);$data=array_merge(is_array($error->get_error_data())?$error->get_error_data():array(),array('run_manifest'=>$run_manifest->path(),'run'=>$manifest,'cleanup'=>$workspace->cleanup('failure')));if(is_wp_error($write)){$data['checkpoint_error']=array('code'=>$write->get_error_code(),'message'=>$write->get_error_message());}return new WP_Error($error->get_error_code(),$error->get_error_message(),$data);} + private static function checkpoint_cache(array &$manifest,Static_Site_Importer_Artifact_Byte_Cache $cache):void{foreach($cache->consume()as$key=>$delta){$manifest['fetch_cache'][$key]=(int)($manifest['fetch_cache'][$key]??0)+(int)$delta;}} + private static function cache_counters(array $counters):array{foreach(array('hits','misses','bytes_read','bytes_written','corrupt_entries','bypassed','negative_hits','negative_writes','negative_expired','network_requests_avoided')as$key){$counters[$key]=(int)($counters[$key]??0);}return $counters;} + private static function retained_runtime(Static_Site_Importer_Artifact_Run_Workspace $workspace,string $stable,string $numeric,string $legacy,array $routes):?string{$raw=$workspace->read_raw($stable);if(is_string($raw)&&self::owns_runtime($raw,$routes)){return $raw;}if(is_string($raw)){$workspace->delete($stable);}foreach(array($numeric,$legacy)as$source){$candidate='batches/'===substr($source,0,8)?$workspace->read_raw($source):(is_file($source)?file_get_contents($source):null);if(!is_string($candidate)||!self::owns_runtime($candidate,$routes)){continue;}$published=$workspace->publish_raw($stable,$candidate);if(is_wp_error($published)||$workspace->read_raw($stable)!==$candidate){continue;}if($source===$numeric){$workspace->delete($numeric);}elseif(is_file($source)){unlink($source);}return $candidate;}return null;} + private static function owns_runtime(string $raw,array $routes):bool{$runtime=json_decode($raw,true);$files=$runtime['source_metadata']['snapshot']['files']??null;if(!is_array($files)){return false;}$actual=array();foreach($files as$file){if('text/html'===strtolower((string)($file['mime_type']??''))&&is_string($file['source_url']??null)){$actual[]=self::page_key($file['source_url']);}}$explicit=array();foreach($runtime['artifact']['files']??array()as$file){if('text/html'!==strtolower((string)($file['mime_type']??''))){continue;}$route=(string)($file['metadata']['route_path']??'');if(''!==$route&&isset($explicit[$route])){return false;}$explicit[$route]=true;}$expected=array_map(array(self::class,'page_key'),$routes);sort($actual);sort($expected);return $actual===array_values(array_unique($expected));} + private static function page_key(string $url):string{$parts=parse_url($url);if(!is_array($parts)||empty($parts['host']))return '';$path=rtrim((string)($parts['path']??'/'),'/');if(''===$path||'/index.html'===$path||'/index.htm'===$path)$path='/';return strtolower((string)($parts['scheme']??'https')).'://'.strtolower((string)$parts['host']).$path.(isset($parts['query'])?'?'.$parts['query']:'');} + private static function existing_manifest(string $path):?array{if(!is_file($path)||is_link($path))return null;$data=json_decode((string)file_get_contents($path),true);return is_array($data)&&is_array($data['contract']??null)&&is_string($data['source']['identity']??null)?array('contract'=>$data['contract'],'identity'=>$data['source']['identity']):null;} + private static function ordered_routes(string $entry,array $routes):array{$routes[]=$entry;$routes=array_values(array_unique(array_filter($routes,'is_string')));usort($routes,static fn(string $a,string $b):int=>substr_count(trim((string)parse_url($a,PHP_URL_PATH),'/'),'/')<=>substr_count(trim((string)parse_url($b,PHP_URL_PATH),'/'),'/')?:strcmp($a,$b));return $routes;} + private static function splittable_collection_error(WP_Error $error):bool{$data=$error->get_error_data();if('static_site_importer_site_collection_incomplete'!==$error->get_error_code()||!is_array($data)){return false;}if(array_intersect($data['collection']['truncated']??array(),array('assets','bytes'))){return true;}foreach($data['collection']['failures']??array()as$failure){if('asset'===($failure['kind']??'')){return true;}}return false;} + private static function result_evidence(array $result,array $runtime):array{return array('theme_slug'=>$result['theme_slug']??'','snapshot_sha256'=>$runtime['source_metadata']['snapshot']['sha256']??'','plan_hash'=>$result['materialization_receipt']['plan_hash']??'','terminal_batch_report_path'=>$result['report_path']??'','quality'=>self::quality_evidence($result['quality']??($result['import_report_summary']['quality_pass']??null)));} + private static function quality_evidence(mixed $quality):mixed{if(!is_array($quality)){return is_bool($quality)?array('pass'=>$quality):null;}return array_filter(array('pass'=>isset($quality['pass'])?(bool)$quality['pass']:null,'status'=>isset($quality['status'])?(string)$quality['status']:null,'metrics'=>is_array($quality['metrics']??null)?$quality['metrics']:array(),'fallback_count'=>is_array($quality['fallbacks']??null)?count($quality['fallbacks']):(int)($quality['fallback_count']??0)),static fn($value):bool=>null!==$value);} + private static function merge_external_assets(array $aggregate,array $current,int $batch):array{$samples=$aggregate['samples']??array();$seen=array_column($samples,'url');foreach($current['samples']??array()as$sample){$url=(string)($sample['url']??'');if(''===$url||count($samples)>=50||in_array($url,$seen,true)){continue;}$sample['batch']=$batch;$samples[]=$sample;$seen[]=$url;}return array('count'=>(int)($aggregate['count']??0)+(int)($current['count']??0),'samples'=>$samples);} + private static function aggregate_result(array $manifest,string $path,array $terminal):array{$batch_quality=array_values(array_filter(array_map(static fn(array $batch):mixed=>self::quality_evidence($batch['result']['quality']??null),$manifest['batches']),static fn($quality):bool=>null!==$quality));$evidence=array('status'=>'completed','run_manifest'=>$path,'fetch_cache'=>$manifest['fetch_cache']??array(),'per_batch_limits'=>$manifest['per_batch_limits']??array(),'total_routes'=>$manifest['total_routes'],'completed_routes'=>array_sum(array_column($manifest['batches'],'completed_routes')),'total_batches'=>count($manifest['batches']),'completed_batches'=>count(array_filter($manifest['batches'],static fn(array $batch):bool=>'completed'===$batch['state'])),'failures'=>$manifest['failures'],'diagnostics'=>$manifest['diagnostics'],'external_asset_retained'=>$manifest['external_asset_retained']??array(),'batch_quality'=>$batch_quality,'terminal_batch_report_path'=>$terminal['report_path']??'');return array('success'=>true,'theme_slug'=>$terminal['theme_slug']??'','theme_name'=>$terminal['theme_name']??'','import_report_summary'=>array('status'=>'completed','scope'=>'url_site_batch_run','total_routes'=>$evidence['total_routes'],'completed_routes'=>$evidence['completed_routes'],'total_batches'=>$evidence['total_batches'],'completed_batches'=>$evidence['completed_batches']),'url_batch_run'=>$evidence,'batch_materialization'=>$manifest['batches'],'terminal_batch_result'=>$terminal);} + private static function contract(string $url,array $input,array $args,int $batch_pages):array{foreach(array_keys($args)as$key){if(str_starts_with((string)$key,'_static_site_importer_')){unset($args[$key]);}}return self::canonical(array('version'=>self::VERSION,'url'=>$url,'slug'=>(string)($input['slug']??''),'name'=>(string)($input['name']??''),'site_title'=>(string)($input['site_title']??''),'activate'=>!empty($input['activate']),'overwrite'=>!empty($input['overwrite']),'report'=>(string)($input['report']??''),'asset_failure_policy'=>'preserve_external_for_single_route_batch','batch_pages'=>min(self::MAX_BATCH_PAGES,$batch_pages),'provider_args'=>$args,'compiler_options'=>$input['compiler_options']??array()));} + private static function canonical(array $value):array{foreach($value as &$item){if(is_array($item)){$item=self::canonical($item);}}unset($item);if(!array_is_list($value)){ksort($value,SORT_STRING);}return $value;} +} diff --git a/includes/class-static-site-importer-url-fetcher.php b/includes/class-static-site-importer-url-fetcher.php index 5818f95b..225bb792 100644 --- a/includes/class-static-site-importer-url-fetcher.php +++ b/includes/class-static-site-importer-url-fetcher.php @@ -17,6 +17,7 @@ class Static_Site_Importer_URL_Fetcher { private const MAX_REDIRECTS = 5; private const DEFAULT_TIMEOUT = 10; private const DEFAULT_MAX_BYTES = 5242880; + private const MAX_RESPONSE_BYTES = 10485760; private const HTML_CONTENT_TYPES = array( 'text/html', 'application/xhtml+xml' ); private const REDIRECT_STATUSES = array( 301, 302, 303, 307, 308 ); private const BODY_READ_CHUNK = 8192; @@ -32,12 +33,52 @@ class Static_Site_Importer_URL_Fetcher { * @return array{html_path:string,metadata:array}|WP_Error */ public static function fetch_to_work_dir( string $url, string $work_dir, array $args = array() ) { - $timeout = max( self::CONNECT_TIMEOUT_FLOOR, (int) ( $args['timeout'] ?? self::DEFAULT_TIMEOUT ) ); - $max_bytes = max( 1, (int) ( $args['max_bytes'] ?? self::DEFAULT_MAX_BYTES ) ); - $initial = self::normalize_url( $url ); - $current = $initial; - $started = gmdate( 'c' ); - $redirects = array(); + $fetch = self::fetch( $url, $args ); + if ( is_wp_error( $fetch ) ) { + return $fetch; + } + + $source_diagnostic = self::html_source_diagnostic( $fetch['body'] ); + if ( ! empty( $source_diagnostic ) && 'error' === ( $source_diagnostic['severity'] ?? '' ) ) { + return new WP_Error( + 'static_site_importer_url_client_rendered_app', + 'This URL appears to be a JavaScript-rendered application shell. Static Site Importer can import server-rendered HTML, but this page needs a browser-rendered capture before it can produce WordPress blocks.', + array( + 'status' => 422, + 'diagnostic' => $source_diagnostic, + ) + ); + } + + wp_mkdir_p( $work_dir ); + $html_path = trailingslashit( $work_dir ) . 'index.html'; + $written = file_put_contents( $html_path, $fetch['body'] ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_file_put_contents -- Writes fetched static HTML to the importer source fixture. + if ( false === $written ) { + return new WP_Error( 'static_site_importer_url_write_failed', 'Failed to write fetched HTML to the import work directory.' ); + } + + return array( + 'html_path' => $html_path, + 'metadata' => $fetch['metadata'], + ); + } + + /** + * Fetch one public resource using the URL intake safety policy. + * + * @param string $url Public resource URL. + * @param array $args Fetch args. `content_types` optionally limits accepted MIME types. + * @return array{body:string,metadata:array}|WP_Error + */ + public static function fetch( string $url, array $args = array() ) { + $timeout = max( self::CONNECT_TIMEOUT_FLOOR, (int) ( $args['timeout'] ?? self::DEFAULT_TIMEOUT ) ); + $max_bytes = min( self::MAX_RESPONSE_BYTES, max( 1, (int) ( $args['max_bytes'] ?? self::DEFAULT_MAX_BYTES ) ) ); + $has_content_types_arg = isset( $args['content_types'] ) && is_array( $args['content_types'] ); + $content_types = $has_content_types_arg ? array_values( array_filter( array_map( static fn ( $value ): string => strtolower( (string) $value ), $args['content_types'] ) ) ) : self::HTML_CONTENT_TYPES; + $initial = self::normalize_url( $url ); + $current = $initial; + $started = gmdate( 'c' ); + $redirects = array(); for ( $attempt = 0; $attempt <= self::MAX_REDIRECTS; $attempt++ ) { $validation = self::validate_url( $current ); @@ -76,40 +117,28 @@ public static function fetch_to_work_dir( string $url, string $work_dir, array $ } if ( $status < 200 || $status >= 300 ) { - return new WP_Error( 'static_site_importer_url_http_status', sprintf( 'The URL returned HTTP status %d.', $status ) ); + return new WP_Error( 'static_site_importer_url_http_status', sprintf( 'The URL returned HTTP status %d.', $status ), array( 'status' => $status ) ); } $content_type = self::first_header( $response['headers'], 'content-type' ); - if ( ! self::is_html_content_type( $content_type ) ) { - return new WP_Error( 'static_site_importer_url_non_html', 'The URL did not return an HTML content type.' ); - } - - if ( '' === trim( $response['body'] ) ) { - return new WP_Error( 'static_site_importer_url_empty_body', 'The URL returned an empty HTML response.' ); - } - - $source_diagnostic = self::html_source_diagnostic( $response['body'] ); - if ( ! empty( $source_diagnostic ) && 'error' === ( $source_diagnostic['severity'] ?? '' ) ) { - return new WP_Error( - 'static_site_importer_url_client_rendered_app', - 'This URL appears to be a JavaScript-rendered application shell. Static Site Importer can import server-rendered HTML, but this page needs a browser-rendered capture before it can produce WordPress blocks.', - array( - 'status' => 422, - 'diagnostic' => $source_diagnostic, - ) - ); + $normalized_content_type = strtolower( trim( explode( ';', $content_type, 2 )[0] ) ); + $content_type_allowed = $has_content_types_arg ? empty( $content_types ) || in_array( $normalized_content_type, $content_types, true ) : self::is_html_content_type( $content_type ); + if ( ! $content_type_allowed ) { + if ( ! $has_content_types_arg ) { + return new WP_Error( 'static_site_importer_url_non_html', 'The URL did not return an HTML content type.' ); + } + return new WP_Error( 'static_site_importer_url_unexpected_content_type', sprintf( 'The URL returned unsupported content type %s.', '' !== $content_type ? $content_type : '(missing)' ) ); } - wp_mkdir_p( $work_dir ); - $html_path = trailingslashit( $work_dir ) . 'index.html'; - $written = file_put_contents( $html_path, $response['body'] ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_file_put_contents -- Writes fetched static HTML to the importer source fixture. - if ( false === $written ) { - return new WP_Error( 'static_site_importer_url_write_failed', 'Failed to write fetched HTML to the import work directory.' ); + // An explicit empty accepted-type list is used for optional binary/text assets. + // HTML and explicitly requested HTML responses must still contain a document. + if ( '' === trim( $response['body'] ) && ( ! $has_content_types_arg || ! empty( $content_types ) ) ) { + return new WP_Error( 'static_site_importer_url_empty_body', $has_content_types_arg ? 'The URL returned an empty response.' : 'The URL returned an empty HTML response.' ); } return array( - 'html_path' => $html_path, - 'metadata' => array( + 'body' => $response['body'], + 'metadata' => array( 'source_type' => 'url', 'source_url' => $initial, 'final_url' => $current, @@ -158,13 +187,14 @@ public static function html_source_diagnostic( string $html ): array { $text_html = preg_replace( '#]*>.*?#is', ' ', $html ); $text_html = preg_replace( '#]*>.*?#is', ' ', (string) $text_html ); $text_html = preg_replace( '#]*>.*?#is', ' ', (string) $text_html ); + $content_elements = preg_match_all( '#<(?:main|article|h[1-6]|p)\b#i', (string) $text_html ); // phpcs:ignore WordPress.WP.AlternativeFunctions.strip_tags_strip_tags -- Fallback only for non-WordPress smoke tests; WordPress runtimes use wp_strip_all_tags(). $stripped = function_exists( 'wp_strip_all_tags' ) ? wp_strip_all_tags( (string) $text_html ) : strip_tags( (string) $text_html ); $text = html_entity_decode( trim( preg_replace( '/\s+/', ' ', $stripped ) ), ENT_QUOTES | ENT_HTML5, 'UTF-8' ); $text_chars = strlen( $text ); $text_ratio = $markup_bytes > 0 ? $text_chars / $markup_bytes : 0; - if ( ( $script_count >= 20 && $text_chars < 1000 && $text_ratio < 0.02 ) || ( $script_count >= 3 && $text_chars < 200 && $app_shell ) ) { + if ( ( $script_count >= 20 && $text_chars < 1000 && $text_ratio < 0.02 && 0 === $content_elements ) || ( $script_count >= 3 && $text_chars < 200 && $app_shell ) ) { return array( 'type' => 'client_rendered_app_shell', 'severity' => 'error', diff --git a/includes/class-static-site-importer-url-import-runtime.php b/includes/class-static-site-importer-url-import-runtime.php index 9b473cbd..06cbe7a5 100644 --- a/includes/class-static-site-importer-url-import-runtime.php +++ b/includes/class-static-site-importer-url-import-runtime.php @@ -12,6 +12,15 @@ if ( ! class_exists( 'Static_Site_Importer_URL_Fetcher' ) ) { require_once __DIR__ . '/class-static-site-importer-url-fetcher.php'; } +if ( ! class_exists( 'Static_Site_Importer_URL_Site_Collector' ) ) { + require_once __DIR__ . '/class-static-site-importer-url-site-collector.php'; +} +if ( ! class_exists( 'Static_Site_Importer_Source_Normalizer' ) ) { + require_once __DIR__ . '/class-static-site-importer-source-normalizer.php'; +} +if ( ! class_exists( 'Static_Site_Importer_URL_Batch_Import' ) ) { + require_once __DIR__ . '/class-static-site-importer-url-batch-import.php'; +} if ( ! class_exists( 'Static_Site_Importer_Website_Artifact_Import_Input' ) ) { require_once __DIR__ . '/class-static-site-importer-website-artifact-import-input.php'; @@ -29,10 +38,29 @@ class Static_Site_Importer_URL_Import_Runtime { * @return array|WP_Error */ public static function import_url( array $input ) { - $runtime = self::website_artifact_from_url( $input ); + $url = isset( $input['url'] ) ? Static_Site_Importer_URL_Fetcher::normalize_url( (string) $input['url'] ) : ''; + if ( '' === $url ) { + return new WP_Error( 'static_site_importer_missing_url', 'The url input is required.' ); + } + $input['url'] = $url; + $request = self::provider_request( $url, $input ); + $provider_output = self::provider_output( $request ); + if ( is_wp_error( $provider_output ) ) { + return $provider_output; + } + if ( is_array( $provider_output ) ) { + $runtime = $provider_output; + } elseif ( ! empty( $request['provider_args']['collect_site'] ) && array_key_exists( 'batch_pages', $request['provider_args'] ) ) { + return Static_Site_Importer_URL_Batch_Import::import( $request, $input ); + } else { + $runtime = self::fetch_public_url_provider( $request ); + } if ( is_wp_error( $runtime ) ) { return $runtime; } + if ( empty( $runtime['artifact'] ) || ! is_array( $runtime['artifact'] ) ) { + return new WP_Error( 'static_site_importer_url_provider_missing_artifact', 'The URL import provider did not return a website artifact.' ); + } $args = self::import_args( $input, $runtime ); return Static_Site_Importer_Theme_Generator::import_website_artifact( $runtime['artifact'], $args ); @@ -79,7 +107,7 @@ private static function provider_request( string $url, array $input ): array { 'url' => $url, 'provider' => isset( $input['provider'] ) ? (string) $input['provider'] : '', 'provider_args' => isset( $input['provider_args'] ) && is_array( $input['provider_args'] ) ? $input['provider_args'] : array(), - 'work_dir' => isset( $input['work_dir'] ) ? (string) $input['work_dir'] : self::default_work_dir(), + 'work_dir' => ! empty( $input['work_dir'] ) ? (string) $input['work_dir'] : self::default_work_dir(), 'source_metadata' => isset( $input['source_metadata'] ) && is_array( $input['source_metadata'] ) ? $input['source_metadata'] : array(), ); } @@ -94,6 +122,15 @@ private static function provider_request( string $url, array $input ): array { * @return array|WP_Error */ private static function resolve_provider( array $request ) { + $provider_output = self::provider_output( $request ); + if ( is_wp_error( $provider_output ) || is_array( $provider_output ) ) { + return $provider_output; + } + return self::fetch_public_url_provider( $request ); + } + + /** @return null|array|WP_Error */ + private static function provider_output( array $request ) { /** * Filters URL import provider output before the built-in public URL fetcher runs. * @@ -104,15 +141,7 @@ private static function resolve_provider( array $request ) { * @param null|array|WP_Error $provider_output Provider output. * @param array $request Provider request. */ - $provider_output = apply_filters( 'static_site_importer_url_import_provider', null, $request ); - if ( is_wp_error( $provider_output ) ) { - return $provider_output; - } - if ( is_array( $provider_output ) ) { - return $provider_output; - } - - return self::fetch_public_url_provider( $request ); + return apply_filters( 'static_site_importer_url_import_provider', null, $request ); } /** @@ -122,10 +151,16 @@ private static function resolve_provider( array $request ) { * @return array|WP_Error */ private static function fetch_public_url_provider( array $request ) { + $provider_args = isset( $request['provider_args'] ) && is_array( $request['provider_args'] ) ? $request['provider_args'] : array(); + if ( ! empty( $provider_args['collect_site'] ) ) { + $provider_args['require_complete_collection'] = true; + return Static_Site_Importer_URL_Site_Collector::collect( (string) $request['url'], $provider_args ); + } + $fetch = Static_Site_Importer_URL_Fetcher::fetch_to_work_dir( (string) $request['url'], (string) $request['work_dir'], - isset( $request['provider_args'] ) && is_array( $request['provider_args'] ) ? $request['provider_args'] : array() + $provider_args ); if ( is_wp_error( $fetch ) ) { return $fetch; @@ -135,6 +170,11 @@ private static function fetch_public_url_provider( array $request ) { if ( false === $html ) { return new WP_Error( 'static_site_importer_url_artifact_read_failed', 'Failed to read fetched URL HTML.' ); } + $normalized = Static_Site_Importer_Source_Normalizer::normalize_html( $html, (string) $request['url'], $provider_args ); + $html = $normalized['html']; + $metadata = $fetch['metadata']; + $metadata['source_exclusions'] = $normalized['exclusions']; + $metadata['diagnostics'] = array_merge( is_array( $metadata['diagnostics'] ?? null ) ? $metadata['diagnostics'] : array(), $normalized['diagnostics'] ); return array( 'provider' => 'public-url-fetcher', @@ -147,7 +187,7 @@ private static function fetch_public_url_provider( array $request ) { ), ), ), - 'source_metadata' => $fetch['metadata'], + 'source_metadata' => $metadata, ); } @@ -170,6 +210,11 @@ private static function import_args( array $input, array $runtime ): array { return Static_Site_Importer_Website_Artifact_Import_Input::normalize( $input ); } + /** @return array */ + public static function batch_import_args( array $input, array $runtime ): array { + return self::import_args( $input, $runtime ); + } + /** * Build the default work directory for the built-in URL provider. * diff --git a/includes/class-static-site-importer-url-site-collector.php b/includes/class-static-site-importer-url-site-collector.php new file mode 100644 index 00000000..32e07d49 --- /dev/null +++ b/includes/class-static-site-importer-url-site-collector.php @@ -0,0 +1,877 @@ +,source_metadata:array}|WP_Error + */ + public static function collect( string $url, array $args = array(), ?callable $fetcher = null ) { + $entry_url = self::canonical_url( Static_Site_Importer_URL_Fetcher::normalize_url( $url ) ); + if ( '' === $entry_url ) { + return new WP_Error( 'static_site_importer_site_collection_invalid_url', 'Enter a valid public site URL.' ); + } + + $max_pages = min( self::MAX_PAGES, max( 1, (int) ( $args['max_pages'] ?? self::DEFAULT_MAX_PAGES ) ) ); + $max_assets = min( self::MAX_ASSETS, max( 0, (int) ( $args['max_assets'] ?? self::DEFAULT_MAX_ASSETS ) ) ); + $max_total_bytes = min( self::MAX_TOTAL_BYTES, max( 1, (int) ( $args['max_total_bytes'] ?? self::DEFAULT_MAX_TOTAL_BYTES ) ) ); + $request_delay = min( 2000, max( 0, (int) ( $args['request_delay_ms'] ?? 100 ) ) ); + $fetcher = $fetcher ?? static fn ( string $resource_url, array $fetch_args ) => Static_Site_Importer_URL_Fetcher::fetch( $resource_url, $fetch_args ); + $fetch_attempts = min( 3, max( 1, (int) ( $args['fetch_attempts'] ?? 2 ) ) ); + $fetch_resource = $fetcher; + $fetcher = static function ( string $resource_url, array $fetch_args ) use ( $fetch_resource, $fetch_attempts ) { + $response = null; + for ( $attempt = 0; $attempt < $fetch_attempts; $attempt++ ) { + $response = $fetch_resource( $resource_url, $fetch_args ); + if ( ! is_wp_error( $response ) ) { + return $response; + } + } + $data = is_wp_error( $response ) && is_array( $response->get_error_data() ) ? $response->get_error_data() : array(); + if ( ! empty( $data['_static_site_importer_cache_aware'] ) ) { + unset( $data['_static_site_importer_cache_aware'] ); + $response = new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null ); + $fetch_resource( $resource_url, $fetch_args + array( '_static_site_importer_cache_failure' => $response ) ); + } + return $response; + }; + $fetch_args = array_intersect_key( $args, array_flip( array( 'timeout' ) ) ); + $fetch_args['max_bytes'] = min( self::MAX_RESPONSE_BYTES, $max_total_bytes, max( 1, (int) ( $args['max_bytes'] ?? 5242880 ) ) ); + + $page_queue = array( $entry_url ); + $asset_queue = array(); + $queued_pages = array( self::page_key( $entry_url ) => true ); + $queued_assets = array(); + $resources = array(); + $failures = array(); + $diagnostics = array(); + $source_exclusions = array(); + $aliases = array(); + $total_bytes = 0; + $truncated = array(); + $external_assets = array(); + $asset_failure_policy = $args['asset_failure_policy'] ?? ''; + $preserve_failed_assets = in_array( $asset_failure_policy, array( 'preserve_external', 'preserve_failed_external_assets' ), true ); + $preserve_asset_limits = 'preserve_external' === $asset_failure_policy; + $entry_resource_url = $entry_url; + $site_url = $entry_url; + + $sitemap_urls = isset( $args['_route_set'] ) && is_array( $args['_route_set'] ) ? array_values( $args['_route_set'] ) : self::sitemap_urls( $entry_url, $fetcher, $fetch_args ); + if ( is_wp_error( $sitemap_urls ) ) { + return $sitemap_urls; + } + foreach ( $sitemap_urls as $page_url ) { + if ( count( $page_queue ) >= $max_pages ) { + $truncated['pages'] = true; + break; + } + $page_key = self::page_key( $page_url ); + if ( ! isset( $queued_pages[ $page_key ] ) ) { + $queued_pages[ $page_key ] = true; + $page_queue[] = $page_url; + } + } + + while ( $page_queue && count( array_filter( $resources, static fn ( array $resource ): bool => 'html' === $resource['kind'] ) ) < $max_pages ) { + $page_url = array_shift( $page_queue ); + $response = $fetcher( $page_url, array_merge( $fetch_args, array( 'content_types' => array( 'text/html', 'application/xhtml+xml' ) ) ) ); + self::delay_after_fetch( $response, $request_delay, $args ); + $response = self::without_cache_marker( $response ); + if ( is_wp_error( $response ) ) { + if ( $page_url === $entry_url ) { + return $response; + } + $failures[] = self::failure( $page_url, $response, 'html' ); + continue; + } + + $final_url = self::response_url( $response, $page_url ); + if ( $page_url === $entry_url ) { + $entry_resource_url = $final_url; + $site_url = $final_url; + } + if ( $final_url !== $page_url ) { + $aliases[ $page_url ] = $final_url; + } + if ( isset( $resources[ $final_url ] ) ) { + continue; + } + + $body = (string) $response['body']; + $normalized = Static_Site_Importer_Source_Normalizer::normalize_html( $body, $final_url, $args ); + $body = $normalized['html']; + $source_exclusions = array_merge( $source_exclusions, $normalized['exclusions'] ); + $diagnostics = array_merge( $diagnostics, $normalized['diagnostics'] ); + $bytes = strlen( $body ); + if ( $total_bytes + $bytes > $max_total_bytes ) { + $truncated['bytes'] = true; + break; + } + + $diagnostic = Static_Site_Importer_URL_Fetcher::html_source_diagnostic( $body ); + if ( ! empty( $diagnostic ) && 'error' === ( $diagnostic['severity'] ?? '' ) ) { + $error = new WP_Error( 'static_site_importer_url_client_rendered_app', (string) $diagnostic['message'], array( 'diagnostic' => $diagnostic ) ); + if ( $page_url === $entry_url ) { + return $error; + } + $diagnostic['severity'] = 'warning'; + $diagnostic['url'] = $page_url; + $diagnostic['disposition'] = 'collected_static_html'; + $diagnostics[] = $diagnostic; + } + + $total_bytes += $bytes; + $resources[ $final_url ] = array( + 'kind' => 'html', + 'body' => $body, + 'content_type' => self::content_type( $response, 'text/html' ), + ); + + $document_base_url = self::html_base_url( $body, $final_url ); + foreach ( isset( $args['_route_set'] ) ? array() : self::html_page_urls( $body, $document_base_url, $site_url ) as $discovered_url ) { + $page_key = self::page_key( $discovered_url ); + if ( isset( $queued_pages[ $page_key ] ) ) { + continue; + } + if ( count( $page_queue ) + self::resource_count( $resources, 'html' ) >= $max_pages ) { + $truncated['pages'] = true; + break; + } + $queued_pages[ $page_key ] = true; + $page_queue[] = $discovered_url; + } + + $include_scripts = ! array_key_exists( 'include_scripts', $args ) || (bool) $args['include_scripts']; + foreach ( self::html_asset_urls( $body, $document_base_url, $include_scripts ) as $asset_url ) { + if ( isset( $queued_assets[ $asset_url ] ) || isset( $resources[ $asset_url ] ) ) { + continue; + } + if ( count( $asset_queue ) + self::resource_count( $resources, 'asset' ) >= $max_assets ) { + if ( $preserve_asset_limits ) { $external_assets[ $asset_url ] = 'asset_limit'; continue; } + $truncated['assets'] = true; + break; + } + $queued_assets[ $asset_url ] = true; + $asset_queue[] = $asset_url; + } + + } + + while ( $asset_queue && self::resource_count( $resources, 'asset' ) < $max_assets ) { + $asset_url = array_shift( $asset_queue ); + $response = $fetcher( $asset_url, array_merge( $fetch_args, array( 'content_types' => array() ) ) ); + self::delay_after_fetch( $response, $request_delay, $args ); + $response = self::without_cache_marker( $response ); + if ( is_wp_error( $response ) ) { + if ( $preserve_failed_assets ) { $external_assets[ $asset_url ] = $response->get_error_code(); continue; } + $failures[] = self::failure( $asset_url, $response, 'asset' ); + continue; + } + + $final_url = self::response_url( $response, $asset_url ); + if ( $final_url !== $asset_url ) { + $aliases[ $asset_url ] = $final_url; + } + if ( isset( $resources[ $final_url ] ) ) { + continue; + } + + $body = (string) $response['body']; + $bytes = strlen( $body ); + if ( $total_bytes + $bytes > $max_total_bytes ) { + if ( $preserve_asset_limits ) { $external_assets[ $asset_url ] = 'byte_limit'; continue; } + $truncated['bytes'] = true; + break; + } + + $content_type = self::content_type( $response, 'application/octet-stream' ); + $total_bytes += $bytes; + $resources[ $final_url ] = array( + 'kind' => 'asset', + 'body' => $body, + 'content_type' => $content_type, + ); + + if ( 'text/css' === $content_type || str_ends_with( strtolower( (string) parse_url( $final_url, PHP_URL_PATH ) ), '.css' ) ) { + foreach ( self::css_asset_urls( $body, $final_url ) as $nested_url ) { + if ( isset( $queued_assets[ $nested_url ] ) || isset( $resources[ $nested_url ] ) ) { + continue; + } + if ( count( $asset_queue ) + self::resource_count( $resources, 'asset' ) >= $max_assets ) { + if ( $preserve_asset_limits ) { $external_assets[ $nested_url ] = 'asset_limit'; continue; } + $truncated['assets'] = true; + break; + } + $queued_assets[ $nested_url ] = true; + $asset_queue[] = $nested_url; + } + } + + } + + if ( ( ! empty( $truncated ) || ! empty( $failures ) ) && ! empty( $args['require_complete_collection'] ) ) { + return new WP_Error( + 'static_site_importer_site_collection_incomplete', + 'The public site could not be collected completely.', + array( + 'collection' => array( + 'pages' => self::resource_count( $resources, 'html' ), + 'assets' => self::resource_count( $resources, 'asset' ), + 'bytes' => $total_bytes, + 'failures' => $failures, + 'truncated' => array_keys( $truncated ), + ), + 'limits' => array( + 'max_pages' => $max_pages, + 'max_assets' => $max_assets, + 'max_total_bytes' => $max_total_bytes, + ), + ) + ); + } + + ksort( $resources, SORT_STRING ); + ksort( $aliases, SORT_STRING ); + ksort( $external_assets, SORT_STRING ); + $paths = self::artifact_paths( $resources, $site_url ); + $route_paths = self::route_paths( $resources ); + $reference_paths = $paths; + foreach ( $aliases as $requested_url => $final_url ) { + if ( isset( $paths[ $final_url ] ) ) { + $reference_paths[ $requested_url ] = $paths[ $final_url ]; + } + } + $files = array(); + $snapshot_files = array(); + foreach ( $resources as $resource_url => $resource ) { + $path = $paths[ $resource_url ]; + $body = (string) $resource['body']; + if ( 'html' === $resource['kind'] ) { + $body = self::rewrite_html( $body, self::html_base_url( $body, $resource_url ), $path, $reference_paths, $aliases, $site_url, $external_assets ); + } elseif ( 'text/css' === $resource['content_type'] || str_ends_with( strtolower( $path ), '.css' ) ) { + $body = self::rewrite_css( $body, $resource_url, $path, $reference_paths, $external_assets ); + } + + $file = array( + 'path' => $path, + 'mime_type' => $resource['content_type'], + ); + if ( 'html' === $resource['kind'] ) { + $file['metadata'] = array( 'route_path' => $route_paths[ $resource_url ] ); + } + if ( self::is_text( $resource['content_type'], $path ) ) { + $file['content'] = $body; + } else { + $file['content_base64'] = base64_encode( $body ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Encodes binary artifact payload bytes. + } + $files[] = $file; + $snapshot_files[] = array( + 'path' => $path, + 'source_url' => $resource_url, + 'mime_type' => $resource['content_type'], + 'bytes' => strlen( $body ), + 'sha256' => hash( 'sha256', $body ), + ); + } + usort( $files, static fn ( array $left, array $right ): int => strcmp( (string) $left['path'], (string) $right['path'] ) ); + usort( $snapshot_files, static fn ( array $left, array $right ): int => strcmp( (string) $left['path'], (string) $right['path'] ) ); + $compiler_limits = array( + 'max_files' => min( 5000, $max_assets + ( 5 * $max_pages ) ), + 'max_file_bytes' => $fetch_args['max_bytes'], + 'max_total_bytes' => min( 335544320, $max_total_bytes + min( 67108864, $max_total_bytes ) ), + ); + $snapshot = array( + 'schema' => 'static-site-importer/url-snapshot/v1', + 'entrypoint' => $paths[ $entry_resource_url ], + 'files' => $snapshot_files, + ); + $snapshot['sha256'] = hash( 'sha256', (string) json_encode( array( 'entrypoint' => $snapshot['entrypoint'], 'compiler_limits' => $compiler_limits, 'files' => $snapshot_files ), JSON_UNESCAPED_SLASHES ) ); + + return array( + 'provider' => 'public-static-site-collector', + 'artifact' => array( + 'schema' => 'blocks-engine/php-transformer/site-artifact/v1', + 'entrypoint' => $paths[ $entry_resource_url ], + 'compiler_limits' => $compiler_limits, + 'metadata' => array( 'snapshot' => $snapshot ), + 'files' => $files, + ), + 'source_metadata' => array( + 'source_type' => 'url', + 'source_url' => $entry_url, + 'final_url' => $site_url, + 'snapshot' => $snapshot, + 'collection' => array( + 'pages' => self::resource_count( $resources, 'html' ), + 'assets' => self::resource_count( $resources, 'asset' ), + 'bytes' => $total_bytes, + 'failures' => $failures, + 'diagnostics' => $diagnostics, + 'source_exclusions' => $source_exclusions, + 'truncated' => array_keys( $truncated ), + 'sitemap_urls' => count( $sitemap_urls ), + 'external_asset_retained' => array( 'count' => count( $external_assets ), 'samples' => array_slice( array_map( static fn( string $url, string $reason ): array => array( 'url' => $url, 'reason' => $reason ), array_keys( $external_assets ), $external_assets ), 0, 50 ) ), + ), + ), + ); + } + + /** + * Discover all same-origin page routes declared by a sitemap index or urlset. + * + * @return array|WP_Error + */ + public static function discover_routes( string $url, array $args = array(), ?callable $fetcher = null ) { + $entry_url = self::canonical_url( Static_Site_Importer_URL_Fetcher::normalize_url( $url ) ); + if ( '' === $entry_url ) { + return new WP_Error( 'static_site_importer_site_collection_invalid_url', 'Enter a valid public site URL.' ); + } + $fetcher = $fetcher ?? static fn ( string $resource_url, array $fetch_args ) => Static_Site_Importer_URL_Fetcher::fetch( $resource_url, $fetch_args ); + $fetch_attempts = min( 3, max( 1, (int) ( $args['fetch_attempts'] ?? 2 ) ) ); + $fetch_resource = $fetcher; + $fetcher = static function ( string $resource_url, array $fetch_args ) use ( $fetch_resource, $fetch_attempts ) { + for ( $attempt = 0; $attempt < $fetch_attempts; $attempt++ ) { + $response = $fetch_resource( $resource_url, $fetch_args ); + if ( ! is_wp_error( $response ) ) { return $response; } + } + $data = is_wp_error( $response ) && is_array( $response->get_error_data() ) ? $response->get_error_data() : array(); + if ( ! empty( $data['_static_site_importer_cache_aware'] ) ) { + unset( $data['_static_site_importer_cache_aware'] ); + $response = new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null ); + $fetch_resource( $resource_url, $fetch_args + array( '_static_site_importer_cache_failure' => $response ) ); + } + return $response; + }; + $fetch_args = array_intersect_key( $args, array_flip( array( 'timeout' ) ) ); + $fetch_args['max_bytes'] = min( 10485760, max( 1, (int) ( $args['max_bytes'] ?? 5242880 ) ) ); + $routes = self::sitemap_urls( $entry_url, $fetcher, $fetch_args ); + if ( is_wp_error( $routes ) ) { + return $routes; + } + if ( ! empty( $routes ) ) { + return $routes; + } + // Public sites frequently omit or block sitemap.xml. Crawl HTML links from + // the entrypoint so batch mode still has a bounded useful route set. + $queue = array( $entry_url ); + $seen = array(); + while ( $queue ) { + $current = array_shift( $queue ); + $key = self::page_key( $current ); + if ( isset( $seen[ $key ] ) ) { continue; } + $response = $fetcher( $current, array_merge( $fetch_args, array( 'content_types' => array( 'text/html', 'application/xhtml+xml' ) ) ) ); + if ( is_wp_error( $response ) || '' === trim( (string) ( $response['body'] ?? '' ) ) ) { continue; } + $seen[ $key ] = $current; + $final = self::response_url( $response, $current ); + foreach ( self::html_page_urls( (string) $response['body'], self::html_base_url( (string) $response['body'], $final ), $entry_url ) as $next ) { + if ( ! isset( $seen[ self::page_key( $next ) ] ) && count( $seen ) + count( $queue ) >= self::MAX_DISCOVERED_ROUTES ) { + return new WP_Error( 'static_site_importer_discovery_incomplete', 'HTML link discovery exceeded its route limit.', array( 'truncated_dimension' => 'routes', 'limit' => self::MAX_DISCOVERED_ROUTES, 'discovered' => count( $seen ), 'queued' => count( $queue ) ) ); + } + if ( ! isset( $seen[ self::page_key( $next ) ] ) ) { $queue[] = $next; } + } + } + return array_values( $seen ); + } + + /** @return array */ + public static function discovery_limits(): array { + return array( 'max_sitemap_documents' => self::MAX_SITEMAP_DOCUMENTS, 'max_discovered_routes' => self::MAX_DISCOVERED_ROUTES, 'max_sitemap_document_bytes' => 1048576 ); + } + + /** @return array */ + private static function sitemap_urls( string $entry_url, callable $fetcher, array $fetch_args ) { + $parts = parse_url( $entry_url ); + if ( ! is_array( $parts ) || empty( $parts['scheme'] ) || empty( $parts['host'] ) ) { + return array(); + } + $origin = self::origin( $entry_url ); + $sitemap_url = $origin . '/sitemap.xml'; + $queue = array( $sitemap_url ); + $seen = array(); + $urls = array(); + while ( $queue ) { + $current = array_shift( $queue ); + if ( isset( $seen[ $current ] ) ) { + continue; + } + $seen[ $current ] = true; + $response = $fetcher( $current, array_merge( $fetch_args, array( 'max_bytes' => min( 1048576, (int) ( $fetch_args['max_bytes'] ?? 1048576 ) ), 'content_types' => array( 'application/xml', 'text/xml', 'text/plain', 'application/rss+xml' ) ) ) ); + if ( is_wp_error( $response ) ) { + continue; + } + preg_match_all( '#]*>(.*?)#is', (string) $response['body'], $matches ); + foreach ( $matches[1] ?? array() as $location ) { + $resolved = self::resolve_url( html_entity_decode( strip_tags( (string) $location ), ENT_QUOTES | ENT_HTML5, 'UTF-8' ), $current ); + if ( '' === $resolved || ! self::same_origin( $resolved, $entry_url ) ) { + continue; + } + if ( str_ends_with( strtolower( (string) parse_url( $resolved, PHP_URL_PATH ) ), '.xml' ) ) { + if ( ! isset( $seen[ $resolved ] ) && count( $seen ) + count( $queue ) >= self::MAX_SITEMAP_DOCUMENTS ) { + return new WP_Error( 'static_site_importer_discovery_incomplete', 'Sitemap discovery exceeded its document limit.', array( 'truncated_dimension' => 'sitemap_documents', 'limit' => self::MAX_SITEMAP_DOCUMENTS, 'discovered' => count( $seen ), 'queued' => count( $queue ) ) ); + } + $queue[] = $resolved; + } elseif ( self::is_page_url( $resolved ) ) { + if ( count( $urls ) >= self::MAX_DISCOVERED_ROUTES ) { + return new WP_Error( 'static_site_importer_discovery_incomplete', 'Sitemap discovery exceeded its route limit.', array( 'truncated_dimension' => 'routes', 'limit' => self::MAX_DISCOVERED_ROUTES, 'discovered' => count( $urls ) ) ); + } + $urls[] = $resolved; + } + } + } + return array_values( array_unique( $urls ) ); + } + + /** @return array */ + private static function html_page_urls( string $html, string $base_url, string $entry_url ): array { + $urls = array(); + foreach ( self::tag_attribute_values( $html, 'a', 'href' ) as $reference ) { + $url = self::resolve_url( (string) $reference, $base_url ); + if ( '' !== $url && self::same_origin( $url, $entry_url ) && self::is_page_url( $url ) ) { + $urls[] = $url; + } + } + return array_values( array_unique( $urls ) ); + } + + /** @return array */ + private static function html_asset_urls( string $html, string $base_url, bool $include_scripts ): array { + $urls = array(); + $source_urls = array_merge( + self::tag_attribute_values( $html, 'img|source|video|audio', 'src' ), + self::tag_attribute_values( $html, 'video', 'poster' ) + ); + $script_urls = $include_scripts ? self::tag_attribute_values( $html, 'script', 'src' ) : array(); + $link_urls = array(); + preg_match_all( '#]*>#is', $html, $link_matches ); + foreach ( $link_matches[0] ?? array() as $link_tag ) { + $relation = self::tag_attribute_value( $link_tag, 'rel' ); + $href = self::tag_attribute_value( $link_tag, 'href' ); + if ( null === $relation || null === $href ) { + continue; + } + $relations = preg_split( '/\s+/', strtolower( trim( $relation ) ) ); + if ( array_intersect( $relations ?: array(), array( 'stylesheet', 'icon', 'preload', 'modulepreload' ) ) ) { + $link_urls[] = $href; + } + } + foreach ( array_merge( $link_urls, $source_urls, $script_urls ) as $reference ) { + $url = self::resolve_url( (string) $reference, $base_url ); + if ( '' !== $url ) { + $urls[] = $url; + } + } + foreach ( self::tag_attribute_values( $html, 'img|source', 'srcset' ) as $srcset ) { + foreach ( explode( ',', (string) $srcset ) as $candidate ) { + $reference = preg_split( '/\s+/', trim( $candidate ) )[0] ?? ''; + $url = self::resolve_url( $reference, $base_url ); + if ( '' !== $url ) { + $urls[] = $url; + } + } + } + return array_values( array_unique( array_merge( $urls, self::html_css_asset_urls( $html, $base_url ) ) ) ); + } + + /** @return array */ + private static function html_css_asset_urls( string $html, string $base_url ): array { + $css = array(); + preg_match_all( '#]*>(.*?)#is', $html, $style_blocks ); + $css = array_merge( $css, $style_blocks[1] ?? array() ); + preg_match_all( '#<[^>]+\bstyle\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+))#is', $html, $style_attributes, PREG_SET_ORDER ); + foreach ( $style_attributes as $attribute ) { + $css[] = self::matched_attribute_value( $attribute, 1 ); + } + $urls = array(); + foreach ( $css as $source ) { + $urls = array_merge( $urls, self::css_asset_urls( (string) $source, $base_url ) ); + } + return array_values( array_unique( $urls ) ); + } + + /** @return array */ + private static function css_asset_urls( string $css, string $base_url ): array { + preg_match_all( '#url\(\s*(["\']?)(.*?)\1\s*\)#is', $css, $matches ); + $urls = array(); + foreach ( $matches[2] ?? array() as $reference ) { + $url = self::resolve_url( (string) $reference, $base_url ); + if ( '' !== $url ) { + $urls[] = $url; + } + } + preg_match_all( '#@import\s+(["\'])(.*?)\1#is', $css, $import_matches ); + foreach ( $import_matches[2] ?? array() as $reference ) { + $url = self::resolve_url( (string) $reference, $base_url ); + if ( '' !== $url ) { + $urls[] = $url; + } + } + return array_values( array_unique( $urls ) ); + } + + /** @param array> $resources @return array */ + private static function artifact_paths( array $resources, string $entry_url ): array { + ksort( $resources, SORT_STRING ); + $paths = array(); + $used = array(); + foreach ( $resources as $resource_url => $resource ) { + $path = self::artifact_path( $resource_url, 'html' === $resource['kind'], $entry_url ); + if ( isset( $used[ $path ] ) ) { + $extension = pathinfo( $path, PATHINFO_EXTENSION ); + $suffix = '-' . substr( hash( 'sha256', $resource_url ), 0, 10 ); + $path = '' !== $extension ? substr( $path, 0, -1 - strlen( $extension ) ) . $suffix . '.' . $extension : $path . $suffix; + } + $used[ $path ] = true; + $paths[ $resource_url ] = $path; + } + return $paths; + } + + /** @param array> $resources @return array */ + private static function route_paths( array $resources ): array { + ksort( $resources, SORT_STRING ); + $paths = array(); + $used = array(); + foreach ( $resources as $resource_url => $resource ) { + if ( 'html' !== $resource['kind'] ) { + continue; + } + $path = self::canonical_route_path( $resource_url ); + if ( isset( $used[ $path ] ) ) { + $path = rtrim( $path, '/' ) . '-' . substr( hash( 'sha256', $resource_url ), 0, 10 ); + } + $used[ $path ] = true; + $paths[ $resource_url ] = $path; + } + return $paths; + } + + private static function artifact_path( string $url, bool $html, string $entry_url ): string { + $parts = parse_url( $url ); + $path = isset( $parts['path'] ) ? rawurldecode( (string) $parts['path'] ) : '/'; + $path = implode( '/', array_map( static fn ( string $segment ): string => sanitize_file_name( $segment ), array_filter( explode( '/', trim( $path, '/' ) ), 'strlen' ) ) ); + if ( ! self::same_origin( $url, $entry_url ) ) { + $host = sanitize_file_name( strtolower( (string) ( $parts['host'] ?? 'external' ) ) ); + $path = '_external/' . $host . '/' . $path; + } + if ( $html ) { + if ( '' === $path || 'index.html' === strtolower( $path ) || 'index.htm' === strtolower( $path ) ) { + $path = 'index.html'; + } elseif ( ! preg_match( '/\.html?$/i', $path ) ) { + $path = rtrim( $path, '/' ) . '/index.html'; + } + } elseif ( '' === $path ) { + $path = 'asset-' . substr( hash( 'sha256', $url ), 0, 12 ); + } + if ( isset( $parts['query'] ) && '' !== $parts['query'] ) { + $extension = pathinfo( $path, PATHINFO_EXTENSION ); + $suffix = '-' . substr( hash( 'sha256', (string) $parts['query'] ), 0, 8 ); + $path = '' !== $extension ? substr( $path, 0, -1 - strlen( $extension ) ) . $suffix . '.' . $extension : $path . $suffix; + } + return 'website/' . ltrim( $path, '/' ); + } + + /** @param array $paths */ + private static function rewrite_html( string $html, string $base_url, string $source_path, array $paths, array $aliases, string $site_url, array $external_assets = array() ): string { + $html = preg_replace_callback( + '#\b(src|href|poster)\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+))#is', + static function ( array $match ) use ( $base_url, $source_path, $paths, $aliases, $site_url, $external_assets ): string { + $value = self::matched_attribute_value( $match ); + $url = self::resolve_url( $value, $base_url ); + if ( isset( $paths[ $url ] ) && preg_match( '/\.html?$/i', $paths[ $url ] ) ) { + if ( isset( $aliases[ $url ] ) ) { + return $match[1] . '="' . self::route_url( $aliases[ $url ], $value ) . '"'; + } + return $match[0]; + } + if ( isset( $paths[ $url ] ) ) { + return $match[1] . '="' . self::relative_path( $source_path, $paths[ $url ] ) . '"'; + } + if ( isset( $external_assets[ $url ] ) ) { return $match[1] . '="' . self::external_asset_url( $url, $value ) . '"'; } + return '' !== $url && self::same_origin( $url, $site_url ) && self::is_page_url( $url ) ? $match[1] . '="' . self::route_url( $url, $value ) . '"' : $match[0]; + }, + $html + ); + $html = preg_replace_callback( + '#\bsrcset\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+))#is', + static function ( array $match ) use ( $base_url, $source_path, $paths, $external_assets ): string { + $candidates = array(); + foreach ( explode( ',', self::matched_attribute_value( $match, 1 ) ) as $candidate ) { + $parts = preg_split( '/\s+/', trim( $candidate ), 2 ); + $url = self::resolve_url( $parts[0] ?? '', $base_url ); + $ref = isset( $paths[ $url ] ) ? self::relative_path( $source_path, $paths[ $url ] ) : ( isset( $external_assets[ $url ] ) ? self::external_asset_url( $url, (string) ( $parts[0] ?? '' ) ) : ( $parts[0] ?? '' ) ); + $candidates[] = trim( $ref . ' ' . ( $parts[1] ?? '' ) ); + } + return 'srcset="' . implode( ', ', $candidates ) . '"'; + }, + (string) $html + ); + return self::rewrite_css( (string) $html, $base_url, $source_path, $paths, $external_assets ); + } + + /** @param array $paths */ + private static function rewrite_css( string $css, string $base_url, string $source_path, array $paths, array $external_assets = array() ): string { + $css = (string) preg_replace_callback( + '#url\(\s*(["\']?)(.*?)\1\s*\)#is', + static function ( array $match ) use ( $base_url, $source_path, $paths, $external_assets ): string { + $url = self::resolve_url( $match[2], $base_url ); + return isset( $paths[ $url ] ) ? 'url(' . $match[1] . self::relative_path( $source_path, $paths[ $url ] ) . $match[1] . ')' : ( isset( $external_assets[ $url ] ) ? 'url(' . $match[1] . self::external_asset_url( $url, $match[2] ) . $match[1] . ')' : $match[0] ); + }, + $css + ); + return (string) preg_replace_callback( + '#@import\s+(["\'])(.*?)\1#is', + static function ( array $match ) use ( $base_url, $source_path, $paths, $external_assets ): string { + $url = self::resolve_url( $match[2], $base_url ); + return isset( $paths[ $url ] ) ? '@import ' . $match[1] . self::relative_path( $source_path, $paths[ $url ] ) . $match[1] : ( isset( $external_assets[ $url ] ) ? '@import ' . $match[1] . self::external_asset_url( $url, $match[2] ) . $match[1] : $match[0] ); + }, + $css + ); + } + + private static function external_asset_url( string $url, string $reference ): string { $fragment = parse_url( html_entity_decode( $reference, ENT_QUOTES | ENT_HTML5, 'UTF-8' ), PHP_URL_FRAGMENT ); return $url . ( is_string( $fragment ) && '' !== $fragment ? '#' . $fragment : '' ); } + + private static function response_url( array $response, string $requested_url ): string { + $final_url = self::canonical_url( (string) ( $response['metadata']['final_url'] ?? '' ) ); + return '' !== $final_url ? $final_url : $requested_url; + } + + private static function html_base_url( string $html, string $document_url ): string { + preg_match( '#]*>#is', $html, $match ); + $base = isset( $match[0] ) ? self::tag_attribute_value( $match[0], 'href' ) : null; + if ( null === $base ) { + return $document_url; + } + $resolved = self::resolve_url( $base, $document_url ); + return '' !== $resolved ? $resolved : $document_url; + } + + /** @return array */ + private static function tag_attribute_values( string $html, string $tags, string $attribute ): array { + preg_match_all( '#<(?:' . $tags . ')\b[^>]*>#is', $html, $matches ); + $values = array(); + foreach ( $matches[0] ?? array() as $tag ) { + $value = self::tag_attribute_value( $tag, $attribute ); + if ( null !== $value ) { + $values[] = $value; + } + } + return $values; + } + + private static function tag_attribute_value( string $tag, string $attribute ): ?string { + $pattern = '#\b' . preg_quote( $attribute, '#' ) . '\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+))#is'; + if ( ! preg_match( $pattern, $tag, $match ) ) { + return null; + } + return self::matched_attribute_value( $match, 1 ); + } + + private static function matched_attribute_value( array $match, int $offset = 2 ): string { + foreach ( array_slice( $match, $offset, 3 ) as $value ) { + if ( '' !== (string) $value ) { + return (string) $value; + } + } + return ''; + } + + private static function route_url( string $url, string $original_reference ): string { + $parts = parse_url( $url ); + $route = (string) ( $parts['path'] ?? '/' ); + $route .= isset( $parts['query'] ) ? '?' . $parts['query'] : ''; + $fragment = parse_url( html_entity_decode( $original_reference, ENT_QUOTES | ENT_HTML5, 'UTF-8' ), PHP_URL_FRAGMENT ); + return $route . ( is_string( $fragment ) && '' !== $fragment ? '#' . $fragment : '' ); + } + + private static function relative_path( string $from, string $to ): string { + $from_segments = explode( '/', trim( dirname( $from ), './' ) ); + $to_segments = explode( '/', trim( $to, '/' ) ); + while ( $from_segments && $to_segments && $from_segments[0] === $to_segments[0] ) { + array_shift( $from_segments ); + array_shift( $to_segments ); + } + return str_repeat( '../', count( array_filter( $from_segments, 'strlen' ) ) ) . implode( '/', $to_segments ); + } + + private static function resolve_url( string $reference, string $base_url ): string { + $reference = trim( html_entity_decode( $reference, ENT_QUOTES | ENT_HTML5, 'UTF-8' ), " \t\n\r\0\x0B\"'" ); + if ( '' === $reference || str_starts_with( $reference, '#' ) || preg_match( '#^(?:data|javascript|mailto|tel|blob):#i', $reference ) ) { + return ''; + } + if ( preg_match( '#^https?://#i', $reference ) ) { + return self::canonical_url( $reference ); + } + $base = parse_url( $base_url ); + if ( ! is_array( $base ) || empty( $base['scheme'] ) || empty( $base['host'] ) ) { + return ''; + } + if ( str_starts_with( $reference, '//' ) ) { + return self::canonical_url( $base['scheme'] . ':' . $reference ); + } + $origin = self::origin( $base_url ); + if ( str_starts_with( $reference, '/' ) ) { + return self::canonical_url( $origin . $reference ); + } + $base_path = (string) ( $base['path'] ?? '/' ); + return self::canonical_url( $origin . preg_replace( '#/[^/]*$#', '/', $base_path ) . $reference ); + } + + private static function canonical_url( string $url ): string { + $parts = parse_url( trim( $url ) ); + if ( ! is_array( $parts ) || ! in_array( strtolower( (string) ( $parts['scheme'] ?? '' ) ), array( 'http', 'https' ), true ) || empty( $parts['host'] ) ) { + return ''; + } + $path = self::normalize_path( (string) ( $parts['path'] ?? '/' ) ); + $port = isset( $parts['port'] ) ? ':' . (int) $parts['port'] : ''; + $query = isset( $parts['query'] ) && '' !== $parts['query'] ? '?' . $parts['query'] : ''; + return strtolower( (string) $parts['scheme'] ) . '://' . strtolower( (string) $parts['host'] ) . $port . $path . $query; + } + + private static function normalize_path( string $path ): string { + $segments = array(); + foreach ( explode( '/', '/' . ltrim( $path, '/' ) ) as $segment ) { + if ( '' === $segment || '.' === $segment ) { + continue; + } + if ( '..' === $segment ) { + array_pop( $segments ); + continue; + } + $segments[] = $segment; + } + $normalized = '/' . implode( '/', $segments ); + return str_ends_with( $path, '/' ) && '/' !== $normalized ? $normalized . '/' : $normalized; + } + + private static function canonical_route_path( string $url ): string { + $path = (string) ( parse_url( $url, PHP_URL_PATH ) ?? '/' ); + $segments = array_values( array_filter( explode( '/', trim( $path, '/' ) ), static fn ( string $segment ): bool => '' !== $segment ) ); + $last = array_key_last( $segments ); + $slugs = array(); + foreach ( $segments as $index => $segment ) { + $decoded = urldecode( $segment ); + if ( $index === $last ) { + $decoded = (string) preg_replace( '/\.html?$/i', '', $decoded ); + } + $slug = function_exists( 'sanitize_title' ) + ? sanitize_title( $decoded ) + : trim( strtolower( (string) preg_replace( '/[^a-z0-9]+/i', '-', $decoded ) ), '-' ); + if ( '' !== $slug ) { + $slugs[] = $slug; + } + } + return array() === $slugs ? '/' : '/' . implode( '/', $slugs ); + } + + private static function origin( string $url ): string { + $parts = parse_url( $url ); + return strtolower( (string) $parts['scheme'] ) . '://' . strtolower( (string) $parts['host'] ) . ( isset( $parts['port'] ) ? ':' . (int) $parts['port'] : '' ); + } + + private static function same_origin( string $left, string $right ): bool { + return self::origin( $left ) === self::origin( $right ); + } + + private static function is_page_url( string $url ): bool { + $path = strtolower( (string) parse_url( $url, PHP_URL_PATH ) ); + $extension = pathinfo( $path, PATHINFO_EXTENSION ); + return '' === $extension || in_array( $extension, array( 'html', 'htm', 'php', 'asp', 'aspx' ), true ); + } + + private static function page_key( string $url ): string { + $parts = parse_url( $url ); + $path = strtolower( rtrim( (string) ( $parts['path'] ?? '/' ), '/' ) ); + if ( '' === $path || '/index.html' === $path || '/index.htm' === $path ) { + $path = '/'; + } + return self::origin( $url ) . $path . ( isset( $parts['query'] ) ? '?' . $parts['query'] : '' ); + } + + /** @param array> $resources */ + private static function resource_count( array $resources, string $kind ): int { + return count( array_filter( $resources, static fn ( array $resource ): bool => $kind === $resource['kind'] ) ); + } + + private static function content_type( array $response, string $fallback ): string { + $content_type = strtolower( trim( explode( ';', (string) ( $response['metadata']['content_type'] ?? '' ), 2 )[0] ) ); + return '' !== $content_type ? $content_type : $fallback; + } + + private static function is_text( string $content_type, string $path ): bool { + return str_starts_with( $content_type, 'text/' ) || in_array( $content_type, array( 'application/javascript', 'application/json', 'application/xml', 'image/svg+xml' ), true ) || (bool) preg_match( '/\.(?:css|js|json|xml|svg)$/i', $path ); + } + + /** @return array */ + private static function failure( string $url, WP_Error $error, string $kind = 'asset' ): array { + return array( + 'url' => $url, + 'kind' => $kind, + 'code' => $error->get_error_code(), + 'message' => $error->get_error_message(), + ); + } + + private static function delay_after_fetch( $response, int $milliseconds, array $args ): void { + $error_data = is_wp_error( $response ) ? $response->get_error_data() : null; + if ( is_array( $error_data ) && ! empty( $error_data['_static_site_importer_negative_cache_hit'] ) ) { + return; + } + if ( is_array( $response ) && ! empty( $response['metadata']['_static_site_importer_cache_hit'] ) ) { + unset( $response['metadata']['_static_site_importer_cache_hit'] ); + return; + } + if ( isset( $args['_static_site_importer_delay_callback'] ) && is_callable( $args['_static_site_importer_delay_callback'] ) ) { + call_user_func( $args['_static_site_importer_delay_callback'] ); + return; + } + self::delay( $milliseconds ); + } + + private static function without_cache_marker( $response ) { + if ( ! is_wp_error( $response ) ) { + return $response; + } + $data = is_array( $response->get_error_data() ) ? $response->get_error_data() : array(); + if ( empty( $data['_static_site_importer_negative_cache_hit'] ) ) { + return $response; + } + unset( $data['_static_site_importer_negative_cache_hit'] ); + return new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null ); + } + + private static function delay( int $milliseconds ): void { + if ( $milliseconds > 0 ) { + usleep( $milliseconds * 1000 ); + } + } +} diff --git a/includes/class-static-site-importer-validation-runtime.php b/includes/class-static-site-importer-validation-runtime.php index 9be274f0..ae61f3b1 100644 --- a/includes/class-static-site-importer-validation-runtime.php +++ b/includes/class-static-site-importer-validation-runtime.php @@ -94,6 +94,7 @@ public static function validate_artifact( array $input ) { 'activate' => true, 'overwrite' => true, 'materialize_dependencies' => true, + 'require_proven_dynamic_client_assets' => true, ) ); diff --git a/includes/class-static-site-importer-website-artifact-import-input.php b/includes/class-static-site-importer-website-artifact-import-input.php index 39b1f3af..5b43fdea 100644 --- a/includes/class-static-site-importer-website-artifact-import-input.php +++ b/includes/class-static-site-importer-website-artifact-import-input.php @@ -26,6 +26,7 @@ class Static_Site_Importer_Website_Artifact_Import_Input { 'allow_missing_woocommerce' => array( 'type' => 'boolean' ), 'allow_missing_jetpack' => array( 'type' => 'boolean' ), 'materialize_dependencies' => array( 'type' => 'boolean' ), + 'require_proven_dynamic_client_assets' => array( 'type' => 'boolean' ), 'seed_entities' => array( 'type' => 'boolean' ), 'products_manifest' => array( 'type' => 'object' ), 'commerce_context' => array( 'type' => 'object' ), @@ -58,6 +59,7 @@ public static function normalize( array $input, array $defaults = array() ): arr 'allow_missing_woocommerce' => false, 'allow_missing_jetpack' => false, 'materialize_dependencies' => true, + 'require_proven_dynamic_client_assets' => true, 'seed_entities' => false, 'products_manifest' => array(), 'commerce_context' => array(), @@ -81,7 +83,7 @@ public static function normalize( array $input, array $defaults = array() ): arr foreach ( array( 'slug', 'name', 'site_title', 'stale_page_action', 'report', 'asset_materialization_policy' ) as $field ) { $values[ $field ] = is_scalar( $values[ $field ] ) ? (string) $values[ $field ] : ''; } - foreach ( array( 'activate', 'overwrite', 'fail_on_quality', 'allow_missing_woocommerce', 'allow_missing_jetpack', 'materialize_dependencies', 'seed_entities', 'write_theme_report_artifacts' ) as $field ) { + foreach ( array( 'activate', 'overwrite', 'fail_on_quality', 'allow_missing_woocommerce', 'allow_missing_jetpack', 'materialize_dependencies', 'require_proven_dynamic_client_assets', 'seed_entities', 'write_theme_report_artifacts' ) as $field ) { $values[ $field ] = (bool) $values[ $field ]; } foreach ( array( 'products_manifest', 'commerce_context', 'asset_map', 'compiler_options', 'source_metadata', 'validation_artifacts' ) as $field ) { diff --git a/includes/class-static-site-importer-wordpress-site-plan-materializer.php b/includes/class-static-site-importer-wordpress-site-plan-materializer.php index 028aeeee..9a505965 100644 --- a/includes/class-static-site-importer-wordpress-site-plan-materializer.php +++ b/includes/class-static-site-importer-wordpress-site-plan-materializer.php @@ -72,10 +72,12 @@ public static function prepare( array $plan, array $args = array() ): array { $theme_uri = trailingslashit( get_theme_root_uri() ) . $slug; try { // Resolver proof is canonical semantics, not an inference from copied files. - $resolved = ( new WordPressSitePlanResolver() )->resolve( $plan, array( 'theme_uri' => $theme_uri, 'require_proven_dynamic_client_assets' => true, 'runtime_capabilities' => array( 'asset_materialization' ) ) ); + $resolved = ( new WordPressSitePlanResolver() )->resolve( $plan, array( 'theme_uri' => $theme_uri, 'require_proven_dynamic_client_assets' => $args['require_proven_dynamic_client_assets'] ?? true, 'runtime_capabilities' => array( 'asset_materialization' ) ) ); } catch ( InvalidArgumentException $error ) { throw new InvalidArgumentException( $error->getMessage(), 0, $error ); } + $state['base_resolved'] = $resolved; + $state['base_resolved_hash'] = self::hash( $resolved ); $state['resolved'] = $resolved; self::apply_runtime_entity_bindings( $state['resolved'], isset( $args['runtime_entity_bindings'] ) && is_array( $args['runtime_entity_bindings'] ) ? $args['runtime_entity_bindings'] : array(), $state['applied']['runtime_declarations']['entity_bindings'] ); $state['theme_dir'] = $theme_dir; @@ -84,13 +86,14 @@ public static function prepare( array $plan, array $args = array() ): array { 'dir' => $theme_dir, 'uri' => $theme_uri, ); - self::preflight_state( $state, ! empty( $args['overwrite'] ) ); + self::preflight_state( $state, ! empty( $args['overwrite'] ), (string) ( $args['import_run_id'] ?? '' ) ); } catch ( InvalidArgumentException $error ) { $state['diagnostics'][] = array( 'reason_code' => $error->getMessage() ); return array( 'status' => 'rejected', 'receipt' => self::receipt( 'rejected', $state ) ); } $state['status'] = 'prepared'; $state['args'] = $args; + $state['preparation'] = array( 'canonical_validations' => 1, 'plan_resolutions' => 1, 'destination_preflights' => 1, 'immutable_projection_reused' => false ); return $state; } @@ -99,7 +102,7 @@ public static function materialize_prepared( array $prepared ): array { if ( 'prepared' !== ( $prepared['status'] ?? '' ) || ! isset( $prepared['plan'] ) || ! is_array( $prepared['plan'] ) ) { return self::receipt( 'rejected', array( 'plan' => array(), 'plan_hash' => '', 'diagnostics' => array( array( 'reason_code' => 'invalid_prepared_state' ) ), 'applied' => array( 'posts' => array(), 'files' => array(), 'operations' => array(), 'runtime_declarations' => array( 'asset_publications' => array(), 'entity_bindings' => array() ) ), 'skipped' => array(), 'existing_matches' => array( 'pages' => array() ) ) ); } - $state = self::prepare( $prepared['plan'], isset( $prepared['args'] ) && is_array( $prepared['args'] ) ? $prepared['args'] : array() ); + $state = self::refresh_prepared_destination( $prepared ); if ( 'prepared' !== ( $state['status'] ?? '' ) ) { return $state['receipt']; } @@ -116,7 +119,7 @@ public static function materialize_prepared( array $prepared ): array { if ( ! empty( $page['skip_materialization'] ) ) { continue; } - $post = self::materialize_page( $page, $state['source_ids'] ); + $post = self::materialize_page( $page, $state['source_ids'], (string) ( $args['import_run_id'] ?? '' ) ); if ( is_wp_error( $post ) ) { return self::failed_receipt( $state, $post->get_error_code() ); } @@ -135,6 +138,17 @@ public static function materialize_prepared( array $prepared ): array { } foreach ( $state['resolved']['writes'] as $write ) { + if ( ! empty( $args['preserve_existing_theme_bootstrap'] ) && 'theme_bootstrap' === ( $write['kind'] ?? '' ) && is_file( $state['theme_dir'] . '/' . $write['target_path'] ) ) { + $result = self::merge_batch_bootstrap( $state['theme_dir'], $write ); + if ( is_wp_error( $result ) ) { + return self::failed_receipt( $state, $result->get_error_code() ); + } + $state['applied']['files'][] = $result; + continue; + } + if ( ! empty( $args['preserve_existing_theme_bootstrap'] ) && in_array( $write['kind'] ?? '', array( 'theme_scaffold', 'theme_bootstrap', 'theme_template' ), true ) && is_file( $state['theme_dir'] . '/' . $write['target_path'] ) ) { + continue; + } $result = self::write_file( $state['theme_dir'], $write ); if ( is_wp_error( $result ) ) { return self::failed_receipt( $state, $result->get_error_code() ); @@ -183,8 +197,67 @@ public static function materialize_prepared( array $prepared ): array { return self::receipt( 'completed', $state ); } + /** + * Recheck mutable destinations without repeating canonical validation and resolution. + * + * @param array $prepared Previously validated immutable projection. + * @return array + */ + private static function refresh_prepared_destination( array $prepared ): array { + $plan = $prepared['plan'] ?? null; + $base_resolved = $prepared['base_resolved'] ?? null; + $args = isset( $prepared['args'] ) && is_array( $prepared['args'] ) ? $prepared['args'] : array(); + if ( ! is_array( $plan ) || ! is_array( $base_resolved ) || self::hash( $plan ) !== ( $prepared['plan_hash'] ?? '' ) || self::hash( $base_resolved ) !== ( $prepared['base_resolved_hash'] ?? '' ) ) { + return array( 'status' => 'rejected', 'receipt' => self::receipt( 'rejected', array( 'plan' => is_array( $plan ) ? $plan : array(), 'plan_hash' => '', 'diagnostics' => array( array( 'reason_code' => 'prepared_projection_changed' ) ), 'applied' => array( 'posts' => array(), 'files' => array(), 'operations' => array(), 'runtime_declarations' => array( 'asset_publications' => array(), 'entity_bindings' => array() ) ), 'skipped' => array(), 'existing_matches' => array( 'pages' => array() ) ) ) ); + } + + $slug = sanitize_key( (string) ( $args['slug'] ?? '' ) ); + $theme_root = get_theme_root(); + $theme_uri = trailingslashit( get_theme_root_uri() ) . $slug; + $theme_dir = is_string( $theme_root ) ? trailingslashit( $theme_root ) . $slug : ''; + $state = array( + 'plan' => $plan, + 'plan_hash' => $prepared['plan_hash'], + 'base_resolved' => $base_resolved, + 'base_resolved_hash' => $prepared['base_resolved_hash'], + 'resolved' => $base_resolved, + 'diagnostics' => array(), + 'applied' => array( 'posts' => array(), 'files' => array(), 'operations' => array(), 'runtime_declarations' => array( 'asset_publications' => array(), 'entity_bindings' => array() ) ), + 'skipped' => array(), + 'existing_matches' => array( 'pages' => array() ), + 'report_destinations' => isset( $args['report_destinations'] ) && is_array( $args['report_destinations'] ) ? $args['report_destinations'] : array(), + 'theme_dir' => $theme_dir, + 'theme' => array( 'slug' => $slug, 'dir' => $theme_dir, 'uri' => $theme_uri ), + 'args' => $args, + 'preparation' => array( 'canonical_validations' => 1, 'plan_resolutions' => 1, 'destination_preflights' => 2, 'immutable_projection_reused' => true ), + ); + + try { + if ( '' === $slug ) { + throw new InvalidArgumentException( 'invalid_theme_slug' ); + } + if ( ! is_string( $theme_root ) || ! is_dir( $theme_root ) || ! is_writable( $theme_root ) ) { + throw new InvalidArgumentException( 'theme_destination_not_ready' ); + } + if ( is_link( $theme_dir ) || ( file_exists( $theme_dir ) && ! is_dir( $theme_dir ) ) ) { + throw new InvalidArgumentException( 'unsafe_theme_destination' ); + } + if ( $theme_dir !== ( $prepared['theme']['dir'] ?? null ) || $theme_uri !== ( $prepared['theme']['uri'] ?? null ) ) { + throw new InvalidArgumentException( 'prepared_destination_changed' ); + } + self::apply_runtime_entity_bindings( $state['resolved'], isset( $args['runtime_entity_bindings'] ) && is_array( $args['runtime_entity_bindings'] ) ? $args['runtime_entity_bindings'] : array(), $state['applied']['runtime_declarations']['entity_bindings'] ); + self::preflight_state( $state, ! empty( $args['overwrite'] ), (string) ( $args['import_run_id'] ?? '' ) ); + } catch ( InvalidArgumentException $error ) { + $state['diagnostics'][] = array( 'reason_code' => $error->getMessage() ); + return array( 'status' => 'rejected', 'receipt' => self::receipt( 'rejected', $state ) ); + } + + $state['status'] = 'prepared'; + return $state; + } + /** @param array $state */ - private static function preflight_state( array &$state, bool $overwrite ): void { + private static function preflight_state( array &$state, bool $overwrite, string $import_run_id = '' ): void { $pages_by_route = array(); $state['page_ids'] = array(); $state['source_ids'] = array(); @@ -204,7 +277,7 @@ private static function preflight_state( array &$state, bool $overwrite ): void continue; } $conflict = '' === trim( $route, '/' ) ? null : get_page_by_path( trim( $route, '/' ), OBJECT, 'page' ); - if ( $conflict && ! $overwrite ) { + if ( $conflict && ! $overwrite && ! self::post_belongs_to_run( $conflict, $import_run_id ) ) { throw new InvalidArgumentException( 'post_conflict' ); } if ( $conflict ) { @@ -212,7 +285,7 @@ private static function preflight_state( array &$state, bool $overwrite ): void $state['resolved']['pages'][ array_search( $page['source_path'], array_column( $state['resolved']['pages'], 'source_path' ), true ) ] = $page; } } - $state['ordered_pages'] = self::parent_ordered_pages( $state['resolved']['pages'] ); + $state['ordered_pages'] = self::parent_ordered_pages( $state['resolved']['pages'], $import_run_id ); if ( null === $state['ordered_pages'] ) { throw new InvalidArgumentException( 'invalid_page_parent_identity' ); } @@ -229,7 +302,7 @@ private static function preflight_state( array &$state, bool $overwrite ): void if ( ! self::safe_destination( $state['theme_dir'], $write['target_path'] ) ) { throw new InvalidArgumentException( 'unsafe_destination_path' ); } - if ( is_dir( $path ) || ( file_exists( $path ) && ! $overwrite && self::file_hash( $path ) !== self::payload_hash( $write ) ) ) { + if ( is_dir( $path ) || ( file_exists( $path ) && ! $overwrite && ! self::theme_belongs_to_run( $state['theme_dir'], $import_run_id ) && self::file_hash( $path ) !== self::payload_hash( $write ) ) ) { throw new InvalidArgumentException( 'file_conflict' ); } } @@ -245,10 +318,10 @@ private static function preflight_state( array &$state, bool $overwrite ): void } /** @param array $page @param array $source_ids */ - private static function materialize_page( array $page, array $source_ids ) { - $parent = '' === $page['parent_source_path'] ? 0 : ( $source_ids[ $page['parent_source_path'] ] ?? false ); - if ( false === $parent ) { - return new WP_Error( 'missing_parent_page' ); + private static function materialize_page( array $page, array $source_ids, string $import_run_id = '' ) { + $parent = '' === $page['parent_source_path'] ? 0 : ( $source_ids[ $page['parent_source_path'] ] ?? self::existing_source_page_id( $page['parent_source_path'], $import_run_id ) ); + if ( $parent <= 0 && '' !== $page['parent_source_path'] ) { + return new WP_Error( 'missing_parent_page', 'The parent route has not been materialized by this import run.', array( 'source_path' => $page['source_path'], 'parent_source_path' => $page['parent_source_path'] ) ); } $post = array( 'ID' => (int) ( $page['planned_existing_id'] ?? 0 ), @@ -362,6 +435,33 @@ private static function write_file( string $theme_dir, array $write ) { return array( 'target_path' => $write['target_path'], 'hash' => self::file_hash( $path ), 'payload_hash' => $write['payload_hash'] ?? hash( 'sha256', $data ), 'reconciliation_identity' => $write['reconciliation_identity'] ?? hash( 'sha256', $write['source_path'] . "\n" . $write['target_path'] ) ); } + /** Merge a validated later-batch bootstrap as an idempotent PHP include. */ + private static function merge_batch_bootstrap( string $theme_dir, array $write ) { + $bootstrap = 'base64' === $write['payload']['encoding'] ? base64_decode( $write['payload']['data'], true ) : $write['payload']['data']; + if ( false === $bootstrap || ! is_string( $bootstrap ) || ! str_starts_with( ltrim( $bootstrap ), ' $include, 'source_path' => $write['source_path'], 'payload' => array( 'encoding' => 'utf8', 'data' => $bootstrap ), 'payload_hash' => $hash ) ); + if ( is_wp_error( $include_write ) ) { + return $include_write; + } + $functions_write = self::write_file( $theme_dir, array( 'target_path' => $write['target_path'], 'source_path' => $write['source_path'], 'payload' => array( 'encoding' => 'utf8', 'data' => $current . $require ), 'payload_hash' => hash( 'sha256', $current . $require ) ) ); + if ( is_wp_error( $functions_write ) ) { + return $functions_write; + } + } + return array( 'target_path' => $write['target_path'], 'hash' => self::file_hash( $functions ), 'payload_hash' => hash( 'sha256', (string) file_get_contents( $functions ) ), 'reconciliation_identity' => $write['reconciliation_identity'] ?? hash( 'sha256', $write['source_path'] . "\n" . $write['target_path'] ) ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reports the merged bootstrap payload. + } + /** @param array $state @param array{writes:array>,diagnostics:array>} $overlay */ private static function apply_font_overlay( array &$state, array $overlay ) { $reports = array(); @@ -565,6 +665,24 @@ private static function reconciled_post( string $identity ) { return isset( $posts[0] ) ? $posts[0] : null; } + private static function existing_source_page_id( string $source_path, string $import_run_id ): int { + if ( '' === $import_run_id ) { return 0; } + $posts = get_posts( array( 'post_type' => 'page', 'post_status' => 'any', 'meta_key' => '_static_site_importer_provenance', 'numberposts' => -1 ) ); + foreach ( $posts as $post ) { + $provenance = json_decode( (string) get_post_meta( $post->ID, '_static_site_importer_provenance', true ), true ); + if ( is_array( $provenance ) && $source_path === ( $provenance['source_path'] ?? '' ) && $import_run_id === ( $provenance['import_run_id'] ?? '' ) ) { + return (int) $post->ID; + } + } + return 0; + } + + private static function post_belongs_to_run( WP_Post $post, string $import_run_id ): bool { + if ( '' === $import_run_id ) { return false; } + $provenance = json_decode( (string) get_post_meta( $post->ID, '_static_site_importer_provenance', true ), true ); + return is_array( $provenance ) && $import_run_id === ( $provenance['import_run_id'] ?? '' ); + } + /** @param array> $pages */ private static function page_exists_in_plan( array $pages, string $identity ): bool { foreach ( $pages as $page ) { @@ -576,19 +694,23 @@ private static function page_exists_in_plan( array $pages, string $identity ): b } /** @param array> $pages @return array>|null */ - private static function parent_ordered_pages( array $pages ): ?array { + private static function parent_ordered_pages( array $pages, string $import_run_id = '' ): ?array { $remaining = array(); foreach ( $pages as $page ) { $remaining[ $page['source_path'] ] = $page; } $ordered = array(); + $external_parents = array(); while ( ! empty( $remaining ) ) { $progress = false; foreach ( $remaining as $source => $page ) { $parent = $page['parent_source_path']; - if ( '' !== $parent && ! isset( $ordered[ $parent ] ) ) { + if ( '' !== $parent && ! isset( $ordered[ $parent ] ) && ! isset( $external_parents[ $parent ] ) ) { if ( ! isset( $remaining[ $parent ] ) ) { - return null; + if ( self::existing_source_page_id( $parent, $import_run_id ) <= 0 ) { return null; } + $external_parents[ $parent ] = true; + $progress = true; + continue; } continue; } @@ -603,6 +725,13 @@ private static function parent_ordered_pages( array $pages ): ?array { return array_values( $ordered ); } + private static function theme_belongs_to_run( string $theme_dir, string $import_run_id ): bool { + if ( '' === $import_run_id ) { return false; } + $manifest = $theme_dir . '/static-site-importer-manifest.json'; + $data = is_file( $manifest ) ? json_decode( (string) file_get_contents( $manifest ), true ) : null; // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reads importer-owned run manifest for batch reconciliation. + return is_array( $data ) && $import_run_id === ( $data['import_run_id'] ?? '' ); + } + private static function safe_destination( string $theme_dir, string $target ): bool { $current = rtrim( $theme_dir, '/' ); foreach ( explode( '/', dirname( $target ) ) as $segment ) { @@ -707,6 +836,7 @@ private static function receipt( string $status, array $state ): array { 'operations' => $state['applied']['operations'], 'skipped_targets' => $state['skipped'], 'existing_matches' => $state['existing_matches'], + 'preparation' => $state['preparation'] ?? array(), 'diagnostics' => $state['diagnostics'], 'errors' => $errors, ); diff --git a/includes/source-exclusion-rules.json b/includes/source-exclusion-rules.json new file mode 100644 index 00000000..8e1c7b1b --- /dev/null +++ b/includes/source-exclusion-rules.json @@ -0,0 +1,12 @@ +{ + "schema": "static-site-importer/source-exclusion-rules/v1", + "rules": [ + { + "id": "weebly-footer-signup", + "selector": "#weebly-footer-signup-container-v3", + "category": "platform_attribution", + "provider": "weebly", + "reason_code": "platform_attribution_removed" + } + ] +} diff --git a/lib/artifact-intake.mjs b/lib/artifact-intake.mjs index 8c837b2b..7d781d6a 100644 --- a/lib/artifact-intake.mjs +++ b/lib/artifact-intake.mjs @@ -4,6 +4,8 @@ import fs from 'node:fs'; import path from 'node:path'; +import { GENERATED_ARTIFACT_METADATA_FILENAME } from './fixture-matrix/shared/constants.mjs'; + const DEFAULT_ENTRYPOINT = 'index.html'; const DEFAULT_MAX_DEPTH = 3; @@ -89,7 +91,7 @@ export function discoverGeneratedArtifacts(root, options = {}) { } function readWebsiteArtifact(directory) { - for (const name of ['artifact.json', 'website-artifact.json', 'static-site-candidate.json']) { + for (const name of ['artifact.json', 'site-artifact.json', 'website-artifact.json', 'static-site-candidate.json']) { const filePath = path.join(directory, name); if (!fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) { continue; @@ -123,6 +125,13 @@ function materializeWebsiteArtifact(artifact, fixtureDirectory) { fs.writeFileSync(destination, Buffer.from(file.content_base64, 'base64')); } } + + if (artifact.compiler_limits && typeof artifact.compiler_limits === 'object' && !Array.isArray(artifact.compiler_limits)) { + fs.writeFileSync(path.join(fixtureDirectory, GENERATED_ARTIFACT_METADATA_FILENAME), JSON.stringify({ + schema: 'static-site-importer/generated-artifact-metadata/v1', + compiler_limits: artifact.compiler_limits, + })); + } } function copyDirectory(source, destination) { diff --git a/lib/fixture-matrix/fixtures.mjs b/lib/fixture-matrix/fixtures.mjs index b0b3f71d..bd32761b 100644 --- a/lib/fixture-matrix/fixtures.mjs +++ b/lib/fixture-matrix/fixtures.mjs @@ -16,6 +16,7 @@ import { FIXTURE_MATRIX_SCHEMA, FIXTURE_CLASSES, FIXTURE_MANIFEST_FILENAME, + GENERATED_ARTIFACT_METADATA_FILENAME, FIXTURE_COMPLEXITY_MIN, FIXTURE_COMPLEXITY_MAX, } from './shared/constants.mjs'; @@ -415,6 +416,7 @@ export function normalizeFixture(input) { const capabilities = normalizeManifestCapabilities(manifest?.capabilities ?? input.capabilities); const riskProfile = normalizeManifestRiskProfile(manifest?.risk_profile ?? manifest?.riskProfile ?? input.risk_profile ?? input.riskProfile); const qualityBudgets = normalizeManifestQualityBudgets(manifest?.quality_budgets ?? manifest?.qualityBudgets ?? input.quality_budgets ?? input.qualityBudgets); + const allowUnprovenDynamicClientAssets = manifest?.allow_unproven_dynamic_client_assets === true || input.allow_unproven_dynamic_client_assets === true; const fixtureCorpus = input.fixture_corpus || corpusLabelForSearchRoot(root, root); return { id, @@ -430,6 +432,7 @@ export function normalizeFixture(input) { capabilities, risk_profile: riskProfile, quality_budgets: qualityBudgets, + allow_unproven_dynamic_client_assets: allowUnprovenDynamicClientAssets, taxonomy: { ...taxonomy, tags, @@ -488,7 +491,7 @@ export function collectFixtureFiles(directory, options = {}) { } // The per-fixture manifest is matrix metadata, not website source — never // pack it into the imported site artifact. - if (entry.isFile() && entry.name === FIXTURE_MANIFEST_FILENAME) { + if (entry.isFile() && [FIXTURE_MANIFEST_FILENAME, GENERATED_ARTIFACT_METADATA_FILENAME].includes(entry.name)) { continue; } const entryPath = path.join(current, entry.name); diff --git a/lib/fixture-matrix/shared/constants.mjs b/lib/fixture-matrix/shared/constants.mjs index e431eab3..513d1153 100644 --- a/lib/fixture-matrix/shared/constants.mjs +++ b/lib/fixture-matrix/shared/constants.mjs @@ -6,6 +6,7 @@ export const FIXTURE_MATRIX_SCHEMA = 'static-site-importer/fixture-matrix/v1'; export const FIXTURE_MATRIX_RESULT_SCHEMA = 'static-site-importer/fixture-matrix-result/v1'; export const WEBSITE_ARTIFACT_SCHEMA = 'blocks-engine/php-transformer/site-artifact/v1'; +export const GENERATED_ARTIFACT_METADATA_FILENAME = '.generated-artifact-metadata.json'; export const DEFAULT_ENTRYPOINT = 'website/index.html'; export const DEFAULT_IMPORTER_SLUG = 'static-site-importer'; @@ -258,6 +259,7 @@ export const FIXTURE_CLASSES = [ // "risk_profile": "low", // optional. Authored risk lane. // "complexity": 1, // optional. Integer 1-5. // "quality_budgets": {} // optional. Authored budget metadata. +// "allow_unproven_dynamic_client_assets": true // optional explicit runtime-preservation policy. // } // // The manifest is authored in each fixture directory and owned by blocks-engine diff --git a/lib/fixture-matrix/steps/recipe-builder.mjs b/lib/fixture-matrix/steps/recipe-builder.mjs index f3f6bace..7ee0a422 100644 --- a/lib/fixture-matrix/steps/recipe-builder.mjs +++ b/lib/fixture-matrix/steps/recipe-builder.mjs @@ -8,6 +8,7 @@ */ import fs from 'node:fs'; import path from 'node:path'; +import { createHash } from 'node:crypto'; import { pathToFileURL } from 'node:url'; import { randomUUID } from 'node:crypto'; @@ -16,6 +17,7 @@ import { randomUUID } from 'node:crypto'; */ import { WEBSITE_ARTIFACT_SCHEMA, + GENERATED_ARTIFACT_METADATA_FILENAME, DEFAULT_ENTRYPOINT, DEFAULT_IMPORTER_SLUG, VISUAL_PARITY_SOURCE_SUBDIR, @@ -35,9 +37,15 @@ import { liveWpParityCaptureStep, liveWpParityEnabled } from './live-wp-parity-s import { fixtureStepMetadata } from './shared.mjs'; import { selectFixtureSurfaces, summarizeSurfaceCoverage } from './surfaces.mjs'; +const SOURCE_EXCLUSION_RULES = JSON.parse( + fs.readFileSync(new URL('../../../includes/source-exclusion-rules.json', import.meta.url), 'utf8'), +).rules; + export function buildFixtureArtifact(fixture, options = {}) { const normalized = normalizeFixture(fixture); const files = collectFixtureFiles(normalized.directory, options); + const generatedArtifactMetadata = readGeneratedArtifactMetadata(normalized.directory); + const sourceExclusions = []; // Encode EVERY file as `content_base64`, byte-for-byte matching the real // product path. The SSI `import-theme` CLI (static-site-importer.php) reads // each source file and emits `'content_base64' => base64_encode( $content )` @@ -50,12 +58,17 @@ export function buildFixtureArtifact(fixture, options = {}) { // encoding exactly means the gate can never again exercise a payload shape the // product does not actually produce. const artifactFiles = files.map((file) => { - const payload = fs.readFileSync(file.absolute_path); + let payload = fs.readFileSync(file.absolute_path); + if (isHtmlPath(file.relative_path)) { + const result = normalizeSourceHtml(payload.toString('utf8'), `website/${file.relative_path}`); + payload = Buffer.from(result.html); + sourceExclusions.push(...result.exclusions); + } return { path: `website/${file.relative_path}`, source_path: file.absolute_path, type: file.type, - bytes: file.bytes, + bytes: payload.length, content_base64: payload.toString('base64'), }; }); @@ -64,6 +77,7 @@ export function buildFixtureArtifact(fixture, options = {}) { schema: WEBSITE_ARTIFACT_SCHEMA, entrypoint: DEFAULT_ENTRYPOINT, entry_path: DEFAULT_ENTRYPOINT, + ...(generatedArtifactMetadata.compiler_limits ? { compiler_limits: generatedArtifactMetadata.compiler_limits } : {}), files: artifactFiles, summary: { file_count: artifactFiles.length, @@ -82,20 +96,34 @@ export function buildFixtureArtifact(fixture, options = {}) { fixture_capabilities: normalized.capabilities, fixture_risk_profile: normalized.risk_profile, fixture_quality_budgets: normalized.quality_budgets, + source_exclusions: sourceExclusions, }, }; } -// Stage a fixture's ORIGINAL static source (index.html + css/js/images) into the +function readGeneratedArtifactMetadata(directory) { + const metadataPath = path.join(directory, GENERATED_ARTIFACT_METADATA_FILENAME); + if (!fs.existsSync(metadataPath)) { + return {}; + } + try { + const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); + return metadata?.schema === 'static-site-importer/generated-artifact-metadata/v1' ? metadata : {}; + } catch { + return {}; + } +} + +// Stage a fixture's normalized static source (index.html + css/js/images) into the // matrix artifacts tree so the in-sandbox WordPress origin can serve it for the // visual-parity `source-url`. Files land at // `//`, preserving // each fixture's own relative asset layout so the served page resolves its CSS, // JS, and images exactly as the original did. The fixture's `artifact.json` -// import payload is unchanged; this is a parallel, web-servable copy of the raw -// source. Returns the list of staged relative paths. Without this, `source-url` -// points at an unserved path and the visual-compare source capture hangs to the -// 120s timeout (the #563 visual-parity gap). +// import payload uses the same normalization policy so excluded source-platform +// chrome cannot create a false visual mismatch. Returns the staged paths. +// Without this, `source-url` points at an unserved path and the visual-compare +// source capture hangs to the 120s timeout (the #563 visual-parity gap). export function stageFixtureSource(fixture, fixtureDirectory, options = {}) { const normalized = normalizeFixture(fixture); const files = collectFixtureFiles(normalized.directory, options); @@ -104,15 +132,74 @@ export function stageFixtureSource(fixture, fixtureDirectory, options = {}) { for (const file of files) { const destination = path.join(sourceRoot, file.relative_path); fs.mkdirSync(path.dirname(destination), { recursive: true }); - fs.copyFileSync(file.absolute_path, destination); if (isHtmlPath(file.relative_path)) { + const result = normalizeSourceHtml(fs.readFileSync(file.absolute_path, 'utf8'), `website/${file.relative_path}`); + fs.writeFileSync(destination, result.html); injectDeterministicSourceCss(destination, normalized.id); + } else { + fs.copyFileSync(file.absolute_path, destination); } staged.push(file.relative_path); } return staged; } +function normalizeSourceHtml(html, sourcePath) { + const original = html; + const exclusions = []; + for (const rule of SOURCE_EXCLUSION_RULES) { + if (!rule?.selector?.startsWith('#')) continue; + const removed = removeElementById(html, rule.selector.slice(1)); + if (!removed) continue; + html = removed.html; + exclusions.push({ + schema: 'static-site-importer/source-exclusion/v1', + action: 'removed', + category: rule.category || 'source_chrome', + provider: rule.provider || '', + rule_id: rule.id || '', + selector: rule.selector, + source_path: sourcePath, + reason_code: rule.reason_code || 'source_chrome_removed', + removed_sha256: sha256(removed.element), + }); + } + for (const exclusion of exclusions) { + exclusion.source_sha256 = sha256(original); + exclusion.normalized_sha256 = sha256(html); + } + return { html, exclusions }; +} + +function removeElementById(html, id) { + const escapedId = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const openingPattern = new RegExp(`<([a-z][a-z0-9:-]*)\\b[^>]*\\bid\\s*=\\s*(?:"${escapedId}"|'${escapedId}'|${escapedId})(?:\\s|/?>)`, 'i'); + const opening = openingPattern.exec(html); + if (!opening) return null; + const tag = opening[1].toLowerCase(); + const start = opening.index; + const remainder = html.slice(start); + const tags = new RegExp(`]*>`, 'gi'); + let depth = 0; + let match; + while ((match = tags.exec(remainder))) { + if (match[0].startsWith('')) depth += 1; + if (depth === 0) { + const length = match.index + match[0].length; + return { + html: html.slice(0, start) + html.slice(start + length), + element: remainder.slice(0, length), + }; + } + } + return null; +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + export function buildFixtureMatrixRecipe(input = {}) { const matrix = input.matrix || createFixtureMatrix(input); const artifactsDirectory = input.artifactsDirectory || input.artifacts_directory || '/artifacts/static-site-importer-fixture-matrix'; @@ -347,10 +434,11 @@ function editorArtifactPrefix(fixture, surface) { function importFixtureStep(fixture, commandArtifactsDirectory, runId, attemptId) { const fixtureDirectory = path.join(commandArtifactsDirectory, fixture.id); const sidecarName = `materialization-receipt--${sidecarToken(attemptId)}.json`; + const dynamicClientAssetsFlag = fixture.allow_unproven_dynamic_client_assets ? ' --allow-unproven-dynamic-client-assets' : ''; return { command: 'wordpress.wp-cli', args: [ - `command=static-site-importer validate-artifact --artifact=${shellToken(path.join(fixtureDirectory, 'artifact.json'))} --slug=${shellToken(fixture.id)} --name=${shellToken(fixture.label)} --format=fixture-matrix --receipt-sidecar=${shellToken(path.join(fixtureDirectory, sidecarName))} --receipt-run-id=${shellToken(runId)} --receipt-step-id=import --receipt-attempt-id=${shellToken(attemptId)} --allow-failure`, + `command=static-site-importer validate-artifact --artifact=${shellToken(path.join(fixtureDirectory, 'artifact.json'))} --slug=${shellToken(fixture.id)} --name=${shellToken(fixture.label)} --format=fixture-matrix --receipt-sidecar=${shellToken(path.join(fixtureDirectory, sidecarName))} --receipt-run-id=${shellToken(runId)} --receipt-step-id=import --receipt-attempt-id=${shellToken(attemptId)} --allow-failure${dynamicClientAssetsFlag}`, ], metadata: fixtureStepMetadata(fixture, 'import', { artifact: path.join(commandArtifactsDirectory, fixture.id, 'artifact.json'), diff --git a/runtime-package-manifest.json b/runtime-package-manifest.json index 28e92869..f80d32bd 100644 --- a/runtime-package-manifest.json +++ b/runtime-package-manifest.json @@ -7,6 +7,7 @@ "description": "WordPress runtime files required to validate, compile, and materialize website artifacts as block themes.", "abilities": [ "static-site-importer/import-website-artifact", + "static-site-importer/import-url", "static-site-importer/materialize-wordpress-site-plan", "static-site-importer/validate-artifact", "static-site-importer/get-runtime-package-manifest" @@ -66,6 +67,9 @@ "static-site-importer.php", "includes/abilities.php", "includes/class-static-site-importer-theme-generator.php", + "includes/class-static-site-importer-url-fetcher.php", + "includes/class-static-site-importer-url-import-runtime.php", + "includes/class-static-site-importer-url-site-collector.php", "includes/class-static-site-importer-validation-runtime.php", "includes/class-static-site-importer-wordpress-site-plan-materializer.php", "vendor/autoload.php", diff --git a/static-site-importer.php b/static-site-importer.php index 1e2e82bf..80020041 100644 --- a/static-site-importer.php +++ b/static-site-importer.php @@ -51,6 +51,9 @@ require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-document.php'; require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-source-page.php'; require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-url-fetcher.php'; +require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-artifact-run.php'; +require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-source-normalizer.php'; +require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-url-site-collector.php'; require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-url-import-runtime.php'; require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-companion-plugin.php'; require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-plugin-materializer.php'; @@ -212,6 +215,50 @@ static function ( array $args, array $assoc_args ): void { } ); + WP_CLI::add_command( + 'static-site-importer import-url', + static function ( array $args, array $assoc_args ): void { + $url = isset( $args[0] ) ? (string) $args[0] : ''; + if ( '' === trim( $url ) ) { + WP_CLI::error( 'Provide a public source URL.' ); + } + + $provider_args = array(); + if ( isset( $assoc_args['collect-site'] ) ) { + $provider_args['collect_site'] = true; + } + if ( isset( $assoc_args['skip-scripts'] ) ) { + $provider_args['include_scripts'] = false; + } + foreach ( array( 'batch-pages', 'max-pages', 'max-assets', 'max-total-bytes', 'request-delay-ms', 'timeout', 'max-bytes' ) as $key ) { + if ( isset( $assoc_args[ $key ] ) ) { + $provider_args[ str_replace( '-', '_', $key ) ] = (int) $assoc_args[ $key ]; + } + } + + $input = array( + 'url' => $url, + 'provider_args' => $provider_args, + 'slug' => isset( $assoc_args['slug'] ) ? (string) $assoc_args['slug'] : '', + 'name' => isset( $assoc_args['name'] ) ? (string) $assoc_args['name'] : '', + 'site_title' => isset( $assoc_args['site-title'] ) ? (string) $assoc_args['site-title'] : '', + 'activate' => isset( $assoc_args['activate'] ), + 'overwrite' => isset( $assoc_args['overwrite'] ), + 'fail_on_quality' => isset( $assoc_args['fail-on-quality'] ), + 'allow_missing_woocommerce' => isset( $assoc_args['allow-missing-woocommerce'] ), + 'report' => isset( $assoc_args['report'] ) ? (string) $assoc_args['report'] : '', + 'work_dir' => isset( $assoc_args['work-dir'] ) ? (string) $assoc_args['work-dir'] : '', + ); + $result = static_site_importer_ability_import_url( $input ); + if ( empty( $result['success'] ) ) { + $error = isset( $result['error'] ) && is_array( $result['error'] ) ? $result['error'] : array(); + WP_CLI::error( (string) ( $error['message'] ?? 'Static site URL import failed.' ) ); + } + + WP_CLI::success( sprintf( 'Imported %s.', (string) ( $result['result']['theme_slug'] ?? $input['slug'] ) ) ); + } + ); + WP_CLI::add_command( 'static-site-importer validate-artifact', static function ( array $args, array $assoc_args ): void { @@ -229,6 +276,7 @@ static function ( array $args, array $assoc_args ): void { 'overwrite' => ! isset( $assoc_args['no-overwrite'] ), 'fail_on_quality' => isset( $assoc_args['fail-on-quality'] ), 'allow_missing_woocommerce' => isset( $assoc_args['allow-missing-woocommerce'] ), + 'require_proven_dynamic_client_assets' => ! isset( $assoc_args['allow-unproven-dynamic-client-assets'] ), ); $output = isset( $assoc_args['output'] ) ? (string) $assoc_args['output'] : ''; if ( isset( $assoc_args['artifact-dir'] ) ) { diff --git a/test-manifest.json b/test-manifest.json index 5e20838d..e7981621 100644 --- a/test-manifest.json +++ b/test-manifest.json @@ -35,9 +35,14 @@ { "path": "tests/smoke-webfont-producer-consumer.php", "environment": "standalone-php" }, { "path": "tests/smoke-website-artifact-import-input.php", "environment": "standalone-php" }, { "path": "tests/smoke-wordpress-site-plan-materializer.php", "environment": "standalone-php" }, + { "path": "tests/smoke-artifact-run-primitives.php", "environment": "standalone-php" }, + { "path": "tests/smoke-figma-workspace-lifecycle.php", "environment": "standalone-php" }, + { "path": "tests/smoke-url-batch-import.php", "environment": "standalone-php" }, + { "path": "tests/smoke-url-site-collector.php", "environment": "standalone-php" }, { "path": "tests/smoke-import-source-of-truth-manifest.php", "environment": "wordpress-runtime" }, { "path": "tests/smoke-rest-import-normalization.php", "environment": "wordpress-runtime" }, { "path": "tests/smoke-static-interactive-artifact.php", "environment": "wordpress-runtime" }, + { "path": "tests/smoke-url-batch-import-wordpress.php", "environment": "wordpress-runtime" }, { "path": "tests/smoke-website-artifact-document-metadata.php", "environment": "wordpress-runtime" }, { "path": "tests/StaticSiteImporterFallbackDiagnosticsTest.php", "environment": "wordpress-runtime", "command": ["phpunit", "tests/StaticSiteImporterFallbackDiagnosticsTest.php"] }, { "path": "tools/fig-intent-parity-report.test.mjs", "environment": "node" }, diff --git a/tests/smoke-artifact-run-primitives.php b/tests/smoke-artifact-run-primitives.php new file mode 100644 index 00000000..0cdf90cf --- /dev/null +++ b/tests/smoke-artifact-run-primitives.php @@ -0,0 +1,25 @@ +code;} } +function is_wp_error( $value ): bool { return $value instanceof WP_Error; } +function wp_mkdir_p( string $path ): bool { return is_dir( $path ) || mkdir( $path, 0777, true ); } +function wp_json_encode( $value, int $options = 0 ) { return json_encode( $value, $options ); } +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-artifact-run.php'; +$root = sys_get_temp_dir() . '/ssi-artifact-primitives-' . bin2hex( random_bytes( 4 ) ); wp_mkdir_p( $root ); file_put_contents( $root . '/unrelated.txt', 'keep' ); +$workspace = new Static_Site_Importer_Artifact_Run_Workspace( $root, 'test', array( 'on_success' => 'purge_on_success' ) ); +if ( ! is_wp_error( $workspace->path( '../escape' ) ) || ! is_wp_error( $workspace->publish_raw( '/escape', 'x' ) ) ) { throw new RuntimeException( 'workspace must reject path traversal and absolute paths' ); } +if ( is_wp_error( $workspace->publish_raw( 'nested/evidence.bin', 'evidence' ) ) ) { throw new RuntimeException( 'workspace must atomically publish owned bytes' ); } +$cache = new Static_Site_Importer_Artifact_Byte_Cache( $workspace, 'payload', 1, 1024 ); $cache->put( 'one', 'body', array( 'kind' => 'test' ) ); $hit = $cache->get( 'one' ); $cache->hit(); +if ( 'body' !== ( $hit['bytes'] ?? '' ) ) { throw new RuntimeException( 'cache must replay checksummed bytes and metadata' ); } +$entry = glob( $workspace->directory() . '/cache/payload/*.entry' )[0] ?? ''; file_put_contents( $entry, 'corrupt' ); if ( null !== $cache->get( 'one' ) || 1 !== ( $cache->evidence()['corrupt_entries'] ?? 0 ) ) { throw new RuntimeException( 'cache corruption must recover as a miss' ); } +$cache->put( 'two', str_repeat( 'x', 2048 ), array() ); if ( 1 > ( $cache->evidence()['bypassed'] ?? 0 ) ) { throw new RuntimeException( 'cache capacity must bypass oversized entries' ); } +$cursor = Static_Site_Importer_Artifact_Batch_Cursor::create( array( 'a', 'b', 'c' ), 2 ); $cursor = Static_Site_Importer_Artifact_Batch_Cursor::complete( $cursor, 0 ); $cursor = Static_Site_Importer_Artifact_Batch_Cursor::split( $cursor, 1 ); if ( 1 !== Static_Site_Importer_Artifact_Batch_Cursor::next( $cursor ) || 'a' !== $cursor[0]['units'][0] || 3 !== count( $cursor ) ) { throw new RuntimeException( 'cursor must preserve deterministic completed work and split lineage' ); } +$path = $root . '/manifest.json'; $manifest = new Static_Site_Importer_Artifact_Run_Manifest( $path, 'identity', 'test/v1', array( 'target' => 'one' ) ); $data = array( 'schema' => 'test/v1', 'source' => array( 'identity' => 'identity' ), 'contract' => array( 'target' => 'one' ), 'state' => 'completed', 'final_result' => array( 'ok' => true ) ); $manifest->save( $data ); if ( array( 'ok' => true ) !== $manifest->load()['final_result'] || array( 'ok' => true ) !== $manifest->replay() ) { throw new RuntimeException( 'manifest must atomically save and replay completed results' ); } +$mismatch = new Static_Site_Importer_Artifact_Run_Manifest( $path, 'other', 'test/v1', array( 'target' => 'two' ) ); if ( ! is_wp_error( $mismatch->load() ) ) { throw new RuntimeException( 'manifest identity and contract mismatches must be rejected' ); } +$receipt = $workspace->purge(); if ( 'purged' !== $receipt['status'] || empty( $receipt['removed'] ) || ! is_file( $root . '/unrelated.txt' ) || is_dir( $workspace->directory() ) ) { throw new RuntimeException( 'purge must remove only owned workspace files and return a receipt' ); } +$retained = new Static_Site_Importer_Artifact_Run_Workspace( $root, 'retained', array( 'on_success' => 'retain', 'expires_at' => gmdate( 'c', time() - 1 ) ) ); $created_at = json_decode( (string) $retained->read_raw( 'workspace.json' ), true )['created_at'] ?? ''; $reopened = new Static_Site_Importer_Artifact_Run_Workspace( $root, 'retained' ); if ( $created_at !== ( json_decode( (string) $reopened->read_raw( 'workspace.json' ), true )['created_at'] ?? '' ) || ! $reopened->is_expired() || 'purged' !== $reopened->purge_expired()['status'] ) { throw new RuntimeException( 'workspace must retain stable ownership metadata and enforce expiry' ); } +$link = $root . '/linked-root'; if ( function_exists( 'symlink' ) && symlink( sys_get_temp_dir(), $link ) ) { try { new Static_Site_Importer_Artifact_Run_Workspace( $link, 'unsafe' ); throw new RuntimeException( 'symlink workspace root must be rejected' ); } catch ( RuntimeException $expected ) {} } +$nested = new Static_Site_Importer_Artifact_Run_Workspace( $root, 'nested' ); if ( function_exists( 'symlink' ) && symlink( sys_get_temp_dir(), $nested->directory() . '/linked' ) && ! is_wp_error( $nested->publish_raw( 'linked/escape', 'no' ) ) ) { throw new RuntimeException( 'nested workspace symlinks must be rejected' ); } $nested->purge(); +$partial = new Static_Site_Importer_Artifact_Run_Workspace( $root, 'partial' ); $outside = $root . '/outside.txt'; file_put_contents( $outside, 'outside' ); if ( function_exists( 'symlink' ) && symlink( $outside, $partial->directory() . '/outside-link' ) ) { $partial_receipt = $partial->purge(); if ( 'partial' !== $partial_receipt['status'] || ! is_file( $outside ) || empty( $partial_receipt['skipped'] ) ) { throw new RuntimeException( 'purge must report skipped symlinks without touching their targets' ); } unlink( $partial->directory() . '/outside-link' ); $partial->purge(); } +echo "Artifact run primitive smoke passed.\n"; diff --git a/tests/smoke-figma-workspace-lifecycle.php b/tests/smoke-figma-workspace-lifecycle.php new file mode 100644 index 00000000..c3665915 --- /dev/null +++ b/tests/smoke-figma-workspace-lifecycle.php @@ -0,0 +1,18 @@ +code;} } +function is_wp_error( $value ): bool { return $value instanceof WP_Error; } +function wp_mkdir_p( string $path ): bool { return is_dir( $path ) || mkdir( $path, 0777, true ); } +function wp_json_encode( $value, int $options = 0 ) { return json_encode( $value, $options ); } +function apply_filters( string $hook, $value ) { return 'static_site_importer_figma_zstd_available' === $hook ? true : $value; } +$seen = array(); $fail = false; +function blocks_engine_figma_transformer_transform_file( string $path, array $options ) { global $seen, $fail; $seen[] = $path; if ( $fail ) { return array( 'status' => 'failed' ); } return array( 'files' => array( array( 'path' => 'website/index.html', 'content' => '
Figma
' ) ) ); } +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-figma-import.php'; +$before = glob( sys_get_temp_dir() . '/.ssi-artifact-run-fig-*' ) ?: array(); +$input = array( 'source' => array( 'figma_file' => array( 'name' => 'design.fig', 'content_base64' => base64_encode( 'fig' ) ) ) ); $success = Static_Site_Importer_Figma_Import::website_artifact_from_input( $input ); +if ( is_wp_error( $success ) || empty( $seen ) || array_diff( glob( sys_get_temp_dir() . '/.ssi-artifact-run-fig-*' ) ?: array(), $before ) ) { throw new RuntimeException( 'base64 Figma staging must clean its owned workspace after a successful transform' ); } +$fail = true; $failed = Static_Site_Importer_Figma_Import::website_artifact_from_input( $input ); if ( ! is_wp_error( $failed ) || array_diff( glob( sys_get_temp_dir() . '/.ssi-artifact-run-fig-*' ) ?: array(), $before ) ) { throw new RuntimeException( 'base64 Figma staging must clean its owned workspace after a transform failure' ); } +$fail = false; $retained = Static_Site_Importer_Figma_Import::website_artifact_from_input( $input + array( 'retain_workspace' => true ) ); $evidence = $retained['provenance']['artifact_workspace'] ?? array(); if ( is_wp_error( $retained ) || ! is_dir( $evidence['path'] ?? '' ) || empty( $evidence['expires_at'] ) || 'retained' !== ( $evidence['cleanup']['status'] ?? '' ) ) { throw new RuntimeException( 'retained Figma staging must expose expiry and cleanup evidence' ); } $workspace = new Static_Site_Importer_Artifact_Run_Workspace( sys_get_temp_dir(), substr( basename( $evidence['path'] ), strlen( '.ssi-artifact-run-' ) ) ); if ( 'purged' !== $workspace->purge()['status'] ) { throw new RuntimeException( 'retained Figma workspace must support explicit purge' ); } +$studio = ABSPATH . '.studio-import'; wp_mkdir_p( $studio ); $staged = $studio . '/design.fig'; file_put_contents( $staged, 'fig' ); Static_Site_Importer_Figma_Import::website_artifact_from_input( array( 'source' => array( 'figma_file' => array( 'name' => 'design.fig', 'staged_path' => $staged ) ) ) ); if ( ! is_file( $staged ) ) { throw new RuntimeException( 'caller-owned Studio staged files must not be deleted' ); } unlink( $staged ); rmdir( $studio ); +echo "Figma workspace lifecycle smoke passed.\n"; diff --git a/tests/smoke-url-batch-import-wordpress.php b/tests/smoke-url-batch-import-wordpress.php new file mode 100644 index 00000000..664eba19 --- /dev/null +++ b/tests/smoke-url-batch-import-wordpress.php @@ -0,0 +1,40 @@ + array( 'application/xml', '' . $origin . '/' . $prefix . '/' . $origin . '/' . $prefix . '/about/' . $origin . '/' . $prefix . '/about/team/' ), + $origin . '/' . $prefix . '/' => array( 'text/html', '
Home
' ), + $origin . '/' . $prefix . '/about/' => array( 'text/html', '
About
' ), + $origin . '/' . $prefix . '/about/team/' => array( 'text/html', '
Team
' ), + $origin . '/' . $prefix . '/first.css' => array( 'text/css', '.first{color:red}' ), $origin . '/' . $prefix . '/second.css' => array( 'text/css', '.second{color:blue}' ), +); +$fetcher = static function ( string $url, array $args ) use ( $responses ) { return isset( $responses[ $url ] ) ? array( 'body' => $responses[ $url ][1], 'metadata' => array( 'content_type' => $responses[ $url ][0], 'final_url' => $url ) ) : new WP_Error( 'fixture_missing', $url ); }; +$request = array( 'url' => $origin . '/' . $prefix . '/', 'work_dir' => $work_dir, 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 2, 'request_delay_ms' => 0, 'max_assets' => 10 ) ); +$input = array( 'slug' => $slug, 'name' => 'SSI Batch WP', 'activate' => true, 'overwrite' => false ); +$before = get_stylesheet(); $calls = 0; +$first = Static_Site_Importer_URL_Batch_Import::import( $request, $input, $fetcher, static function ( array $artifact, array $args ) use ( &$calls ) { if ( 1 === $calls++ ) { return new WP_Error( 'injected_batch_failure', 'resume test' ); } return Static_Site_Importer_Theme_Generator::import_website_artifact( $artifact, $args ); } ); +$run_id = is_wp_error( $first ) ? (string) ( $first->get_error_data()['run']['source']['identity'] ?? '' ) : ''; +$first_about = get_page_by_path( $prefix . '/about', OBJECT, 'page' ); +if ( ! is_wp_error( $first ) || '' === $run_id || $before !== get_stylesheet() || ! $first_about || $run_id !== (string) ( json_decode( (string) get_post_meta( $first_about->ID, '_static_site_importer_provenance', true ), true )['import_run_id'] ?? '' ) ) { throw new RuntimeException( 'first batch must materialize this invocation routes and provenance without activating after injected failure' ); } +$resumed = Static_Site_Importer_URL_Batch_Import::import( $request, $input, $fetcher, static fn( array $artifact, array $args ) => Static_Site_Importer_Theme_Generator::import_website_artifact( $artifact, $args ) ); +$active_slug = sanitize_key( $slug ); +$theme_dir = get_theme_root() . '/' . $active_slug; +$about = get_page_by_path( $prefix . '/about', OBJECT, 'page' ); $team = get_page_by_path( $prefix . '/about/team', OBJECT, 'page' ); +$bootstrap = (string) file_get_contents( $theme_dir . '/functions.php' ) . implode( '', array_map( static fn( string $path ): string => (string) file_get_contents( $path ), glob( $theme_dir . '/static-site-importer-batch-bootstrap/*.php' ) ?: array() ) ); +if ( is_wp_error( $resumed ) || get_stylesheet() !== $active_slug || ! $about || ! $team || $run_id !== (string) ( json_decode( (string) get_post_meta( $about->ID, '_static_site_importer_provenance', true ), true )['import_run_id'] ?? '' ) || $run_id !== (string) ( json_decode( (string) get_post_meta( $team->ID, '_static_site_importer_provenance', true ), true )['import_run_id'] ?? '' ) || (int) $team->post_parent !== (int) $about->ID || ! is_file( $theme_dir . '/functions.php' ) || ! str_contains( (string) file_get_contents( $theme_dir . '/assets/website/' . $prefix . '/first.css' ), '.first' ) || ! str_contains( (string) file_get_contents( $theme_dir . '/assets/website/' . $prefix . '/second.css' ), '.second' ) || ! str_contains( $bootstrap, 'assets/website/' . $prefix . '/first.css' ) || ! str_contains( $bootstrap, 'assets/website/' . $prefix . '/second.css' ) || 'completed' !== ( $resumed['url_batch_run']['status'] ?? '' ) ) { throw new RuntimeException( 'resumed batches must retain this invocation active bootstrap CSS behavior, assets, nested parents, terminal activation, and aggregate evidence' ); } +echo "WordPress URL batch import smoke passed.\n"; diff --git a/tests/smoke-url-batch-import.php b/tests/smoke-url-batch-import.php new file mode 100644 index 00000000..4992014d --- /dev/null +++ b/tests/smoke-url-batch-import.php @@ -0,0 +1,149 @@ +code; } public function get_error_message(): string { return $this->message; } public function get_error_data(): mixed { return $this->data; } } +function is_wp_error( $value ): bool { return $value instanceof WP_Error; } +function sanitize_file_name( string $name ): string { return trim( (string) preg_replace( '/[^A-Za-z0-9._-]+/', '-', $name ), '-' ); } +function trailingslashit( string $path ): string { return rtrim( $path, '/' ) . '/'; } +function wp_mkdir_p( string $path ): bool { return is_dir( $path ) || mkdir( $path, 0777, true ); } +function wp_json_encode( $value, int $options = 0 ) { return json_encode( $value, $options ); } +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-url-fetcher.php'; +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-url-site-collector.php'; +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-url-import-runtime.php'; + +$responses = array( + 'https://batch.test/sitemap.xml' => array( 'application/xml', 'https://batch.test/one.xmlhttps://batch.test/two.xml' ), + 'https://batch.test/one.xml' => array( 'application/xml', 'https://batch.test/https://batch.test/about/' ), + 'https://batch.test/two.xml' => array( 'application/xml', 'https://batch.test/about/team/' ), + 'https://batch.test/' => array( 'text/html', '
About
' ), + 'https://batch.test/about/' => array( 'text/html', '
Team
' ), + 'https://batch.test/about/team/' => array( 'text/html', '
Team
' ), + 'https://batch.test/empty.css' => array( 'text/css', '' ), +); +$requests = array(); +$transient_failures = array(); +$fetcher = static function ( string $url, array $args ) use ( &$requests, &$transient_failures, $responses ) { + $requests[] = $url; + if ( 'https://batch.test/about/team/' === $url && empty( $transient_failures[ $url ] ) ) { $transient_failures[ $url ] = true; return new WP_Error( 'transient_timeout', 'retry me' ); } + if ( ! isset( $responses[ $url ] ) ) { return new WP_Error( 'missing_fixture', $url ); } + return array( 'body' => $responses[ $url ][1], 'metadata' => array( 'content_type' => $responses[ $url ][0], 'final_url' => $url ) ); +}; +$work_dir = sys_get_temp_dir() . '/ssi-url-batch-' . bin2hex( random_bytes( 4 ) ); +$request = array( 'url' => 'https://batch.test/', 'work_dir' => $work_dir, 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 2, 'request_delay_ms' => 0, 'max_assets' => 10 ) ); +$input = array( 'activate' => true ); +$artifacts = array(); +$attempt = 0; +$importer = static function ( array $artifact, array $args ) use ( &$artifacts, &$attempt ) { + $artifacts[] = array( 'artifact' => $artifact, 'args' => $args ); + if ( 1 === $attempt++ ) { return new WP_Error( 'injected_batch_failure', 'stop after the first completed batch' ); } + return array( 'theme_slug' => 'batch-site', 'quality' => array( 'pass' => true, 'status' => 'success_with_warnings', 'metrics' => array( 'fallback_count' => 1 ), 'fallbacks' => array( array( 'html' => str_repeat( 'x', 1024 ) ) ) ), 'import_report_summary' => array( 'status' => 'completed' ) ); +}; +$first = Static_Site_Importer_URL_Batch_Import::import( $request, $input, $fetcher, $importer ); +if ( ! is_wp_error( $first ) || 'injected_batch_failure' !== $first->get_error_code() ) { throw new RuntimeException( 'injected batch failure must retain a resumable run' ); } +$manifest_path = $first->get_error_data()['run_manifest'] ?? ''; +$manifest = json_decode( (string) file_get_contents( $manifest_path ), true ); +if ( 'failed' !== ( $manifest['state'] ?? '' ) || 'completed' !== ( $manifest['batches'][0]['state'] ?? '' ) || 3 !== ( $manifest['total_routes'] ?? 0 ) ) { throw new RuntimeException( 'manifest must checkpoint discovery and completed batches' ); } +if ( ! preg_match( '/^batch-[a-f0-9]{16}$/', (string) ( $manifest['batches'][0]['batch_id'] ?? '' ) ) || 64 !== strlen( (string) ( $manifest['batches'][0]['result']['snapshot_sha256'] ?? '' ) ) ) { throw new RuntimeException( 'checkpointed batches must retain stable identities and source snapshot evidence' ); } +$legacy_cache = $work_dir . '/url-response-cache-' . $manifest['source']['identity']; wp_mkdir_p( $legacy_cache ); foreach ( glob( $work_dir . '/.ssi-artifact-run-url-' . $manifest['source']['identity'] . '/cache/http-response/*.entry' ) ?: array() as $entry ) { copy( $entry, $legacy_cache . '/' . basename( $entry ) ); } $legacy_workspace = new Static_Site_Importer_Artifact_Run_Workspace( $work_dir, 'url-' . $manifest['source']['identity'] ); $legacy_workspace->purge(); +$resumed = Static_Site_Importer_URL_Batch_Import::import( $request, $input, $fetcher, static fn( array $artifact, array $args ) => array( 'theme_slug' => 'batch-site', 'artifact' => $artifact, 'args' => $args, 'quality' => array( 'pass' => true, 'status' => 'success', 'metrics' => array( 'fallback_count' => 0 ), 'fallbacks' => array() ), 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( is_wp_error( $resumed ) || true !== ( $resumed['terminal_batch_result']['args']['activate'] ?? false ) || isset( $resumed['pages'] ) || 2 !== ( $resumed['url_batch_run']['completed_batches'] ?? 0 ) || 3 !== ( $resumed['import_report_summary']['completed_routes'] ?? 0 ) ) { throw new RuntimeException( 'aggregate results must retain terminal output explicitly without misrepresenting it as whole-site output' ); } +$batch_quality = $resumed['url_batch_run']['batch_quality'] ?? array(); +if ( 2 !== count( $batch_quality ) || 1 !== ( $batch_quality[0]['fallback_count'] ?? -1 ) || isset( $batch_quality[0]['fallbacks'] ) || true !== ( $batch_quality[0]['pass'] ?? null ) || true !== ( $batch_quality[1]['pass'] ?? null ) ) { throw new RuntimeException( 'resumed aggregates must derive bounded quality evidence for every completed batch without retaining fallback payloads' ); } +$first_files = array_column( $artifacts[0]['artifact']['files'], null, 'path' ); +$second_files = array_column( $artifacts[1]['artifact']['files'], null, 'path' ); +if ( ! str_contains( (string) $first_files['website/about/index.html']['content'], 'href="/about/team/"' ) || ! isset( $first_files['website/empty.css'] ) || 2 < count( array_filter( $artifacts[0]['artifact']['files'], static fn( array $file ) => str_ends_with( $file['path'], 'index.html' ) ) ) || isset( $second_files['website/index.html'] ) || ! isset( $second_files['website/about/team/index.html'] ) || empty( $artifacts[1]['args']['preserve_existing_theme_bootstrap'] ) || 2 !== count( array_keys( $requests, 'https://batch.test/about/team/', true ) ) || 1 > ( $resumed['url_batch_run']['fetch_cache']['hits'] ?? 0 ) || is_dir( $legacy_cache ) ) { throw new RuntimeException( 'later batches must exclude the unrelated root page while legacy response cache migration preserves payload reuse' ); } +$again = Static_Site_Importer_URL_Batch_Import::import( $request, $input, $fetcher, static fn() => new WP_Error( 'should_not_run' ) ); +if ( is_wp_error( $again ) || 'batch-site' !== ( $again['theme_slug'] ?? '' ) ) { throw new RuntimeException( 'terminal runs must return their saved SSI result without reimporting' ); } +$mismatch = Static_Site_Importer_URL_Batch_Import::import( $request, array( 'activate' => true, 'slug' => 'other-target' ), $fetcher, static fn() => new WP_Error( 'should_not_run' ) ); +if ( ! is_wp_error( $mismatch ) || 'static_site_importer_batch_contract_mismatch' !== $mismatch->get_error_code() ) { throw new RuntimeException( 'a reused work directory must reject a mismatched import target contract' ); } +$activation_mismatch = Static_Site_Importer_URL_Batch_Import::import( $request, array( 'activate' => false ), $fetcher, static fn() => new WP_Error( 'should_not_run' ) ); +if ( ! is_wp_error( $activation_mismatch ) || 'static_site_importer_batch_contract_mismatch' !== $activation_mismatch->get_error_code() ) { throw new RuntimeException( 'activation must be part of the resumable import contract' ); } +if ( glob( $work_dir . '/url-site-batch-cache-*.json' ) ) { throw new RuntimeException( 'completed batches must remove their fetched artifact cache' ); } +if ( 'completed' !== ( $resumed['url_batch_run']['status'] ?? '' ) || empty( $resumed['url_batch_run']['terminal_batch_report_path'] ) && ! array_key_exists( 'terminal_batch_report_path', $resumed['url_batch_run'] ) ) { throw new RuntimeException( 'aggregate evidence must label terminal-batch report fields explicitly' ); } +$scale_routes = array(); +for ( $i = 0; $i < 1144; $i++ ) { $scale_routes[] = 'https://scale.test/page-' . $i . '/'; } +$scale = Static_Site_Importer_URL_Site_Collector::discover_routes( 'https://scale.test/', array( 'request_delay_ms' => 0 ), static fn( string $url, array $args ) => array( 'body' => 'https://scale.test/sitemap.xml' === $url ? '' . implode( '', $scale_routes ) . '' : '', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ) ); +if ( 1144 !== count( $scale ) || 5000 !== ( Static_Site_Importer_URL_Site_Collector::discovery_limits()['max_discovered_routes'] ?? 0 ) ) { throw new RuntimeException( 'discovery must support the acceptance sitemap scale within explicit limits' ); } +$overflow = Static_Site_Importer_URL_Site_Collector::discover_routes( 'https://overflow.test/', array(), static fn( string $url, array $args ) => array( 'body' => '' . implode( '', array_map( static fn( int $i ): string => 'https://overflow.test/p-' . $i . '/', range( 1, 5001 ) ) ) . '', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ) ); +if ( ! is_wp_error( $overflow ) || 'static_site_importer_discovery_incomplete' !== $overflow->get_error_code() || 'routes' !== ( $overflow->get_error_data()['truncated_dimension'] ?? '' ) ) { throw new RuntimeException( 'route discovery must reject queue/route overflow with structured evidence' ); } +$asset_urls = array(); +for ( $i = 0; $i < 201; $i++ ) { $asset_urls[] = 'https://assets.test/a-' . $i . '.png'; } +$asset_request = array( 'url' => 'https://assets.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-url-batch-assets-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'request_delay_ms' => 0 ) ); +$asset_result = Static_Site_Importer_URL_Batch_Import::import( $asset_request, array(), static function ( string $url, array $args ) use ( $asset_urls ) { if ( 'https://assets.test/sitemap.xml' === $url ) { return array( 'body' => 'https://assets.test/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://assets.test/' === $url ) { return array( 'body' => '
' . implode( '', array_map( static fn( string $asset ): string => '', $asset_urls ) ) . '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return array( 'body' => 'x', 'metadata' => array( 'content_type' => 'image/png', 'final_url' => $url ) ); }, static fn( array $artifact, array $args ) => array( 'theme_slug' => 'assets', 'asset_count' => count( $artifact['files'] ?? array() ) - 1, 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( is_wp_error( $asset_result ) || 2000 !== ( $asset_result['url_batch_run']['per_batch_limits']['max_assets'] ?? 0 ) || 268435456 !== ( $asset_result['url_batch_run']['per_batch_limits']['max_total_bytes'] ?? 0 ) || 201 > ( $asset_result['terminal_batch_result']['asset_count'] ?? 0 ) ) { throw new RuntimeException( 'batch defaults must support more than legacy 200 assets with bounded per-batch limits' ); } +$lower_request = $asset_request; $lower_request['work_dir'] .= '-lower'; $lower_request['provider_args']['max_assets'] = 1; +$lower_result = Static_Site_Importer_URL_Batch_Import::import( $lower_request, array(), static fn( string $url, array $args ) => 'https://assets.test/sitemap.xml' === $url ? array( 'body' => 'https://assets.test/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ) : new WP_Error( 'fixture_stop', 'lower override reached collection' ), static fn() => array() ); +if ( ! is_wp_error( $lower_result ) || 1 !== ( $lower_result->get_error_data()['run']['per_batch_limits']['max_assets'] ?? 0 ) ) { throw new RuntimeException( 'caller lower per-batch asset overrides must remain honored' ); } +$split_work_dir = sys_get_temp_dir() . '/ssi-url-split-' . bin2hex( random_bytes( 4 )); +$split_routes = array( 'https://split.test/', 'https://split.test/p1/', 'https://split.test/p2/', 'https://split.test/p3/', 'https://split.test/p4/' ); +$split_fetch_counts = array(); $split_fetcher = static function ( string $url, array $args ) use ( $split_routes, &$split_fetch_counts ) { $split_fetch_counts[ $url ] = ( $split_fetch_counts[ $url ] ?? 0 ) + 1; if ( 'https://split.test/sitemap.xml' === $url ) { return array( 'body' => '' . implode( '', array_map( static fn( string $route ): string => '' . $route . '', $split_routes ) ) . '', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( in_array( $url, $split_routes, true ) ) { return array( 'body' => '
' . str_repeat( 'x', 60 ) . '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return new WP_Error( 'unexpected_asset', $url ); }; +$split_request = array( 'url' => 'https://split.test/', 'work_dir' => $split_work_dir, 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 4, 'max_assets' => 1, 'max_total_bytes' => 160, 'request_delay_ms' => 0 ) ); +$split_calls = array(); $split_attempt = 0; +$split_first = Static_Site_Importer_URL_Batch_Import::import( $split_request, array(), $split_fetcher, static function ( array $artifact, array $args ) use ( &$split_calls, &$split_attempt ) { $split_calls[] = array_column( $artifact['files'], 'path'); if ( 1 === $split_attempt++ ) { return new WP_Error( 'split_resume_failure', 'after a completed split child' ); } return array( 'theme_slug' => 'split', 'import_report_summary' => array( 'status' => 'completed' ) ); } ); +if ( ! is_wp_error( $split_first ) || 'split_resume_failure' !== $split_first->get_error_code() ) { throw new RuntimeException( 'split test must fail after a completed child' ); } +$split_manifest_path = $split_first->get_error_data()['run_manifest']; $split_manifest = json_decode( (string) file_get_contents( $split_manifest_path ), true ); +if ( empty( array_filter( $split_manifest['diagnostics'] ?? array(), static fn( array $row ): bool => 'batch_subdivided' === ( $row['code'] ?? '' ) ) ) || 'completed' !== ( $split_manifest['batches'][0]['state'] ?? '' ) ) { throw new RuntimeException( 'oversized batch must checkpoint deterministic split lineage before resume' ); } +$completed_before_resume = count( $split_calls ); +$split_final = Static_Site_Importer_URL_Batch_Import::import( $split_request, array(), $split_fetcher, static function ( array $artifact, array $args ) use ( &$split_calls ) { $split_calls[] = array_column( $artifact['files'], 'path'); return array( 'theme_slug' => 'split', 'import_report_summary' => array( 'status' => 'completed' ) ); } ); +$split_cache = $split_final['url_batch_run']['fetch_cache'] ?? array(); $split_underlying = array_sum( $split_fetch_counts ); +if ( is_wp_error( $split_final ) || 5 !== ( $split_final['url_batch_run']['completed_routes'] ?? 0 ) || count( $split_calls ) <= $completed_before_resume || 1 !== ( $split_fetch_counts['https://split.test/'] ?? 0 ) || $split_underlying !== ( $split_cache['misses'] ?? -1 ) || ( $split_cache['hits'] ?? 0 ) < 1 ) { throw new RuntimeException( 'split resume must complete routes once and reuse cached root/shared fetches with truthful counters' ); } +$asset_count_split_routes = array( 'https://asset-count-split.test/', 'https://asset-count-split.test/one/', 'https://asset-count-split.test/two/' ); +$asset_count_split = Static_Site_Importer_URL_Batch_Import::import( array( 'url' => 'https://asset-count-split.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-url-asset-count-split-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 3, 'max_assets' => 1, 'request_delay_ms' => 0 ) ), array(), static function ( string $url, array $args ) use ( $asset_count_split_routes ) { if ( 'https://asset-count-split.test/sitemap.xml' === $url ) { return array( 'body' => '' . implode( '', array_map( static fn( string $route ): string => '' . $route . '', $asset_count_split_routes ) ) . '', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( in_array( $url, $asset_count_split_routes, true ) ) { return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return array( 'body' => 'asset', 'metadata' => array( 'content_type' => 'image/png', 'final_url' => $url ) ); }, static fn() => array( 'theme_slug' => 'asset-count-split', 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( is_wp_error( $asset_count_split ) || 3 !== ( $asset_count_split['url_batch_run']['completed_routes'] ?? 0 ) || empty( array_filter( $asset_count_split['url_batch_run']['diagnostics'] ?? array(), static fn( array $row ): bool => 'batch_subdivided' === ( $row['code'] ?? '' ) ) ) ) { throw new RuntimeException( 'multi-route asset-count pressure must subdivide until singleton batches fit' ); } +$asset_failure_request = array( 'url' => 'https://failure-split.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-url-failure-split-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 2, 'request_delay_ms' => 0 ) ); +$asset_failure_imports = 0; +$asset_failure_result = Static_Site_Importer_URL_Batch_Import::import( $asset_failure_request, array( 'activate' => true ), static function ( string $url, array $args ) { if ( 'https://failure-split.test/sitemap.xml' === $url ) { return array( 'body' => 'https://failure-split.test/https://failure-split.test/p/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://failure-split.test/' === $url ) { return array( 'body' => '
' . implode( '', array_map( static fn( int $i ): string => '', range( 1, 4 ) ) ) . '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } if ( 'https://failure-split.test/p/' === $url ) { return array( 'body' => '
' . implode( '', array_map( static fn( int $i ): string => '', range( 5, 8 ) ) ) . '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return new WP_Error( 'optional_asset_404', 'failed' ); }, static function ( array $artifact, array $args ) use ( &$asset_failure_imports ) { $asset_failure_imports++; return array( 'theme_slug' => 'failure-split', 'activated' => ! empty( $args['activate'] ), 'import_report_summary' => array( 'status' => 'completed' ) ); } ); +if ( is_wp_error( $asset_failure_result ) || 1 !== $asset_failure_imports || 1 !== ( $asset_failure_result['url_batch_run']['completed_batches'] ?? 0 ) || true !== ( $asset_failure_result['terminal_batch_result']['activated'] ?? false ) || 8 !== ( $asset_failure_result['url_batch_run']['external_asset_retained']['count'] ?? 0 ) || 8 !== count( array_filter( $asset_failure_result['url_batch_run']['external_asset_retained']['samples'] ?? array(), static fn( array $sample ): bool => 'optional_asset_404' === ( $sample['reason'] ?? '' ) ) ) || ! empty( array_filter( $asset_failure_result['url_batch_run']['diagnostics'] ?? array(), static fn( array $row ): bool => 'batch_subdivided' === ( $row['code'] ?? '' ) ) ) ) { throw new RuntimeException( 'two-route optional asset 404 variants must import once, retain exact evidence, and activate without subdivision' ); } +$rewrite_context = Static_Site_Importer_URL_Site_Collector::collect( 'https://contexts.test/', array( 'asset_failure_policy' => 'preserve_external', 'require_complete_collection' => true, 'request_delay_ms' => 0 ), static function ( string $url, array $args ) { if ( 'https://contexts.test/sitemap.xml' === $url ) { return new WP_Error( 'no_sitemap', '' ); } if ( 'https://contexts.test/' === $url ) { return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } if ( 'https://contexts.test/style.css' === $url ) { return array( 'body' => '@import "/import.css?x=1#f";.x{background:url(/nested.png)}', 'metadata' => array( 'content_type' => 'text/css', 'final_url' => $url ) ); } return new WP_Error( 'optional_asset_timeout', 'failed' ); } ); +$context_files = is_wp_error( $rewrite_context ) ? array() : array_column( $rewrite_context['artifact']['files'], null, 'path'); $context_html = $context_files['website/index.html']['content'] ?? ''; $context_css = $context_files['website/style.css']['content'] ?? ''; +if ( is_wp_error( $rewrite_context ) || ! str_contains( $context_html, 'https://contexts.test/one.png' ) || ! str_contains( $context_html, 'https://contexts.test/two.png?x=1#f' ) || ! str_contains( $context_html, 'https://contexts.test/inline.png' ) || ! str_contains( $context_html, 'https://contexts.test/style.png' ) || ! str_contains( $context_css, 'https://contexts.test/import.css?x=1#f' ) || ! str_contains( $context_css, 'https://contexts.test/nested.png' ) || 6 !== ( $rewrite_context['source_metadata']['collection']['external_asset_retained']['count'] ?? 0 ) ) { throw new RuntimeException( 'external asset preservation must cover srcset, inline CSS, fetched CSS urls and imports' ); } +$nested_limit = Static_Site_Importer_URL_Site_Collector::collect( 'https://nested.test/', array( 'max_assets' => 1, 'asset_failure_policy' => 'preserve_external', 'require_complete_collection' => true, 'request_delay_ms' => 0 ), static function ( string $url, array $args ) { if ( 'https://nested.test/sitemap.xml' === $url ) { return new WP_Error( 'none', '' ); } if ( 'https://nested.test/' === $url ) { return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return array( 'body' => '.x{background:url(/nested.png)}', 'metadata' => array( 'content_type' => 'text/css', 'final_url' => $url ) ); } ); +if ( is_wp_error( $nested_limit ) || 1 !== ( $nested_limit['source_metadata']['collection']['external_asset_retained']['count'] ?? 0 ) || 'asset_limit' !== ( $nested_limit['source_metadata']['collection']['external_asset_retained']['samples'][0]['reason'] ?? '' ) ) { throw new RuntimeException( 'nested CSS asset admission must preserve external URLs at the asset limit' ); } +$retained_failure = Static_Site_Importer_URL_Batch_Import::import( array( 'url' => 'https://persist.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-retained-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'request_delay_ms' => 0 ) ), array(), static function ( string $url, array $args ) { if ( 'https://persist.test/sitemap.xml' === $url ) { return array( 'body' => 'https://persist.test/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://persist.test/' === $url ) { return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return new WP_Error( 'optional_timeout', 'failed' ); }, static fn() => array( 'theme_slug' => 'retained', 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( is_wp_error( $retained_failure ) || 1 !== ( $retained_failure['url_batch_run']['external_asset_retained']['count'] ?? 0 ) || 'optional_timeout' !== ( $retained_failure['url_batch_run']['external_asset_retained']['samples'][0]['reason'] ?? '' ) ) { throw new RuntimeException( 'singleton batch asset failures must retain the external URL and complete' ); } +$preserved_asset = Static_Site_Importer_URL_Site_Collector::collect( 'https://preserve.test/', array( 'max_assets' => 10, 'require_complete_collection' => true, 'asset_failure_policy' => 'preserve_external', 'request_delay_ms' => 0 ), static function ( string $url, array $args ) { if ( 'https://preserve.test/sitemap.xml' === $url ) { return new WP_Error( 'no_sitemap', '' ); } if ( 'https://preserve.test/' === $url ) { return array( 'body' => '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return new WP_Error( 'static_site_importer_url_too_large', 'too large' ); } ); +$preserved_files = is_wp_error( $preserved_asset ) ? array() : array_column( $preserved_asset['artifact']['files'], null, 'path' ); +if ( is_wp_error( $preserved_asset ) || isset( $preserved_files['website/too-big.jpg'] ) || ! str_contains( (string) $preserved_files['website/index.html']['content'], 'https://preserve.test/too-big.jpg' ) || 1 !== ( $preserved_asset['source_metadata']['collection']['external_asset_retained']['count'] ?? 0 ) || 'static_site_importer_url_too_large' !== ( $preserved_asset['source_metadata']['collection']['external_asset_retained']['samples'][0]['reason'] ?? '' ) ) { throw new RuntimeException( 'single-route optional oversized assets must remain absolute with retained-asset evidence' ); } +$strict_asset = Static_Site_Importer_URL_Site_Collector::collect( 'https://strict.test/', array( 'require_complete_collection' => true, 'request_delay_ms' => 0 ), static function ( string $url, array $args ) { if ( 'https://strict.test/sitemap.xml' === $url ) { return new WP_Error( 'no_sitemap', '' ); } if ( 'https://strict.test/' === $url ) { return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); } return new WP_Error( 'optional_asset_failed', 'failed' ); } ); +if ( ! is_wp_error( $strict_asset ) || 'static_site_importer_site_collection_incomplete' !== $strict_asset->get_error_code() ) { throw new RuntimeException( 'strict non-batch complete collection must retain optional asset failures' ); } +$html_failed = Static_Site_Importer_URL_Site_Collector::collect( 'https://html-failed.test/', array( 'asset_failure_policy' => 'preserve_external' ), static fn( string $url, array $args ) => new WP_Error( 'html_fetch_failed', 'failed' ) ); +if ( ! is_wp_error( $html_failed ) || 'html_fetch_failed' !== $html_failed->get_error_code() ) { throw new RuntimeException( 'asset preservation must never downgrade HTML page failures' ); } +$ownership_dir = sys_get_temp_dir() . '/ssi-retained-ownership-' . bin2hex( random_bytes( 4 ) ); wp_mkdir_p( $ownership_dir ); $ownership_workspace = new Static_Site_Importer_Artifact_Run_Workspace( $ownership_dir, 'ownership' ); +$ownership_workspace->publish_json( 'batches/batch-stable.json', array( 'source_metadata' => array( 'snapshot' => array( 'files' => array( array( 'mime_type' => 'text/html', 'source_url' => 'https://ownership.test/stale/' ) ) ) ) ) ); +$retained_runtime_method = new ReflectionMethod( Static_Site_Importer_URL_Batch_Import::class, 'retained_runtime' ); +$owned_runtime = $retained_runtime_method->invoke( null, $ownership_workspace, 'batches/batch-stable.json', 'batches/0.json', $ownership_dir . '/legacy.json', array( 'https://ownership.test/current/' ) ); +if ( null !== $owned_runtime || null !== $ownership_workspace->read_raw( 'batches/batch-stable.json' ) ) { throw new RuntimeException( 'stable retained batch payloads must prove exact route ownership before reuse' ); } +$duplicate_runtime = array( 'source_metadata' => array( 'snapshot' => array( 'files' => array( array( 'mime_type' => 'text/html', 'source_url' => 'https://ownership.test/a/' ), array( 'mime_type' => 'text/html', 'source_url' => 'https://ownership.test/b/' ) ) ) ), 'artifact' => array( 'files' => array( array( 'mime_type' => 'text/html', 'metadata' => array( 'route_path' => '/same' ) ), array( 'mime_type' => 'text/html', 'metadata' => array( 'route_path' => '/same' ) ) ) ) ); +$ownership_workspace->publish_json( 'batches/batch-duplicate.json', $duplicate_runtime ); +$duplicate_owned = $retained_runtime_method->invoke( null, $ownership_workspace, 'batches/batch-duplicate.json', 'batches/1.json', $ownership_dir . '/legacy-duplicate.json', array( 'https://ownership.test/a/', 'https://ownership.test/b/' ) ); +if ( null !== $duplicate_owned || null !== $ownership_workspace->read_raw( 'batches/batch-duplicate.json' ) ) { throw new RuntimeException( 'retained batch payloads with colliding explicit routes must be recollected after route canonicalization upgrades' ); } +$ownership_workspace->purge(); +$cache_dir = sys_get_temp_dir() . '/ssi-response-cache-' . bin2hex( random_bytes( 4 ) ); wp_mkdir_p( $cache_dir ); $network_calls = 0; $cache_workspace = new Static_Site_Importer_Artifact_Run_Workspace( $cache_dir, 'cache' ); $cache = new Static_Site_Importer_Artifact_Byte_Cache( $cache_workspace, 'http-response' ); $cached_fetch = static function ( string $url, array $args ) use ( &$network_calls, $cache ) { $types = isset( $args['content_types'] ) ? $args['content_types'] : null; if ( is_array( $types ) ) { sort( $types ); } $key = hash( 'sha256', $url . "\n" . wp_json_encode( array( 'max_bytes' => $args['max_bytes'] ?? null, 'content_types' => $types, 'timeout' => $args['timeout'] ?? null ) ) ); $cached = $cache->get( $key ); if ( is_array( $cached ) ) { $cache->hit(); return array( 'body' => $cached['bytes'], 'metadata' => $cached['value'] ); } $cache->miss(); $network_calls++; $response = array( 'body' => 'body-' . $network_calls, 'metadata' => array( 'content_type' => 'text/plain', 'final_url' => $url ) ); $cache->put( $key, $response['body'], $response['metadata'] ); return $response; }; +$cached_fetch( 'https://cache.test/a', array( 'max_bytes' => 10, 'content_types' => array() ) ); $cached_fetch( 'https://cache.test/a', array( 'max_bytes' => 10, 'content_types' => array() ) ); $cached_fetch( 'https://cache.test/a', array( 'max_bytes' => 11, 'content_types' => array() ) ); +if ( 2 !== $network_calls || 1 !== ( $cache->evidence()['hits'] ?? 0 ) ) { throw new RuntimeException( 'response cache must hit compatible constraints and miss incompatible ones' ); } +$corrupt = glob( $cache_workspace->directory() . '/cache/http-response/*.entry' ) ?: array(); foreach ( $corrupt as $path ) { file_put_contents( $path, 'corrupt' ); } $cached_fetch( 'https://cache.test/a', array( 'max_bytes' => 10, 'content_types' => array() ) ); +if ( 3 !== $network_calls || 1 > ( $cache->evidence()['corrupt_entries'] ?? 0 ) ) { throw new RuntimeException( 'corrupt response cache entries must recover as misses' ); } +$cache_workspace->purge(); if ( is_dir( $cache_workspace->directory() ) ) { throw new RuntimeException( 'successful response cache cleanup must remove payloads' ); } +$poison_calls = 0; $poison_dir = sys_get_temp_dir() . '/ssi-poison-cache-' . bin2hex( random_bytes( 4 ) ); +$poison_request = array( 'url' => 'https://poison.test/', 'work_dir' => $poison_dir, 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'request_delay_ms' => 0 ) ); +$poison_fetcher = static function ( string $url, array $args ) use ( &$poison_calls ) { if ( 'https://poison.test/sitemap.xml' === $url ) { return array( 'body' => 'https://poison.test/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } $poison_calls++; return array( 'body' => 1 === $poison_calls ? '
' . str_repeat( '', 20 ) : '
' . str_repeat( 'server-rendered ', 100 ) . '
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); }; +$poisoned = Static_Site_Importer_URL_Batch_Import::import( $poison_request, array(), $poison_fetcher, static fn() => array() ); +$recovered = Static_Site_Importer_URL_Batch_Import::import( $poison_request, array(), $poison_fetcher, static fn() => array( 'theme_slug' => 'recovered', 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( ! is_wp_error( $poisoned ) || 'static_site_importer_url_client_rendered_app' !== $poisoned->get_error_code() || is_wp_error( $recovered ) || 2 !== $poison_calls ) { throw new RuntimeException( 'transient client-rendered HTML must fail truthfully without poisoning a resumable response cache' ); } +$tiny_calls = 0; $tiny_dir = sys_get_temp_dir() . '/ssi-tiny-cache-' . bin2hex( random_bytes( 4 ) ); wp_mkdir_p( $tiny_dir ); $tiny_workspace = new Static_Site_Importer_Artifact_Run_Workspace( $tiny_dir, 'cache' ); $tiny = new Static_Site_Importer_Artifact_Byte_Cache( $tiny_workspace, 'payload', 1, 1 ); $tiny_fetch = static function () use ( &$tiny_calls, $tiny ) { $tiny_calls++; $tiny->put( 'a', 'body', array() ); }; $tiny_fetch(); $tiny_fetch(); +if ( 2 !== $tiny_calls || 2 !== ( $tiny->evidence()['bypassed'] ?? 0 ) || glob( $tiny_workspace->directory() . '/cache/payload/*.entry' ) ) { throw new RuntimeException( 'cache guards must bypass writes without creating entries' ); } +$tiny_workspace->purge(); +$negative_dir = sys_get_temp_dir() . '/ssi-negative-cache-' . bin2hex( random_bytes( 4 ) ); wp_mkdir_p( $negative_dir ); $negative_workspace = new Static_Site_Importer_Artifact_Run_Workspace( $negative_dir, 'cache' ); $negative = new Static_Site_Importer_Artifact_Byte_Cache( $negative_workspace, 'http-response' ); +$negative->put_failure( 'transient', array( 'code' => 'transient_timeout', 'message' => 'retry', 'data' => array() ), 130 ); $negative_hit = $negative->get_failure( 'transient', 100 ); $negative_expired = $negative->get_failure( 'transient', 130 ); $negative->put( 'transient', 'recovered', array( 'content_type' => 'text/plain' ) ); $negative_recovered = $negative->get( 'transient' ); +if ( 'transient_timeout' !== ( $negative_hit['code'] ?? '' ) || null !== $negative_expired || 'recovered' !== ( $negative_recovered['bytes'] ?? '' ) || 1 !== ( $negative->evidence()['negative_writes'] ?? 0 ) || 1 !== ( $negative->evidence()['negative_hits'] ?? 0 ) || 1 !== ( $negative->evidence()['negative_expired'] ?? 0 ) || 1 !== ( $negative->evidence()['network_requests_avoided'] ?? 0 ) ) { throw new RuntimeException( 'negative cache entries must avoid repeated requests only until expiry, then permit successful replacement' ); } +$negative_workspace->purge(); +$delay_calls = 0; $shared_asset_calls = 0; +$delay_result = Static_Site_Importer_URL_Batch_Import::import( array( 'url' => 'https://delay.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-delay-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'request_delay_ms' => 1, '_static_site_importer_delay_callback' => static function () use ( &$delay_calls ) { $delay_calls++; } ) ), array(), static function ( string $url, array $args ) use ( &$shared_asset_calls ) { if ( 'https://delay.test/sitemap.xml' === $url ) { return array( 'body' => 'https://delay.test/https://delay.test/p/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://delay.test/shared.png' === $url ) { $shared_asset_calls++; return array( 'body' => 'asset', 'metadata' => array( 'content_type' => 'image/png', 'final_url' => $url ) ); } return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); }, static fn() => array( 'theme_slug' => 'delay', 'import_report_summary' => array( 'status' => 'completed' ) ) ); +if ( is_wp_error( $delay_result ) || 1 !== $shared_asset_calls || 3 !== $delay_calls || 1 > ( $delay_result['url_batch_run']['fetch_cache']['network_requests_avoided'] ?? 0 ) ) { throw new RuntimeException( 'cached shared assets must avoid request delay while cache misses retain it' ); } +$negative_asset_calls = 0; $negative_delays = 0; +$negative_request = array( 'url' => 'https://negative.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-negative-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'fetch_attempts' => 2, 'request_delay_ms' => 1, '_static_site_importer_delay_callback' => static function () use ( &$negative_delays ) { $negative_delays++; } ) ); +$negative_fetcher = static function ( string $url, array $args ) use ( &$negative_asset_calls ) { if ( 'https://negative.test/sitemap.xml' === $url ) { return array( 'body' => 'https://negative.test/https://negative.test/p/', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://negative.test/shared.png' === $url ) { $negative_asset_calls++; return new WP_Error( 'asset_timeout', 'temporary failure' ); } return array( 'body' => '', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); }; +$negative_result = Static_Site_Importer_URL_Batch_Import::import( $negative_request, array(), $negative_fetcher, static fn() => array( 'theme_slug' => 'negative', 'import_report_summary' => array( 'status' => 'completed' ) ) ); +$negative_resume = Static_Site_Importer_URL_Batch_Import::import( $negative_request, array(), $negative_fetcher, static fn() => array() ); +if ( is_wp_error( $negative_result ) || is_wp_error( $negative_resume ) || 2 !== $negative_asset_calls || 3 !== $negative_delays || 1 > ( $negative_result['url_batch_run']['fetch_cache']['negative_writes'] ?? 0 ) || 2 !== ( $negative_result['url_batch_run']['external_asset_retained']['count'] ?? 0 ) ) { throw new RuntimeException( 'negative cache failures must retain optional singleton assets without exposing internal cache hooks' ); } +echo "URL batch import smoke passed.\n"; diff --git a/tests/smoke-url-import-runtime.php b/tests/smoke-url-import-runtime.php index f91cb07d..f9764d52 100644 --- a/tests/smoke-url-import-runtime.php +++ b/tests/smoke-url-import-runtime.php @@ -165,6 +165,7 @@ static function ( mixed $output, array $request ): array { 'overwrite' => true, 'site_title' => 'Private Import', 'stale_page_action' => 'draft', + 'require_proven_dynamic_client_assets' => false, 'source_metadata' => array( 'requested_by' => 'external-caller' ), ) ); @@ -175,10 +176,14 @@ static function ( mixed $output, array $request ): array { $assert( 'private-import' === ( $result['theme_slug'] ?? '' ), 'result-passes-through' ); $assert( 'website/index.html' === ( Static_Site_Importer_Theme_Generator::$last_artifact['files'][0]['path'] ?? '' ), 'provider-artifact-imported' ); $assert( true === ( Static_Site_Importer_Theme_Generator::$last_args['overwrite'] ?? null ), 'import-args-preserved' ); +$assert( false === ( Static_Site_Importer_Theme_Generator::$last_args['require_proven_dynamic_client_assets'] ?? null ), 'url-import-forwards-dynamic-client-policy' ); $assert( 'external-caller' === ( Static_Site_Importer_Theme_Generator::$last_args['source_metadata']['requested_by'] ?? '' ), 'caller-metadata-preserved' ); $assert( 'private' === ( Static_Site_Importer_Theme_Generator::$last_args['source_metadata']['visibility'] ?? '' ), 'provider-metadata-merged' ); $assert( 'test-private-runtime' === ( Static_Site_Importer_Theme_Generator::$last_args['source_metadata']['url_import_provider'] ?? '' ), 'provider-recorded' ); +$batch_provider_result = Static_Site_Importer_URL_Import_Runtime::import_url( array( 'url' => 'private.example.test/', 'slug' => 'private-batch-request', 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 2 ) ) ); +$assert( ! is_wp_error( $batch_provider_result ) && 'private-batch-request' === ( $batch_provider_result['theme_slug'] ?? '' ) && empty( $batch_provider_result['url_batch_run'] ), 'external-provider-keeps-normal-import-contract-for-batch-request' ); + $runtime_artifact = Static_Site_Importer_URL_Import_Runtime::website_artifact_from_url( array( 'url' => 'facebook.com', @@ -196,6 +201,8 @@ static function ( mixed $output, array $request ): array { $server_rendered_diagnostic = Static_Site_Importer_URL_Fetcher::html_source_diagnostic( 'Server

Server rendered

' . str_repeat( 'Useful page content. ', 80 ) . '

' ); $assert( array() === $server_rendered_diagnostic, 'server-rendered-html-not-flagged-as-client-shell' ); +$framework_server_rendered_html = '' . str_repeat( '', 42 ) . str_repeat( ' ', 100000 ) . '

Billboard

' . str_repeat( 'Server content. ', 50 ) . '

'; +$assert( array() === Static_Site_Importer_URL_Fetcher::html_source_diagnostic( $framework_server_rendered_html ), 'framework-heavy-server-html-not-flagged-as-client-shell' ); $ability = static_site_importer_ability_import_url( array( diff --git a/tests/smoke-url-site-collector.php b/tests/smoke-url-site-collector.php new file mode 100644 index 00000000..8941384f --- /dev/null +++ b/tests/smoke-url-site-collector.php @@ -0,0 +1,271 @@ +code; + } + public function get_error_message(): string { + return $this->message; + } + public function get_error_data(): mixed { + return $this->data; + } + } +} + +if ( ! function_exists( 'is_wp_error' ) ) { + function is_wp_error( mixed $value ): bool { + return $value instanceof WP_Error; + } +} + +if ( ! function_exists( 'sanitize_file_name' ) ) { + function sanitize_file_name( string $name ): string { + return trim( (string) preg_replace( '/[^A-Za-z0-9._-]+/', '-', $name ), '-' ); + } +} + +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-url-fetcher.php'; +require_once dirname( __DIR__ ) . '/includes/class-static-site-importer-url-site-collector.php'; +require_once dirname( __DIR__ ) . '/vendor/autoload.php'; +require_once dirname( __DIR__ ) . '/vendor/automattic/blocks-engine-php-transformer/php-transformer/php-transformer.php'; + +$responses = array( + 'https://example.test/sitemap.xml' => array( + 'content_type' => 'application/xml', + 'body' => 'https://example.test/index.htmlhttps://example.test/services.htmlhttps://example.test/team.htmlhttps://example.test/contact.html', + ), + 'https://example.test/' => array( + 'content_type' => 'text/html; charset=utf-8', + 'body' => '

Home

', + ), + 'https://example.test/services.html' => array( + 'content_type' => 'text/html', + 'body' => 'Team

Services

', + ), + 'https://example.test/team.html' => array( + 'content_type' => 'text/html', + 'body' => '

Team

', + ), + 'https://example.test/contact.html' => array( + 'content_type' => 'text/html', + 'body' => '

Contact

', + ), + 'https://example.test/files/main.css?v=1' => array( + 'content_type' => 'text/css', + 'body' => '@import "components.css";@font-face{src:url("https://cdn.example.test/font.woff2")}body{background:url(../uploads/pattern.svg)}', + ), + 'https://example.test/files/components.css' => array( 'content_type' => 'text/css', 'body' => '.component{display:block}' ), + 'https://example.test/platform-runtime.js' => array( 'content_type' => 'application/javascript', 'body' => 'window.platformRuntime = true;' ), + 'https://example.test/uploads/hero.jpg' => array( 'content_type' => 'image/jpeg', 'body' => "\xff\xd8hero" ), + 'https://example.test/uploads/logo.png' => array( 'content_type' => 'image/png', 'body' => "\x89PNGlogo" ), + 'https://example.test/uploads/logo-2x.png' => array( 'content_type' => 'image/png', 'body' => "\x89PNGlogo2" ), + 'https://example.test/uploads/pattern.svg' => array( 'content_type' => 'image/svg+xml', 'body' => '' ), + 'https://cdn.example.test/team.webp' => array( 'content_type' => 'image/webp', 'body' => 'webp-team' ), + 'https://cdn.example.test/font.woff2' => array( 'content_type' => 'font/woff2', 'body' => 'woff2-font' ), +); + +$requests = array(); +$fetcher = static function ( string $url, array $args ) use ( &$requests, $responses ) { + $requests[] = array( 'url' => $url, 'args' => $args ); + if ( ! isset( $responses[ $url ] ) ) { + return new WP_Error( 'missing_fixture', 'No response fixture for ' . $url ); + } + $response = $responses[ $url ]; + return array( + 'body' => $response['body'], + 'metadata' => array( 'content_type' => $response['content_type'], 'source_url' => $url, 'final_url' => $url ), + ); +}; + +$result = Static_Site_Importer_URL_Site_Collector::collect( + 'https://example.test/', + array( + 'max_pages' => 10, + 'max_assets' => 20, + 'max_bytes' => PHP_INT_MAX, + 'request_delay_ms' => 0, + ), + $fetcher +); + +$assertions = 0; +$failures = array(); +$assert = static function ( bool $condition, string $label, string $detail = '' ) use ( &$assertions, &$failures ): void { + ++$assertions; + if ( ! $condition ) { + $failures[] = 'FAIL [' . $label . ']' . ( '' !== $detail ? ': ' . $detail : '' ); + } +}; + +$assert( ! is_wp_error( $result ), 'collection-succeeds', is_wp_error( $result ) ? $result->get_error_message() : '' ); +$assert( 'public-static-site-collector' === ( $result['provider'] ?? '' ), 'provider-recorded' ); +$assert( 'website/index.html' === ( $result['artifact']['entrypoint'] ?? '' ), 'root-entrypoint' ); +$assert( array( 'max_files' => 70, 'max_file_bytes' => 10485760, 'max_total_bytes' => 104857600 ) === ( $result['artifact']['compiler_limits'] ?? null ), 'collector-declares-bounded-compiler-limits' ); +$assert( 4 === ( $result['source_metadata']['collection']['pages'] ?? 0 ), 'sitemap-index-alias-deduplicated' ); +$assert( 9 === ( $result['source_metadata']['collection']['assets'] ?? 0 ), 'html-css-and-script-assets-collected' ); +$assert( array() === ( $result['source_metadata']['collection']['failures'] ?? null ), 'no-collection-failures' ); +$snapshot = $result['source_metadata']['snapshot'] ?? array(); +$assert( 'static-site-importer/url-snapshot/v1' === ( $snapshot['schema'] ?? '' ) && 64 === strlen( (string) ( $snapshot['sha256'] ?? '' ) ), 'snapshot-hash-recorded' ); +$assert( count( $result['artifact']['files'] ?? array() ) === count( $snapshot['files'] ?? array() ) && array() === array_filter( $snapshot['files'] ?? array(), static fn ( array $file ): bool => 64 !== strlen( (string) ( $file['sha256'] ?? '' ) ) ), 'snapshot-records-every-file-hash' ); + +$files = array(); +foreach ( $result['artifact']['files'] ?? array() as $file ) { + $files[ $file['path'] ?? '' ] = $file; +} +$assert( isset( $files['website/services.html'], $files['website/team.html'], $files['website/contact.html'] ), 'all-pages-packaged' ); +$assert( '/' === ( $files['website/index.html']['metadata']['route_path'] ?? null ) && '/services' === ( $files['website/services.html']['metadata']['route_path'] ?? null ), 'html-files-declare-canonical-source-routes' ); +$assert( isset( $files['website/files/main-a798de8e.css'] ), 'query-addressed-stylesheet-packaged' ); +$assert( isset( $files['website/_external/cdn.example.test/font.woff2'] ), 'external-font-packaged' ); +$assert( isset( $files['website/_external/cdn.example.test/team.webp'] ), 'external-image-packaged' ); +$assert( isset( $files['website/files/components.css'] ), 'quoted-css-import-packaged' ); +$assert( str_contains( (string) ( $files['website/index.html']['content'] ?? '' ), 'href="/services.html"' ), 'page-link-preserved-for-route-rewriting' ); +$assert( str_contains( (string) ( $files['website/index.html']['content'] ?? '' ), 'src="uploads/logo.png"' ), 'image-link-rewritten' ); +$assert( str_contains( (string) ( $files['website/index.html']['content'] ?? '' ), 'url(uploads/hero.jpg)' ), 'inline-background-rewritten' ); +$assert( isset( $files['website/uploads/logo.png']['content_base64'] ), 'binary-assets-base64-encoded' ); +$assert( ! in_array( 'https://example.test/index.html', array_column( $requests, 'url' ), true ), 'root-index-not-fetched-twice' ); +$assert( array() === array_filter( array_column( $requests, 'url' ), static fn ( string $url ): bool => str_contains( $url, 'new Blob' ) ), 'inline-javascript-url-functions-are-not-css-assets' ); +$assert( in_array( 'https://example.test/platform-runtime.js', array_column( $requests, 'url' ), true ), 'remote-scripts-collected-by-default' ); +$assert( isset( $files['website/platform-runtime.js']['content'] ), 'script-payload-packaged' ); +$assert( str_contains( (string) ( $files['website/index.html']['content'] ?? '' ), 'href="mailto:a@b.co"' ), 'cloudflare-email-link-decoded' ); +$assert( ! in_array( 'https://example.test/cdn-cgi/l/email-protection', array_column( $requests, 'url' ), true ), 'cloudflare-email-action-not-crawled-as-page' ); +$assert( ! str_contains( (string) ( $files['website/index.html']['content'] ?? '' ), 'weebly-footer-signup-container-v3' ), 'platform-attribution-removed-before-packaging' ); +$assert( ! in_array( 'https://cdn.example.test/platform-badge.png', array_column( $requests, 'url' ), true ), 'excluded-platform-assets-not-collected' ); +$source_exclusions = $result['source_metadata']['collection']['source_exclusions'] ?? array(); +$assert( 1 === count( $source_exclusions ) && 'platform_attribution_removed' === ( $source_exclusions[0]['reason_code'] ?? '' ) && 64 === strlen( (string) ( $source_exclusions[0]['removed_sha256'] ?? '' ) ), 'platform-attribution-removal-retains-receipt' ); +$assert( 10485760 >= max( array_map( static fn ( array $request ): int => (int) ( $request['args']['max_bytes'] ?? 0 ), $requests ) ), 'configured-response-limit-hard-clamped' ); +$artifact_paths = array_column( $result['artifact']['files'] ?? array(), 'path' ); +$sorted_paths = $artifact_paths; +sort( $sorted_paths, SORT_STRING ); +$assert( $sorted_paths === $artifact_paths, 'artifact-file-order-is-canonical' ); + +$shuffled_responses = $responses; +$shuffled_responses['https://example.test/sitemap.xml']['body'] = 'https://example.test/team.htmlhttps://example.test/contact.htmlhttps://example.test/services.htmlhttps://example.test/index.html'; +$shuffled = Static_Site_Importer_URL_Site_Collector::collect( + 'https://example.test/', + array( 'max_pages' => 10, 'max_assets' => 20, 'max_bytes' => PHP_INT_MAX, 'request_delay_ms' => 0 ), + static function ( string $url, array $args ) use ( $shuffled_responses ) { + unset( $args ); + if ( ! isset( $shuffled_responses[ $url ] ) ) { + return new WP_Error( 'missing_fixture', $url ); + } + $response = $shuffled_responses[ $url ]; + return array( 'body' => $response['body'], 'metadata' => array( 'content_type' => $response['content_type'], 'final_url' => $url ) ); + } +); +$assert( ! is_wp_error( $shuffled ) && ( $snapshot['sha256'] ?? '' ) === ( $shuffled['source_metadata']['snapshot']['sha256'] ?? null ), 'snapshot-hash-independent-of-discovery-order' ); + +$compiled = blocks_engine_php_transformer_compile_artifact( $result['artifact'] ); +$site_plan = $compiled['source_reports']['wordpress_site_plan'] ?? array(); +$site_diagnostics = $compiled['source_reports']['wordpress_site_plan_diagnostics'] ?? array(); +$routes = array_column( $site_plan['routes'] ?? array(), 'target_path', 'source_path' ); +$assert( array() === $site_diagnostics, 'collected-artifact-is-self-contained', json_encode( $site_diagnostics ) ?: '' ); +$assert( 4 === count( $routes ), 'collected-artifact-produces-four-routes', json_encode( $routes ) ?: '' ); +$assert( '/' === ( $routes['website/index.html'] ?? null ) && '/services' === ( $routes['website/services.html'] ?? null ), 'collected-routes-preserve-source-paths' ); + +$encoded_route = Static_Site_Importer_URL_Site_Collector::collect( + 'https://example.test/news/category/Americana%2FCountry+Artist', + array( 'max_pages' => 1, 'max_assets' => 0, 'max_bytes' => PHP_INT_MAX, 'request_delay_ms' => 0, '_route_set' => array( 'https://example.test/news/category/Americana%2FCountry+Artist' ) ), + static fn ( string $url, array $args ): array => array( 'body' => '
Category
', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ) +); +$encoded_file = $encoded_route['artifact']['files'][0] ?? array(); +$assert( '/news/category/americana-country-artist' === ( $encoded_file['metadata']['route_path'] ?? null ), 'encoded-source-route-is-canonicalized' ); + +$colliding_urls = array( 'https://example.test/news/tag/inc-+richlyn+marketing', 'https://example.test/news/tag/inc-richlyn+marketing' ); +$colliding_routes = Static_Site_Importer_URL_Site_Collector::collect( + $colliding_urls[0], + array( 'max_pages' => 2, 'max_assets' => 0, 'max_bytes' => PHP_INT_MAX, 'request_delay_ms' => 0, '_route_set' => $colliding_urls ), + static fn ( string $url, array $args ): array => array( 'body' => '

Tag

Server-rendered tag archive.

', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ) +); +$colliding_files = is_wp_error( $colliding_routes ) ? array() : array_filter( $colliding_routes['artifact']['files'], static fn ( array $file ): bool => 'text/html' === ( $file['mime_type'] ?? '' ) ); +$colliding_paths = array_column( $colliding_files, 'metadata' ); +$colliding_paths = array_column( $colliding_paths, 'route_path' ); +$colliding_compiled = is_wp_error( $colliding_routes ) ? array() : blocks_engine_php_transformer_compile_artifact( $colliding_routes['artifact'] ); +$colliding_diagnostics = $colliding_compiled['source_reports']['wordpress_site_plan_diagnostics'] ?? array(); +$route_collision_diagnostics = array_filter( $colliding_diagnostics, static fn ( array $diagnostic ): bool => str_contains( (string) ( $diagnostic['message'] ?? '' ), 'colliding page routes' ) ); +$assert( 2 === count( array_unique( $colliding_paths ) ) && array() === $route_collision_diagnostics, 'canonical-route-collisions-receive-stable-distinct-routes', json_encode( array( 'routes' => $colliding_paths, 'diagnostics' => $colliding_diagnostics ) ) ?: '' ); + +$complete_result = Static_Site_Importer_URL_Site_Collector::collect( + 'https://example.test/', + array( + 'max_pages' => 1, + 'max_assets' => 20, + 'max_bytes' => PHP_INT_MAX, + 'request_delay_ms' => 0, + 'require_complete_collection' => true, + ), + $fetcher +); +$assert( is_wp_error( $complete_result ) && 'static_site_importer_site_collection_incomplete' === $complete_result->get_error_code(), 'complete-collection-rejects-truncation' ); +$complete_error_data = is_wp_error( $complete_result ) ? $complete_result->get_error_data() : array(); +$assert( array( 'pages' ) === ( $complete_error_data['collection']['truncated'] ?? null ) && 1 === ( $complete_error_data['limits']['max_pages'] ?? null ), 'complete-collection-reports-reached-limits' ); + +$incomplete_responses = $responses; +unset( $incomplete_responses['https://example.test/uploads/logo.png'] ); +$incomplete_fetcher = static function ( string $url, array $args ) use ( $incomplete_responses ) { + unset( $args ); + if ( ! isset( $incomplete_responses[ $url ] ) ) { + return new WP_Error( 'missing_fixture', 'No response fixture for ' . $url ); + } + $response = $incomplete_responses[ $url ]; + return array( + 'body' => $response['body'], + 'metadata' => array( 'content_type' => $response['content_type'], 'source_url' => $url, 'final_url' => $url ), + ); +}; +$incomplete_result = Static_Site_Importer_URL_Site_Collector::collect( 'https://example.test/', array( 'max_pages' => 10, 'max_assets' => 20, 'request_delay_ms' => 0, 'require_complete_collection' => true ), $incomplete_fetcher ); +$assert( is_wp_error( $incomplete_result ) && 'missing_fixture' === ( $incomplete_result->get_error_data()['collection']['failures'][0]['code'] ?? null ), 'complete-collection-rejects-fetch-failures' ); + +$redirect_responses = array( + 'https://redirect.test/sitemap.xml' => array( 'content_type' => 'application/xml', 'body' => 'https://redirect.test/https://redirect.test/go' ), + 'https://redirect.test/' => array( 'content_type' => 'text/html', 'body' => '

Home

Docs
' ), + 'https://redirect.test/go' => array( 'content_type' => 'text/html', 'final_url' => 'https://redirect.test/docs/', 'body' => '

Docs

Child
' ), + 'https://redirect.test/docs/child.html' => array( 'content_type' => 'text/html', 'body' => '

Child

' ), + 'https://redirect.test/static/style.css' => array( 'content_type' => 'text/css', 'final_url' => 'https://redirect.test/assets/css/style.css', 'body' => '@import "theme.css";body{background:url(../background.png)}' ), + 'https://redirect.test/assets/css/theme.css' => array( 'content_type' => 'text/css', 'body' => 'body{color:#000}' ), + 'https://redirect.test/assets/background.png' => array( 'content_type' => 'image/png', 'body' => "\x89PNGredirect" ), + 'https://redirect.test/hero.png' => array( 'content_type' => 'image/png', 'body' => "\x89PNGhero" ), +); +$redirect_requests = array(); +$redirect_fetcher = static function ( string $url, array $args ) use ( &$redirect_requests, $redirect_responses ) { + $redirect_requests[] = $url; + if ( ! isset( $redirect_responses[ $url ] ) ) { + return new WP_Error( 'missing_redirect_fixture', 'No response fixture for ' . $url ); + } + $response = $redirect_responses[ $url ]; + return array( + 'body' => $response['body'], + 'metadata' => array( 'content_type' => $response['content_type'], 'source_url' => $url, 'final_url' => $response['final_url'] ?? $url ), + ); +}; +$redirect_result = Static_Site_Importer_URL_Site_Collector::collect( 'https://redirect.test/', array( 'request_delay_ms' => 0 ), $redirect_fetcher ); +$redirect_paths = array_column( $redirect_result['artifact']['files'] ?? array(), 'path' ); +$redirect_files = array_column( $redirect_result['artifact']['files'] ?? array(), null, 'path' ); +$assert( in_array( 'https://redirect.test/docs/child.html', $redirect_requests, true ), 'redirected-html-relative-link-uses-final-url' ); +$assert( in_array( 'https://redirect.test/static/style.css', $redirect_requests, true ), 'html-base-url-applied' ); +$assert( in_array( 'https://redirect.test/hero.png', $redirect_requests, true ), 'unquoted-html-asset-collected' ); +$assert( in_array( 'https://redirect.test/assets/css/theme.css', $redirect_requests, true ), 'redirected-css-import-uses-final-url' ); +$assert( in_array( 'https://redirect.test/assets/background.png', $redirect_requests, true ), 'redirected-css-url-uses-final-url' ); +$assert( in_array( 'website/docs/index.html', $redirect_paths, true ) && in_array( 'website/assets/css/style.css', $redirect_paths, true ), 'redirected-resources-use-final-identities' ); +$assert( str_contains( (string) ( $redirect_files['website/index.html']['content'] ?? '' ), 'href="/docs/"' ), 'redirected-page-link-rewritten-to-final-route' ); + +if ( ! empty( $failures ) ) { + fwrite( STDERR, implode( PHP_EOL, $failures ) . PHP_EOL ); + exit( 1 ); +} + +echo sprintf( "URL site collector smoke passed (%d assertions).\n", $assertions ); diff --git a/tests/smoke-website-artifact-import-input.php b/tests/smoke-website-artifact-import-input.php index a9c0f733..9f322c24 100644 --- a/tests/smoke-website-artifact-import-input.php +++ b/tests/smoke-website-artifact-import-input.php @@ -108,9 +108,10 @@ public static function import_website_artifact( array $artifact, array $args = a 'overwrite' => true, 'fail_on_quality' => true, 'allow_missing_woocommerce' => true, - 'allow_missing_jetpack' => true, - 'materialize_dependencies' => false, - 'seed_entities' => true, + 'allow_missing_jetpack' => true, + 'materialize_dependencies' => false, + 'require_proven_dynamic_client_assets' => false, + 'seed_entities' => true, 'products_manifest' => array( 'products' => array() ), 'commerce_context' => array( 'currency' => 'USD' ), 'report' => '/tmp/report.json', diff --git a/tests/smoke-wordpress-site-plan-materializer.php b/tests/smoke-wordpress-site-plan-materializer.php index 82afe75f..76ee7261 100644 --- a/tests/smoke-wordpress-site-plan-materializer.php +++ b/tests/smoke-wordpress-site-plan-materializer.php @@ -68,11 +68,12 @@ function update_option( string $key, $value ): void { $GLOBALS['ssi_plan_options function switch_theme( string $slug ): void { $GLOBALS['ssi_plan_options']['stylesheet'] = $slug; } function sanitize_text_field( string $value ): string { return $value; } function update_post_meta( int $id, string $key, string $value ): void { $GLOBALS['ssi_plan_meta'][ $id ][ $key ] = $value; } +function get_post_meta( int $id, string $key, bool $single = true ): string { return (string) ( $GLOBALS['ssi_plan_meta'][ $id ][ $key ] ?? '' ); } function get_posts( array $args ): array { foreach ( $GLOBALS['ssi_plan_meta'] as $id => $meta ) { - if ( ( $meta[ $args['meta_key'] ] ?? null ) === $args['meta_value'] ) { return array( new WP_Post( $id ) ); } + if ( isset( $meta[ $args['meta_key'] ] ) && ( ! isset( $args['meta_value'] ) || $meta[ $args['meta_key'] ] === $args['meta_value'] ) ) { $matches[] = new WP_Post( $id ); } } - return array(); + return $matches ?? array(); } function get_page_by_path( string $slug, $output, string $type ) { foreach ( $GLOBALS['ssi_plan_posts'] as $id => $post ) { if ( $post['post_name'] === $slug ) { return new WP_Post( $id ); } } @@ -351,6 +352,7 @@ function wp_insert_post( array $post, bool $wp_error ) { $GLOBALS['ssi_plan_options'] = array( 'show_on_front' => 'posts', 'page_on_front' => 0, 'blogname' => 'Before' ); $preview = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $plan, array( 'slug' => 'site-plan', 'overwrite' => true ) ); $assert( 'completed' === $preview['status'], 'preview materialization completes' ); +$assert( array( 'canonical_validations' => 1, 'plan_resolutions' => 1, 'destination_preflights' => 2, 'immutable_projection_reused' => true ) === ( $preview['preparation'] ?? array() ), 'materialization reuses one immutable projection while repeating destination preflight' ); $assert( 'posts' === $GLOBALS['ssi_plan_options']['show_on_front'] && ! isset( $GLOBALS['ssi_plan_options']['stylesheet'] ), 'activate=false preserves runtime options' ); $activated = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $plan, array( 'slug' => 'site-plan', 'overwrite' => true, 'activate' => true, 'site_title' => 'Activated Plan' ) ); $assert( 'site-plan' === $GLOBALS['ssi_plan_options']['stylesheet'] && 'page' === $GLOBALS['ssi_plan_options']['show_on_front'] && 'Activated Plan' === $GLOBALS['ssi_plan_options']['blogname'], 'activate=true applies theme title and reading policy' ); @@ -368,6 +370,17 @@ function wp_insert_post( array $post, bool $wp_error ) { $assert( $before_posts === count( $GLOBALS['ssi_plan_posts'] ), 'invalid plan creates no posts' ); $assert( $before_files === count( glob( $GLOBALS['ssi_plan_root'] . '/reject/**/*' ) ?: array() ), 'invalid plan writes no files' ); +$tampered_prepared = Static_Site_Importer_WordPress_Site_Plan_Materializer::prepare( $plan, array( 'slug' => 'tampered-prepared', 'overwrite' => true ) ); +$tampered_prepared['base_resolved']['pages'][0]['resolved_block_markup'] .= '

tampered

'; +$tampered_result = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize_prepared( $tampered_prepared ); +$assert( 'rejected' === $tampered_result['status'] && 'prepared_projection_changed' === ( $tampered_result['diagnostics'][0]['reason_code'] ?? '' ), 'changed immutable prepared projections are rejected before mutation' ); + +$destination_prepared = Static_Site_Importer_WordPress_Site_Plan_Materializer::prepare( $plan, array( 'slug' => 'changed-prepared-destination', 'overwrite' => true ) ); +symlink( sys_get_temp_dir(), $GLOBALS['ssi_plan_root'] . '/changed-prepared-destination' ); +$destination_changed = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize_prepared( $destination_prepared ); +unlink( $GLOBALS['ssi_plan_root'] . '/changed-prepared-destination' ); +$assert( 'rejected' === $destination_changed['status'] && 'unsafe_theme_destination' === ( $destination_changed['diagnostics'][0]['reason_code'] ?? '' ), 'mutable destination safety is rechecked immediately before writes' ); + $unsafe = $GLOBALS['ssi_plan_root'] . '/unsafe'; mkdir( $unsafe, 0777, true ); symlink( sys_get_temp_dir(), $unsafe . '/assets' ); @@ -396,6 +409,9 @@ function wp_insert_post( array $post, bool $wp_error ) { $assert( $dynamic_before_posts === $GLOBALS['ssi_plan_posts'] && $dynamic_before_meta === $GLOBALS['ssi_plan_meta'] && $dynamic_before_options === $GLOBALS['ssi_plan_options'], 'materialization rejects external dynamic scripts before page or option mutation' ); $assert( ! is_dir( $GLOBALS['ssi_plan_root'] . '/external-dynamic-plan' ), 'materialization rejects external dynamic scripts before file mutation' ); +$dynamic_allowed = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $external_dynamic_plan, array( 'slug' => 'allowed-external-dynamic-plan', 'require_proven_dynamic_client_assets' => false ) ); +$assert( 'completed' === $dynamic_allowed['status'], 'explicit policy can preserve unproven dynamic client scripts' ); + $dynamic_artifact = $artifact; $dynamic_artifact['files']['index.html'] .= ''; $dynamic_artifact['files']['assets/site.js'] = 'window.sitePlan = true;'; @@ -467,4 +483,21 @@ function wp_insert_post( array $post, bool $wp_error ) { $assert( 'partial' === $partial['status'], 'runtime mutation failure returns partial receipt' ); $assert( 'simulated_post_failure' === $partial['diagnostics'][0]['reason_code'], 'partial receipt keeps mutation failure identity' ); +$GLOBALS['ssi_plan_posts'] = array(); +$GLOBALS['ssi_plan_meta'] = array(); +$GLOBALS['ssi_plan_fail_after'] = 0; +$parent_plan = ( new ArtifactCompiler() )->compile( array( 'entrypoint' => 'website/index.html', 'files' => array( 'website/index.html' => '
Home
', 'website/about/index.html' => '
About
' ) ) )->toArray()['source_reports']['wordpress_site_plan']; +$child_plan = ( new ArtifactCompiler() )->compile( array( 'entrypoint' => 'website/index.html', 'files' => array( 'website/index.html' => '
Home
', 'website/about/team/index.html' => '
Team
' ) ) )->toArray()['source_reports']['wordpress_site_plan']; +$parent_batch = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $parent_plan, array( 'slug' => 'batch-parent-plan', 'import_run_id' => 'batch-parent-run' ) ); +file_put_contents( $GLOBALS['ssi_plan_root'] . '/batch-parent-plan/static-site-importer-manifest.json', json_encode( array( 'import_run_id' => 'batch-parent-run' ) ) ); +$child_batch = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $child_plan, array( 'slug' => 'batch-parent-plan', 'import_run_id' => 'batch-parent-run', 'preserve_existing_theme_bootstrap' => true, 'overwrite' => true ) ); +$about_id = (int) ( $parent_batch['completed']['pages']['website/about/index.html'] ?? 0 ); +$team_id = (int) ( $child_batch['completed']['pages']['website/about/team/index.html'] ?? 0 ); +$assert( 'completed' === $child_batch['status'] && $about_id > 0 && $about_id === (int) ( $GLOBALS['ssi_plan_posts'][ $team_id ]['post_parent'] ?? 0 ), 'later batch resolves an existing parent only through matching run provenance' ); +$parent_order = new ReflectionMethod( Static_Site_Importer_WordPress_Site_Plan_Materializer::class, 'parent_ordered_pages' ); +$GLOBALS['ssi_plan_posts'][999] = array( 'post_name' => 'external-parent' ); +$GLOBALS['ssi_plan_meta'][999]['_static_site_importer_provenance'] = json_encode( array( 'import_run_id' => 'batch-parent-run', 'source_path' => 'website/external/index.html' ) ); +$descendant_only = $parent_order->invoke( null, array( array( 'source_path' => 'website/external/child/index.html', 'parent_source_path' => 'website/external/index.html' ) ), 'batch-parent-run' ); +$assert( is_array( $descendant_only ) && 1 === count( $descendant_only ) && 'website/external/child/index.html' === ( $descendant_only[0]['source_path'] ?? '' ), 'external provenance parent satisfies ordering without being emitted as a page' ); + echo "WordPress site plan materializer smoke passed.\n"; diff --git a/tools/fixture-matrix.test.mjs b/tools/fixture-matrix.test.mjs index be078fd9..c7c806fd 100644 --- a/tools/fixture-matrix.test.mjs +++ b/tools/fixture-matrix.test.mjs @@ -4,7 +4,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -600,6 +600,29 @@ test('builds a generic WP Codebox recipe with SSI-owned plugin defaults', () => assert.deepEqual(recipe.inputs.mounts, []); }); +test('fixture manifests explicitly opt into unproven dynamic client assets', () => { + const root = mkdtempSync(path.join(tmpdir(), 'ssi-dynamic-client-assets-')); + try { + const fixture = path.join(root, 'websites', 'runtime-site'); + mkdirSync(fixture, { recursive: true }); + writeFileSync(path.join(fixture, 'index.html'), '
Runtime site
'); + writeFileSync(path.join(fixture, 'fixture.json'), JSON.stringify({ + fixture_class: 'marketing/static', + allow_unproven_dynamic_client_assets: true, + })); + const matrix = createFixtureMatrix({ fixture_root: root }); + const recipe = buildFixtureMatrixRecipe({ + matrix, + artifactsDirectory: '/tmp/artifacts', + playgroundArtifactsDirectory: '/wordpress/wp-content/uploads/static-site-importer-fixture-matrix', + staticSiteImporterPath: '/tmp/static-site-importer', + }); + assert.match(recipe.workflow.steps[1].args[0], /--allow-unproven-dynamic-client-assets/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test('matrix import recipes declare the complete required sidecar contract', () => { const matrix = createFixtureMatrix({ fixture_root: fixtureRoot, id: 'complete-sidecar-contract' }); const recipe = buildFixtureMatrixRecipe({ matrix, staticSiteImporterPath: '/tmp/ssi', runId: 'run-1', attemptId: 'attempt-1' }); @@ -2896,9 +2919,10 @@ test('materializes generated artifact roots into matrix-compatible fixtures', () writeFileSync(path.join(sourceRoot, 'static-sites', 'alpha', 'index.html'), '

Alpha

'); writeFileSync(path.join(sourceRoot, 'static-sites', 'alpha', 'assets', 'style.css'), 'body { color: black; }'); mkdirSync(path.join(sourceRoot, 'artifact-candidate'), { recursive: true }); - writeFileSync(path.join(sourceRoot, 'artifact-candidate', 'artifact.json'), JSON.stringify({ + writeFileSync(path.join(sourceRoot, 'artifact-candidate', 'site-artifact.json'), JSON.stringify({ schema: 'blocks-engine/php-transformer/site-artifact/v1', metadata: { site: 'Beta Site' }, + compiler_limits: { max_files: 25, max_file_bytes: 10485760, max_total_bytes: 335544320 }, files: [ { path: 'website/index.html', content: '

Beta

' }, { path: 'website/assets/style.css', content: 'body { color: blue; }' }, @@ -2912,6 +2936,9 @@ test('materializes generated artifact roots into matrix-compatible fixtures', () assert.deepEqual(matrix.fixtures.map((fixture) => fixture.id), ['alpha', 'beta-site']); assert.equal(readFileSync(path.join(fixtureOutput, 'alpha', 'index.html'), 'utf8'), '

Alpha

'); assert.equal(readFileSync(path.join(fixtureOutput, 'beta-site', 'index.html'), 'utf8'), '

Beta

'); + const betaArtifact = buildFixtureArtifact(matrix.fixtures.find((fixture) => fixture.id === 'beta-site')); + assert.deepEqual(betaArtifact.compiler_limits, { max_files: 25, max_file_bytes: 10485760, max_total_bytes: 335544320 }); + assert.equal(betaArtifact.files.some((file) => file.path.includes('generated-artifact-metadata')), false); }); test('resolves Blocks Engine PHP transformer override paths', () => { @@ -6172,7 +6199,7 @@ test('default visual-parity source-url follows nested fixture entrypoint', () => ); }); -test('stageFixtureSource copies the raw fixture source into the served source/ subdir', () => { +test('stageFixtureSource copies the normalized fixture source into the served source/ subdir', () => { const outputDirectory = mkdtempSync(path.join(tmpdir(), 'ssi-visual-parity-stage-')); const matrix = createFixtureMatrix({ fixture_root: fixtureRoot, id: 'visual-parity-stage-test' }); const written = writeFixtureMatrixArtifacts({ outputDirectory, matrix }); @@ -6196,6 +6223,25 @@ test('stageFixtureSource copies the raw fixture source into the served source/ s assert.ok(Number.isFinite(written.metadata.performance.artifact_writing_ms)); }); +test('platform attribution is excluded from both import artifacts and visual baselines', () => { + const fixtureDirectory = mkdtempSync(path.join(tmpdir(), 'ssi-platform-chrome-source-')); + const sourceDirectory = path.join(fixtureDirectory, 'fixture'); + mkdirSync(sourceDirectory, { recursive: true }); + writeFileSync(path.join(sourceDirectory, 'index.html'), '

Authored page

'); + + const fixture = { id: 'Platform Chrome', directory: sourceDirectory }; + const artifact = buildFixtureArtifact(fixture); + const artifactHtml = Buffer.from(artifact.files[0].content_base64, 'base64').toString('utf8'); + assert.doesNotMatch(artifactHtml, /weebly-footer-signup-container-v3/); + assert.equal(artifact.source_metadata.source_exclusions[0].reason_code, 'platform_attribution_removed'); + assert.match(artifact.source_metadata.source_exclusions[0].removed_sha256, /^[a-f0-9]{64}$/); + + stageFixtureSource(fixture, fixtureDirectory); + const stagedHtml = readFileSync(path.join(fixtureDirectory, 'source', 'index.html'), 'utf8'); + assert.match(stagedHtml, /Authored page/); + assert.doesNotMatch(stagedHtml, /weebly-footer-signup-container-v3/); +}); + test('staged visual source uses the generated self-contained font stylesheet', () => { const fixtureDirectory = mkdtempSync(path.join(tmpdir(), 'ssi-visual-parity-font-source-')); const sourceDirectory = path.join(fixtureDirectory, 'fixture'); diff --git a/tools/runtime-package-manifest.test.mjs b/tools/runtime-package-manifest.test.mjs index 4d263693..f2b4e093 100644 --- a/tools/runtime-package-manifest.test.mjs +++ b/tools/runtime-package-manifest.test.mjs @@ -18,6 +18,7 @@ test("website artifact import profile is complete and capability scoped", async assert.ok(profile) assert.deepEqual(profile.abilities, [ "static-site-importer/import-website-artifact", + "static-site-importer/import-url", "static-site-importer/materialize-wordpress-site-plan", "static-site-importer/validate-artifact", "static-site-importer/get-runtime-package-manifest",