lanyue-llk · Nov 5, 2026 · Nov 5, 2026 · Nov 6, 2026 · Nov 6, 2026 · Nov 6, 2026
diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md
@@ -11,6 +11,13 @@ Reports are optional; `AGENTS.md` owns the build and publication authorization r
 
 Read the repository's `AGENTS.md` first. Project rules override this general skill. Before changing a benchmark, read [evaluation design](references/evaluation-design.md). Before reviewing scores or deciding whether to rerun, read [audit and revision](references/audit-and-revision.md). Before building or publishing a report, read [report and release](references/report-and-release.md) and [presentation quality](references/presentation-quality.md).
 
+Before expensive collection, follow the shared [execution contract](../../../docs/benchmark-execution.md).
+Plan dependent stages and shared resources, measure a bounded readiness/throughput
+sample, and freeze concurrency, pacing, call budgets and timing scope. Reuse the
+shared scheduler and registered adapters. Review unexplained global serialization
+or repeated initialization before a full run. New tracks must register an execution
+owner and behavior tests alongside their report template.
+
 Before writing report HTML, resolve the track through `scripts/report_template.py`. Inspect the maintained implementation and its pinned example. Reuse that implementation and its release checks; a different cohort or diagnostic evidence state calls for a data adapter or explicit mode. If no track fits, create the smallest reusable renderer with a synthetic runnable example and behavioral checks, then register it in `configs/report-templates.json`. Improve the owning template and this skill from demonstrated gaps, without embedding run-specific prose or copying old measurements. A report must carry the selected template identity and input hashes so its release check can detect a bypass.
 
 Track-specific report rules live in the guides that `AGENTS.md` lists: Web Fetch cohort, diagnostic, local-audit and oversized-report rules in the [Web Fetch guide](../../../src/harness/web_fetch/README.md#certified-cohorts-and-report-builds), Search relevance, SERP and comparison rules in the [Web Search guide](../../../src/harness/web_search/README.md).

diff --git a/.agents/skills/benchmark-report/references/evaluation-design.md b/.agents/skills/benchmark-report/references/evaluation-design.md
@@ -42,6 +42,11 @@ Ground truth must be independent of the provider under test. Retain source locat
 
 ## Machine-readable current profile
 
+The [shared execution contract](../../../../docs/benchmark-execution.md) owns
+resource admission, parallelism, preflight estimates, startup cost and resumable
+execution for every track. Freeze those decisions before the full matrix. Keep
+quality collection, isolated latency and loaded throughput explicitly scoped.
+
 Keep one current profile that a fresh session can validate before running, scoring, rebuilding, or publishing. Include exact case IDs or their canonical digest, the ordered method/version set, run count, timeout, concurrency policy, output contract, reference state, review requirement, and release target. Fail closed on drift instead of reconstructing intent from old reports.
 
 The [live-URL guide](../../../../src/harness/web_fetch/HEADLESS.md#native-browser-routes) owns native browser route rules.

diff --git a/.agents/skills/benchmark-report/references/report-and-release.md b/.agents/skills/benchmark-report/references/report-and-release.md
@@ -114,6 +114,11 @@ For usage-priced services, convert the declared workload with
 and source dates; validate the displayed interval against that conversion.
 A formula alone does not replace a dollar figure. Label scenario estimates and
 plan quotes separately, and keep full billing terms off the compact home table.
+When a complete workload charge cannot be observed, use the owning renderer's
+explicit unavailable state. Bind the missing population and reason to evidence;
+do not substitute software license cost, a model-only subtotal or zero for the
+complete charge. Web Extract binds its cost disclosure to the frozen source plan
+and validates that receipt at both build and release boundaries.
 
 A result detail leads with the requested fields, reference and actual values,
 and the explanation of their differences. Give developer records one secondary

diff --git a/.env.example b/.env.example
@@ -140,3 +140,19 @@ OXYLABS_API_KEY=
 
 # Comma-separated private hostnames/IPs to omit from extraction report metadata views.
 WEB_EXTRACT_PUBLICATION_HOSTS=
+
+# Self-hosted open-source Web Extract service (caller authentication).
+EXTRACTION_API_KEY=
+# Self-hosted deployment (DeepSeek API key acquisition: https://platform.deepseek.com/api_keys)
+EXTRACTION_MODEL_API_KEY=
+EXTRACTION_MODEL=deepseek-flash
+EXTRACTION_MODEL_BASE=https://api.deepseek.com
+EXTRACTION_MODEL_CONTEXT_TOKENS=1000000
+# Comma-separated exact origins returned by the managed Cloudflare snapshot binding.
+SNAPSHOT_ORIGINS=
+# Separate, URL-safe passwords for the isolated deployment; generate fresh random hex values.
+WEB_EXTRACT_POSTGRES_PASSWORD=
+WEB_EXTRACT_QUEUE_PASSWORD=
+WEB_EXTRACT_PORT=8080
+# Compose supplies CHROME_PATH and the private FIRECRAWL_BACKEND_URL,
+# FIRECRAWL_IDENTITY_URL and FIRECRAWL_IMAGE settings.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -92,6 +92,11 @@ jobs:
           BROWSER_EVAL_REPORT_LANGUAGE=zh uv run --frozen --no-sync python -m harness.web_fetch run --providers fixture --delay 0 --output "$output"
           uv run --frozen --no-sync python -c 'import json, sys; from pathlib import Path; root = Path(sys.argv[1]); contract = json.loads((root / "report-languages.json").read_text()); assert contract["primary_language"] == "zh"; assert contract["languages"] == ["zh"]; assert (root / "en/index.html").is_file()' "$output/report"
       - run: make package-check
+      - name: Verify the documented minimal Web Extract installation
+        run: |
+          extract_env="$(mktemp -d)/venv"
+          UV_PROJECT_ENVIRONMENT="$extract_env" uv sync --frozen --extra web-extract-eval --extra test
+          UV_PROJECT_ENVIRONMENT="$extract_env" uv run --frozen --no-sync python -m pytest tests/web_extract/test_swde.py -q
 
   browser:
     name: Rendered reference and report checks

diff --git a/.gitignore b/.gitignore
@@ -39,3 +39,5 @@ artifacts/browser_use/
 /docs/reports/trajectory-replay-evaluation-*
 .ai/topics/
 .claude/
+/.rainbond/
+/rainbond.app.json
diff --git a/AGENTS.md b/AGENTS.md
@@ -38,6 +38,11 @@ Use `Browser Use` for the autonomous track and `Browser Trajectory Replay` for t
 
 # Execution and evidence
 
+- Follow `docs/benchmark-execution.md` for every existing or new track. Freeze and
+  validate resource admission, concurrency, pacing, initialization and resume
+  behavior before expensive collection. Use the shared execution owner and
+  registered track adapters; global serialization needs a concrete resource or
+  measurement reason. Preserve active frozen cohorts while improving later runs.
 - For benchmark design, execution, scoring, result audit, report generation or release, read `.agents/skills/benchmark-report/SKILL.md`. Use `.agents/skills/benchmark-editorial/SKILL.md` additionally only for a new or substantially rewritten standalone interpretation article, official blog or public-account article.
 - Run repository Python only through `make` targets or `uv run --frozen [--extra ...] python`. `.python-version`, `uv.lock`, the Makefile and CI are the runtime contract; never fall back to system Python.
 - Web, dynamic, popular-site and document text use shared `score_attempt` orchestration and one task-result predicate. Task-specific requirements live in case configuration, not provider- or track-specific scoring branches. Binary acceptance and text diagnostics remain separate; missing reference coverage is unavailable, not zero or perfect.

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,11 @@
+FROM docker.1ms.run/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
+RUN apt-get update && apt-get install -y --no-install-recommends chromium libgeos-c1v5 libspatialindex6 && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir uv==0.7.8
+WORKDIR /app
+COPY deploy/web-extract/pyproject.toml deploy/web-extract/uv.lock ./
+RUN uv sync --frozen --no-dev
+COPY deploy/web-extract/app.py ./
+COPY src/harness ./harness
+ENV PYTHONPATH=/app CHROME_PATH=/usr/lib/chromium/chromium PORT=8080
+EXPOSE 8080
+CMD ["/app/.venv/bin/uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -14,3 +14,4 @@ recursive-include src/harness/web_fetch/templates *.css
 recursive-include src/harness/web_search/assets *.css *.js
 
 recursive-include src/harness/web_extract/assets *.css *.js
+recursive-include src/harness/web_extract *.mjs
diff --git a/Makefile b/Makefile
@@ -71,6 +71,7 @@ require_profile = @test -n "$($(or $(1),PROFILE))" || (echo '$(or $(1),PROFILE)=
   webmainbench-review \
   webmainbench-assemble \
   test-native-adapters \
+  test-extraction-sdk \
   pin-dataset
 setup: ## Install the locked base environment
 	$(UV) sync --frozen
@@ -80,6 +81,10 @@ setup-test: ## Install every extra, Node packages and benchmark datasets
 	npm ci
 	$(TEST_PY) -m harness.benchmarks.data
 
+EXTRACTION_SDK_ENV ?= $(or $(BROWSER_EVAL_DATA_HOME),$(HOME)/.browser-eval)/environments/web-extract-sdk
+test-extraction-sdk: ## Test native extraction protocols with the service's locked SDKs
+	BROWSER_EVAL_TEST_NATIVE_SDKS=1 LITELLM_LOCAL_MODEL_COST_MAP=True PYTHONPATH=src UV_PROJECT_ENVIRONMENT="$(EXTRACTION_SDK_ENV)" $(UV) run --frozen --project deploy/web-extract --python 3.12 python -m unittest discover -s tests/web_extract -p test_native_sdk_protocol.py
+
 help: ## List documented targets
 	@awk 'BEGIN {FS = ":.*## "} /^[a-z][a-z0-9-]*:.*## / {printf "  %-36s %s\n", $$1, $$2}' $(firstword $(MAKEFILE_LIST))
 
@@ -138,7 +143,8 @@ fixture-smoke: runtime-check ## Run the offline fixture CLI
 
 test: test-python test-ui test-native-adapters ## Run Python, UI and native adapter tests
 
-ci-tests: track-naming-check lint-check shell-check test fixture-smoke ## Run the CI test and static check job
+ci-tests: track-naming-check lint-check shell-check fixture-smoke ## Run static checks and independent test suites
+	$(MAKE) -j2 test test-extraction-sdk
 
 ci: ci-tests package-check browser-check ## Run every CI job: tests, packaging and browser checks
 
@@ -151,7 +157,7 @@ browser-check: runtime-check ## Run the CI rendered reference and report checks
 	BROWSER_EVAL_TEST_BROWSER_BINARY="$$($(TEST_PY) -c 'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')" \
 	  $(TEST_PY) -m pytest $(BROWSER_TEST_PATHS) -q
 
-package-check: runtime-check ## Verify the sdist and wheel outside the checkout
+package-check: runtime-check ## Verify sdist and wheel installation
 	$(PY) scripts/check_package.py --uv "$(UV)"
 
 report: runtime-check ## Build the Web Fetch report: PROFILE, EVIDENCE

diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ Choose a workflow for its task contract, setup and runnable example.
 | --- | --- | --- |
 | Web Fetch | [LexBench-Web-Fetch](https://huggingface.co/datasets/Lexmount/LexBench-Web-Fetch), [LexBench-WebMainBench](https://huggingface.co/datasets/Lexmount/LexBench-WebMainBench), [LexBench-WCXB](https://huggingface.co/datasets/Lexmount/LexBench-WCXB) | [Local example](examples/web_fetch/README.md) · [Full guide](src/harness/web_fetch/README.md) |
 | Web Search | [LexBench-Web-Search](https://huggingface.co/datasets/Lexmount/LexBench-Web-Search): scenario queries and [LexBench-SimpleQA-Verified](https://huggingface.co/datasets/Lexmount/LexBench-SimpleQA-Verified) retrieval tasks | [Local example](examples/web_search/README.md) · [Full guide](src/harness/web_search/README.md) |
-| Web Extract | [Product extraction benchmark (Zyte)](https://huggingface.co/datasets/Lexmount/LexBench-Zyte-Product-Extraction): 140 archived product pages with original and source-qualified price, SKU and stock labels; [SWDE](examples/web_extract/README.md#swde-multi-domain-extraction): a predeclared 800-page input subset from 80 sites | [Structured field extraction](examples/web_extract/README.md) |
+| Web Extract | [Product extraction benchmark (Zyte)](https://huggingface.co/datasets/Lexmount/LexBench-Zyte-Product-Extraction): 140 archived product pages with original and source-qualified price, SKU and stock labels; [SWDE-800](examples/web_extract/README.md#swde-multi-domain-extraction): a predeclared 800-page input subset from 80 sites | [Structured field extraction](examples/web_extract/README.md) |
 | Browser Trajectory Replay | [LexBench-Headless-Browser](https://huggingface.co/datasets/Lexmount/LexBench-Headless-Browser) scenarios; [curated fixed-action tasks](https://huggingface.co/datasets/Lexmount/LexBench-Trajectory-Replay) with explicit checks; [WebArena replay](https://huggingface.co/datasets/Lexmount/LexBench-WebArena-Replay) with the official evaluator | [Headless inputs and upstream runner](https://huggingface.co/datasets/Lexmount/LexBench-Headless-Browser#execute-with-the-original-runner) · [Curated suite](examples/trajectory_replay/README.md) · [WebArena replay](examples/trajectory_replay/public_webarena/README.md) |
 
 Use these suites or [bring your own tasks](docs/benchmarks.md). Add browsers and tools through the existing [adapter contracts](docs/provider-contract.md).

diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md
@@ -14,6 +14,8 @@ The root Apache-2.0 license covers project-owned code and documentation. It does
 | Independently captured pages, PDFs, media and reference snapshots | Task provenance, `evaluation/source-inputs.json` and each track's external evidence bindings | Private data storage; import approved copies through manifests. Public accessibility does not imply redistribution permission |
 | Human trajectories and task definitions | Replay configurations, provenance and qualification evidence | Retain origins and usage terms; distinguish definitions, recordings and measured outputs |
 | Repository editorial skills | Origins and notices under `.agents/skills/` | Retain their notices; they are not browser-package runtime dependencies |
+| Self-hosted extraction dependencies | [Crawl4AI](https://github.com/unclecode/crawl4ai) (Apache-2.0), [ScrapeGraphAI](https://github.com/ScrapeGraphAI/Scrapegraph-ai) (MIT), [Stagehand](https://github.com/browserbase/stagehand) (MIT); versions in `deploy/web-extract/uv.lock` | Installed in the optional extraction service; preserve upstream licenses and dependency notices |
+| Firecrawl server | [Firecrawl](https://github.com/firecrawl/firecrawl) (AGPL-3.0); deployments bind an immutable upstream image digest | Runs as a separate service. The facade does not incorporate the server source; retain upstream notices and comply with its license when distributing or modifying it |
 | Synthetic offline examples | `src/harness/web_fetch/data/` | Project-owned test inputs with no third-party captures |
 | Resource-collection overhead calibration | `src/harness/resource_calibration.py`, derived from a [pinned Lexbench source](https://github.com/lexmount/Lexbench-Headless-Browser/blob/fd8d64c7623d46cd165607f216a0e28967d03040/runner/resources.py); source hash recorded in the module | Apache-2.0; retain the [full license](src/harness/licenses/Lexbench-Apache-2.0.txt) and attribution in source and wheel distributions |
 

diff --git a/configs/report-templates.json b/configs/report-templates.json
@@ -30,7 +30,9 @@
       "snapshot_audit_example_check": "uv run --frozen python -m pytest tests/test_published_snapshot_report.py tests/test_published_snapshot_rescoring.py",
       "snapshot_audit_release_check": "uv run --frozen python scripts/check_report.py /absolute/new-report --profile /absolute/profile.json --visual-receipt /absolute/private/visual.json",
       "snapshot_audit_package": "uv run --frozen python -m harness.web_fetch site-package /absolute/new-report /absolute/package.zip --snapshot-audit",
-      "snapshot_audit_publish": "uv run --frozen python -m harness.web_fetch site-publish --create --profile /absolute/publication-target.json --source /absolute/new-report --archive /absolute/package.zip --receipt /absolute/receipt.json --operation-journal /absolute/pending.json --snapshot-audit --visibility private"
+      "snapshot_audit_publish": "uv run --frozen python -m harness.web_fetch site-publish --create --profile /absolute/publication-target.json --source /absolute/new-report --archive /absolute/package.zip --receipt /absolute/receipt.json --operation-journal /absolute/pending.json --snapshot-audit --visibility private",
+      "execution_owner": "src/harness/web_fetch/runner.py",
+      "execution_tests": "tests/test_runner_concurrency.py"
     },
     "trajectory-replay": {
       "renderer": "src/harness/trajectory_replay/webarena_report.py",
@@ -40,7 +42,9 @@
       "tests": "tests/trajectory_replay/test_webarena_report.py",
       "example_env": "TRAJECTORY_REPLAY_REPORT_EXAMPLE_URL",
       "example_version": "ver_c0f0693e9c08401bb4d2b88393234e55",
-      "package": "uv run --frozen python -m harness.trajectory_replay package --report /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/results.json --review /absolute/review.json --output /absolute/package.zip --attestation /absolute/attestation.json --visual-receipt /absolute/visual.json"
+      "package": "uv run --frozen python -m harness.trajectory_replay package --report /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/results.json --review /absolute/review.json --output /absolute/package.zip --attestation /absolute/attestation.json --visual-receipt /absolute/visual.json",
+      "execution_owner": "src/harness/trajectory_replay/webarena_plan.py",
+      "execution_tests": "tests/trajectory_replay"
     },
     "browser-use": {
       "renderer": "src/harness/browser_use/report.py",
@@ -50,7 +54,9 @@
       "tests": "tests/browser_use/test_report.py",
       "package": "uv run --frozen python -m harness.browser_use package --report /absolute/new-report --profile /absolute/instance-profile.json --evidence /absolute/evidence --output /absolute/package.zip --attestation /absolute/attestation.json --visual-receipt /absolute/visual.json",
       "example_env": "BROWSER_USE_REPORT_EXAMPLE_URL",
-      "example_version": "ver_ce4dd70fde2e444ab18d24bba3ee0e93"
+      "example_version": "ver_ce4dd70fde2e444ab18d24bba3ee0e93",
+      "execution_owner": "src/harness/browser_use/runner.py",
+      "execution_tests": "tests/browser_use"
     },
     "web-search": {
       "renderer": "src/harness/web_search/report.py",
@@ -66,7 +72,9 @@
         "synthetic",
         "combined-cohorts"
       ],
-      "release_check": "uv run --frozen python -m harness.web_search report --output /absolute/report --release-check /absolute/visual-receipt.json --publication-plan /absolute/release/publication-plan.json"
+      "release_check": "uv run --frozen python -m harness.web_search report --output /absolute/report --release-check /absolute/visual-receipt.json --publication-plan /absolute/release/publication-plan.json",
+      "execution_owner": "src/harness/web_search/runner.py",
+      "execution_tests": "tests/web_search/test_runner.py"
     },
     "web-extract": {
       "renderer": "src/harness/web_extract/report.py",
@@ -83,12 +91,14 @@
         "synthetic",
         "field-extraction-cohort"
       ],
-      "release_check": "uv run --frozen python -m harness.web_extract.cli check /absolute/report --release --visual-receipt /absolute/visual-receipt.json",
+      "release_check": "uv run --frozen python -m harness.web_extract.cli --home /absolute/data check /absolute/report --release --profile /absolute/report-profile.json --visual-receipt /absolute/visual-receipt.json",
       "package": "uv run --frozen python -m harness.web_extract.cli package /absolute/report --output /absolute/report.zip --visual-receipt /absolute/visual-receipt.json",
       "benchmark": "src/harness/benchmarks/cards/zyte-product-extraction.json",
       "cohort_example_build": "uv run --frozen python -m harness.web_extract.example --output /tmp/extract-cohort-example --mode full-cohort-progress",
       "publish": "uv run --frozen python -m harness.web_extract.cli publish --source /absolute/report --target /absolute/publication-target.json --work /absolute/new-publication --visual-receipt /absolute/visual-receipt.json",
-      "record_example_build": "uv run --frozen python -m harness.web_extract.example --output /tmp/extract-record-example --mode field-extraction-cohort"
+      "record_example_build": "uv run --frozen python -m harness.web_extract.example --output /tmp/extract-record-example --mode field-extraction-cohort",
+      "execution_owner": "src/harness/web_extract/pipeline.py",
+      "execution_tests": "tests/web_extract/test_concurrent_collection.py"
     }
   }
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,3 +14,4 @@ recursive-include src/harness/web_fetch/templates *.css
		recursive-include src/harness/web_search/assets .css .js

		recursive-include src/harness/web_extract/assets .css .js
		recursive-include src/harness/web_extract *.mjs