English edition not generated
This report is available in Chinese. Ask to generate the English edition when needed.
"
@@ -1677,6 +1693,8 @@ def build(output: Path, content: Path, *, home: Path, run=None, language=None, p
from harness.web_extract.dataset import require_product_reference
require_external_output(output)
+ if content.is_file() and read_json(content).get("mode") == "schema-extraction-cohort":
+ return build_schema_cohort(output, content, home=home, language=language, profile=profile)
if content.is_file() and read_json(content).get("mode") == "field-extraction-cohort":
return build_record_cohort(output, content, home=home, language=language, profile=profile)
require_product_reference(home)
@@ -2509,6 +2527,8 @@ def check_reader_contract(output, manifest, *, release=False):
def check(output: Path, *, release=False, visual_receipt=None):
output = open_report(output)
manifest = read_json(output / "report-manifest.json")
+ if manifest.get("mode") == "schema-extraction-cohort":
+ return check_schema_cohort(output, release=release, visual_receipt=visual_receipt)
if manifest.get("mode") == "field-extraction-cohort":
return check_record_cohort(output, release=release, visual_receipt=visual_receipt)
check_report_identity(output, manifest, "web-extract")
@@ -3102,11 +3122,13 @@ def record_result_href(method, case_id):
return record_route("c", case_id) + ".html#" + method
-def record_comparison(headers, rows, metrics, *, sort_values=None):
+def record_comparison(headers, rows, metrics, *, sort_values=None, highlight_ties=True):
"""Use the common table owner and raw-value extrema checks for each metric."""
soup = BeautifulSoup(
table(headers, rows, attrs="data-sortable", sort_values=sort_values), "html.parser"
)
+ if not highlight_ties:
+ soup.table.attrs.pop("data-highlight-ties", None)
for column, (name, direction, values) in metrics.items():
observed = []
for row, value in zip(soup.select("tbody tr"), values, strict=True):
@@ -3118,7 +3140,7 @@ def record_comparison(headers, rows, metrics, *, sort_values=None):
if observed and direction != "neutral":
extreme = (min if direction == "min" else max)(value for value, _ in observed)
for value, cell in observed:
- if value == extreme:
+ if value == extreme and (highlight_ties or sum(v == extreme for v, _ in observed) == 1):
span = soup.new_tag("span", attrs={"class": "comparison-winner"})
span.string = cell.get_text()
cell.clear()
@@ -4388,3 +4410,451 @@ def check_record_cohort(output, *, release=False, visual_receipt=None):
if not required.issubset(observed):
raise ValueError("Incomplete record report detail, theme or viewport coverage")
return manifest
+
+
+def schema_metric_definitions(language, count):
+ labels = {"delivered": "delivery", "exact_record_match": "exact_match"}
+ return [{
+ "id": key, "label": message("schema_" + labels.get(key, key), language),
+ "meaning": message("schema_meaning_" + key, language), "unit": "0–1",
+ "population": message("schema_all_tasks", language, count=count),
+ "direction": message("schema_higher", language),
+ } for key in ("delivered", "key_f1", "value_score", "item_f1", "exact_record_match", "reviewed_correct")]
+
+
+def schema_report_inputs(home, model):
+ """Read complete arbitrary-schema evidence without converting it to product fields."""
+ from harness.web_extract.review import json_leaf_paths
+
+ if model.get("synthetic"):
+ binding = model["synthetic_data"]
+ raw = Path(binding["path"]).read_bytes()
+ if digest(raw) != binding["sha256"]:
+ raise ValueError("Synthetic schema fixture changed")
+ data = json.loads(raw)
+ return data["cases"], data["rows"], data["scores"], data["audit"]
+ validate_dataset(home)
+ truth = read_json(home / "dataset/dataset/ground-truth.json")
+ binding = model["audit"]
+ audit = read_json(Path(binding["path"]))
+ if digest(Path(binding["path"]).read_bytes()) != binding["sha256"]:
+ raise ValueError("Schema report audit changed")
+ if audit.get("review_scope") != "all" or audit.get("accepted_unreviewed_passes") != 0:
+ raise ValueError("Schema report requires full source-evidence review")
+ if audit.get("dataset_manifest_sha256") != digest((home / "dataset/manifest.json").read_bytes()):
+ raise ValueError("Schema audit belongs to another dataset")
+ methods, rows, scores = list(model["participants"]), [], {}
+ for selection in model["evidence_runs"]:
+ run = Path(selection["path"])
+ contract, selected = load_run(run)
+ if len(contract["methods"]) != 1:
+ raise ValueError("Schema report needs one source run per method")
+ method = contract["methods"][0]
+ certificate = read_json(run / "certificate.json")
+ if certificate["manifest_sha256"] != digest((run / "manifest.json").read_bytes()):
+ raise ValueError("Schema report source certificate changed")
+ score_path = Path(selection["scores"]["path"])
+ if digest(score_path.read_bytes()) != selection["scores"]["sha256"]:
+ raise ValueError("Schema report selected score changed")
+ scored = read_json(score_path)
+ if scored["call_manifest_sha256"] != certificate["manifest_sha256"]:
+ raise ValueError("Schema score belongs to another call matrix")
+ if method in scores:
+ raise ValueError("Schema report duplicates a method")
+ scores[method] = scored["scores"][method]
+ rows.extend(selected)
+ expected = {(method, cid) for method in methods for cid in truth}
+ observed = {(row["method"], row["case_id"]) for row in rows}
+ if len(rows) != len(expected) or observed != expected or set(scores) != set(methods):
+ raise ValueError("Schema report requires the complete declared result matrix")
+ if set(audit["results"]) != {m + ":" + cid for m, cid in expected}:
+ raise ValueError("Schema report audit omits or adds results")
+ for row in rows:
+ opinion = audit["results"][row["method"] + ":" + row["case_id"]]
+ if opinion["execution_identity"] != execution_identity(row):
+ raise ValueError("Schema report review does not match the current native response")
+ actual = row.get("normalized_fields")
+ paths = json_leaf_paths(actual) if isinstance(actual, dict) else []
+ if sorted(opinion["checked_paths"]) != sorted(paths):
+ raise ValueError("Schema report review skipped nested values")
+ if set(audit["reference_reviews"]) != set(truth):
+ raise ValueError("Schema report reference audit is incomplete")
+ return truth, rows, scores, audit
+
+
+def build_schema_cohort(output, content, *, home, language, profile):
+ """Render arbitrary JSON cohorts with the existing shell, tables and evidence UI."""
+ from collections import Counter
+
+ from harness.reporting.presentation import metric_guide
+
+ require_external_output(output)
+ model, authority = read_json(content), read_json(profile)
+ if authority.get("research_report", {}).get("content", {}).get("sha256") != digest(
+ content.read_bytes()
+ ):
+ raise ValueError("Schema report content is not bound to its profile")
+ if authority.get("independent_result_review_scope") != "all":
+ raise ValueError("Schema report requires explicit full-review scope")
+ language = resolve_report_language(output, language)
+ languages = ("zh",) if language == "zh" else ("en", "zh")
+ if output.exists() and any(output.iterdir()):
+ raise ValueError("Use an empty report directory")
+ truth, rows, scores, audit = schema_report_inputs(home, model)
+ methods = list(model["participants"])
+ labels = {m: model["participants"][m]["label"] for m in methods}
+ output.mkdir(parents=True, exist_ok=True)
+ (output / "public-evidence").mkdir()
+ (output / "data").mkdir()
+ (output / "style.css").write_text(CSS, encoding="utf-8")
+ (output / "app.js").write_text(JS, encoding="utf-8")
+ by_case = {cid: [r for r in rows if r["case_id"] == cid] for cid in truth}
+ copy_report_assets(output, content, model)
+
+ def dump(name, value):
+ save_json(output / name, value)
+
+ for cid, definition in truth.items():
+ markdown = (home / "dataset/markdown" / (cid + ".md")).read_text(encoding="utf-8")
+ dump("public-evidence/t-" + cid + ".json", {
+ "case_id": cid, "schema": json.loads(definition["schema_text"]),
+ "reference": definition["response"], "markdown": markdown,
+ })
+ (output / "public-evidence" / ("source-" + cid + ".html")).write_text(
+ '' + esc(cid)
+ + "
" + esc(markdown) + "
", encoding="utf-8",
+ )
+ for row in by_case[cid]:
+ method = row["method"]
+ opinion = audit["results"][method + ":" + cid]
+ dump("public-evidence/r-" + method + "-" + cid + ".json", {
+ "kind": "schema-json-result", "case_id": cid, "method": labels[method],
+ "review": opinion, "request": row.get("request"),
+ "response": row.get("response"), "runtime_error": row.get("error"),
+ "normalized_fields": row.get("normalized_fields"),
+ "scores": scores[method]["per_case"][cid],
+ })
+ dump("data/metrics.json", {m: scores[m] for m in methods})
+ dump("data/review.json", {k: audit[k] for k in (
+ "review_kind", "reviewer", "reviewed_results", "case_count", "cause_distribution"
+ )})
+ for attachment in model.get("attachments", []):
+ path = Path(attachment["path"])
+ name = attachment["name"]
+ if not re.fullmatch(r"[a-z0-9-]+\.json", name) or path.is_symlink():
+ raise ValueError("Invalid schema report attachment")
+ if digest(path.read_bytes()) != attachment["sha256"]:
+ raise ValueError("Schema report attachment changed")
+ dump("public-evidence/" + name, read_json(path))
+
+ for lang in languages:
+ (output / lang).mkdir()
+ def t(key, _lang=lang, **kw):
+ return message("schema_" + key, _lang, **kw)
+ title = model["title"][lang]
+ notice = t("synthetic_notice") if model.get("synthetic") else t("audit_notice")
+
+ def prose(route, _lang=lang):
+ blocks = model.get("authored", {}).get(_lang, {}).get(route, [])
+ body = "".join(
+ "" + esc(b["heading"]) + "
" + esc(b["text"])
+ + "
" + (
+ '
'
+ return body
+
+ def evidence_button(src, *, label=None, _t=t, target="schema-evidence", key=None):
+ return ('")
+
+ def evidence_dialog(target="schema-evidence", _t=t):
+ return ('")
+
+ dialog = evidence_dialog()
+ quality = []
+ metrics = {i: (key, "max", []) for i, key in enumerate(
+ ("delivered", "key_f1", "value_score", "item_f1", "exact_record_match", "reviewed_correct"), 1
+ )}
+ for method in methods:
+ mean = scores[method]["mean"]
+ correct = sum(audit["results"][method + ":" + cid]["verdict"] == "correct" for cid in truth)
+ values = [mean["delivered"], mean["key_f1"], mean["value_score"], mean["item_f1"],
+ mean["exact_record_match"], correct / len(truth)]
+ quality.append([esc(labels[method]), f'{scores[method]["delivery_count"]}/{len(truth)}',
+ f'{mean["key_f1"]:.4f}', f'{mean["value_score"]:.4f}', f'{mean["item_f1"]:.4f}',
+ f'{sum(v["exact_record_match"] for v in scores[method]["per_case"].values())}/{len(truth)}',
+ f'{correct}/{len(truth)}'])
+ for i, value in enumerate(values, 1):
+ metrics[i][2].append(value)
+ core = record_comparison([t(k) for k in (
+ "method", "delivery", "key_f1", "value_score", "item_f1", "exact_match", "reviewed_correct"
+ )], quality, metrics, highlight_ties=False)
+ core += '" + esc(b["heading"]) + "
" + esc(b["text"]) + + "
" + ( + '' + t("all_tasks", count=len(truth)) + "
" + core += "" + t("whole_result_note") + "
" + core += "" + t("review_kind") + "
" + core += metric_guide(schema_metric_definitions(lang, len(truth)), + title=t("metric_guide"), headings=t("metric_headers")) + timing = operating_metrics(methods, rows, model["prices"], lang, labels, + quantile="linear", native_pricing=True) + bodies = {route: prose(route) for route in PAGES} + bodies["index"] = "" + t("core") + "
" + core + timing + bodies["index"] + bodies["accounts"] += price_table(methods, model["prices"], lang, labels, native_pricing=True) + bodies["accounts"] += usage_table(methods, rows, lang, labels) + bodies["review"] = ("" + t("reviewed", done=audit["reviewed_results"], total=len(rows)) + + "
" + bodies["review"]) + bodies["interpretation"] = core + bodies["interpretation"] + successful_timing, conditional_metrics = [], {i: ("conditional-" + key, "min", []) + for i, key in enumerate(("mean", "p50", "p95"), 2)} + for method in methods: + values = sorted(r["elapsed_seconds"] for r in rows if r["method"] == method + and isinstance(r.get("normalized_fields"), dict)) + statistics_values = ([statistics.mean(values), statistics.median(values), + latency_p95(values, "linear")] if values else [None] * 3) + successful_timing.append([esc(labels[method]), str(len(values)), *[ + f"{v:.3f}" if v is not None else "—" for v in statistics_values + ]]) + for column, value in enumerate(statistics_values, 2): + conditional_metrics[column][2].append(value) + bodies["protocol"] += "" + t("conditional_latency") + "
" + bodies["protocol"] += t("conditional_latency_scope") + "
" + record_comparison( + t("conditional_latency_headers"), successful_timing, conditional_metrics, + highlight_ties=False, + ) + inventory = [] + for row in rows: + method, cid = row["method"], row["case_id"] + opinion = audit["results"][method + ":" + cid] + metric = scores[method]["per_case"][cid] + inventory.append([esc(cid), esc(labels[method]), t("verdict_" + opinion["verdict"]), + t("cause_" + opinion["cause"]), f'{metric["value_score"]:.4f}', + '' + t("comparison") + " " + + evidence_button("public-evidence/r-" + method + "-" + cid + ".json")]) + inventory_html = BeautifulSoup(table( + [t(k) for k in ("case", "method", "verdict", "cause", "value_score", "detail")], + inventory, inventory=True, attrs="data-sortable", + ), "html.parser") + for tr, row in zip(inventory_html.select("tbody tr"), rows, strict=True): + opinion = audit["results"][row["method"] + ":" + row["case_id"]] + tr.attrs.update({"data-result": "", "data-case": row["case_id"], + "data-method": row["method"], "data-status": "pass" if opinion["verdict"] == "correct" else "fail", + "data-stage": failure_stage(row), + "data-search": " ".join((row["case_id"], labels[row["method"]], opinion["cause"])), + "data-scored-fields": "1"}) + controls = BeautifulSoup(result_controls( + methods, [{**r, "failure_stage": failure_stage(r)} for r in rows], labels, lang + ), "html.parser") + # Arbitrary JSON has no shared price/SKU/stock field inventory. + field_control = controls.select_one('[data-result-filter="field"]') + field_control.find_parent("label").decompose() + for value in ("pass", "fail"): + controls.select_one('[data-result-filter="status"] option[value="' + value + '"]').string = ( + t("review_pass" if value == "pass" else "review_fail") + ) + bodies["results"] += str(controls) + str(inventory_html) + dialog + dataset_rows = [] + for cid, definition in truth.items(): + source = (home / "dataset/markdown" / (cid + ".md")).read_text(encoding="utf-8") + schema = json.loads(definition["schema_text"]) + dataset_rows.append([esc(cid), str(len(source)), str(len(schema.get("properties", {}))), + '' + t("source") + ""]) + task = "" + t("input_scope") + "
" + task += '' + t("source_preview") + "" + for key, value in (("schema", schema), ("reference", definition["response"]), ("source", source)): + task += "" + t(key) + "
" + esc( + value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2) + ) + "
' + esc(error) + "" if error else + '
' + t("no_runtime_error") + "
") + native = row.get("normalized_fields") + native = native if isinstance(native, dict) else {} + expected_values = definition["response"] + + def preview(value): + text = json.dumps(value, ensure_ascii=False) + return "" + esc(text[:280] + ("…" if len(text) > 280 else "")) + ""
+
+ fields = []
+ for field in opinion["fields"]:
+ key = field["field"]
+ fields.append([esc(key), preview(expected_values[key]) if key in expected_values else t("absent"),
+ preview(native[key]) if key in native else t("absent"), t("field_" + field["status"])])
+ values_table = table([t(k) for k in ("field", "reference", "actual", "verdict")], fields)
+ case_body += ('' + + esc(labels[method]) + " · " + t("verdict_" + opinion["verdict"]) + "
" + + '' + t("cause_" + opinion["cause"]) + "
" + + values_table + + evidence_button("public-evidence/t-" + cid + ".json", label=t("reference"), + target="reference-" + method, key="reference") + + evidence_button("public-evidence/r-" + method + "-" + cid + ".json", + label=t("actual"), target="actual-" + method, key="normalized_fields") + + evidence_dialog("reference-" + method) + evidence_dialog("actual-" + method) + + "" + t("source_support") + "
" + + esc(json.dumps(opinion["fields"], ensure_ascii=False, indent=2)) + "
" + esc(labels[method]) + "
" + bodies["interpretation"] += table([t("cause"), t("case_count")], [ + [t("cause_" + cause), str(count)] for cause, count in sorted(distribution.items()) + ], attrs='data-schema-causes style="min-width:0"') + "' + t("all_tasks", count=len(truth)) + "
" core += "" + t("whole_result_note") + "
" core += "" + t("review_kind") + "
" @@ -4624,6 +4650,7 @@ def evidence_dialog(target="schema-evidence", _t=t): for i, key in enumerate(("mean", "p50", "p95"), 2)} for method in methods: values = sorted(r["elapsed_seconds"] for r in rows if r["method"] == method + and r.get("http_status") == 200 and isinstance(r.get("normalized_fields"), dict)) statistics_values = ([statistics.mean(values), statistics.median(values), latency_p95(values, "linear")] if values else [None] * 3) @@ -4635,7 +4662,6 @@ def evidence_dialog(target="schema-evidence", _t=t): bodies["protocol"] += "" + t("conditional_latency") + "
" bodies["protocol"] += t("conditional_latency_scope") + "
" + record_comparison( t("conditional_latency_headers"), successful_timing, conditional_metrics, - highlight_ties=False, ) inventory = [] for row in rows: diff --git a/src/harness/web_extract/review.py b/src/harness/web_extract/review.py index ad1bfbc51..845bc688a 100644 --- a/src/harness/web_extract/review.py +++ b/src/harness/web_extract/review.py @@ -68,6 +68,30 @@ def schema_audit_schema(): "properties": {"cases": {"type": "array", "items": case}}, "required": ["cases"]} +def schema_audit_case(home, cid, case, rows): + """One source/schema/native contract for reviewer packets and report ingestion.""" + schema = json.loads(case["schema_text"]) + requested = set(schema.get("properties", {})) | set(case["response"]) + outputs = [] + for row in rows: + native = row.get("normalized_fields") + outputs.append({ + "method": row["method"], "native_response": row.get("response"), + "normalized_fields": native, "http_status": row.get("http_status"), + "runtime_error": row.get("error"), "execution_identity": execution_identity(row), + "leaf_paths": json_leaf_paths(native) if isinstance(native, dict) else [], + "field_inventory": sorted( + requested | (set(native) if isinstance(native, dict) else set()) + ), + }) + return {"case_id": cid, "audit_kind": "schema-json-audit/1", + "source_outline": str(Path(home) / "dataset/markdown" / (cid + ".md")), + "source_html": str(Path(home) / "dataset/html" / (cid + ".html")), + "field_schema": schema, "reference": case["response"], + "reference_leaf_paths": json_leaf_paths(case["response"]), + "reference_field_inventory": sorted(requested), "outputs": outputs} + + def prepare_schema_audit(home, content, work): """Freeze arbitrary JSON tasks with explicit complete leaf and root-field inventories.""" from harness.web_extract.pipeline import load_run, validate_dataset @@ -83,24 +107,18 @@ def prepare_schema_audit(home, content, work): for selection in model["evidence_runs"]: run = Path(selection["path"]) contract, rows = load_run(run) + require_schema_dataset(home, contract) for name in ("manifest.json", "contract.json", "certificate.json"): bindings[str(run / name)] = digest((run / name).read_bytes()) for row in rows: - native = row.get("normalized_fields") - outputs[row["case_id"]].append({ - "method": row["method"], "native_response": row.get("response"), - "normalized_fields": native, "http_status": row.get("http_status"), - "runtime_error": row.get("error"), "execution_identity": execution_identity(row), - "leaf_paths": json_leaf_paths(native) if isinstance(native, dict) else [], - "field_inventory": sorted(set(json.loads(truth[row["case_id"]]["schema_text"]) - .get("properties", {})) | set(truth[row["case_id"]]["response"]) - | (set(native) if isinstance(native, dict) else set())), - }) + outputs[row["case_id"]].append(row) methods = sorted(o["method"] for o in next(iter(outputs.values()))) if len(methods) != len(set(methods)) or any( sorted(o["method"] for o in items) != methods for items in outputs.values() ): raise ValueError("Audit requires each original result exactly once") + if not model.get("participants") or set(methods) != set(model["participants"]): + raise ValueError("Schema audit omits a declared participant") work.mkdir(parents=True, exist_ok=True) cases, packets = [], [] for cid, case in sorted(truth.items()): @@ -108,13 +126,7 @@ def prepare_schema_audit(home, content, work): source_html = home / "dataset/html" / (cid + ".html") for path in (markdown, source_html): bindings[str(path)] = digest(path.read_bytes()) - schema = json.loads(case["schema_text"]) - item = {"case_id": cid, "audit_kind": "schema-json-audit/1", - "source_outline": str(markdown), "source_html": str(source_html), - "field_schema": schema, "reference": case["response"], - "reference_leaf_paths": json_leaf_paths(case["response"]), - "reference_field_inventory": sorted(set(schema.get("properties", {})) - | set(case["response"])), "outputs": outputs[cid]} + item = schema_audit_case(home, cid, case, outputs[cid]) path = work / "packets" / (cid + ".json") save_packet(path, {"cases": [item]}, home) packets.append({"path": str(path), "sha256": digest(path.read_bytes())}) @@ -133,6 +145,13 @@ def prepare_schema_audit(home, content, work): return manifest +def require_schema_dataset(home, contract): + if contract.get("dataset_manifest_sha256") != digest( + (Path(home) / "dataset/manifest.json").read_bytes() + ): + raise ValueError("Schema execution belongs to a different dataset") + + def validate_schema_audit(result, packet): from jsonschema import Draft202012Validator @@ -156,6 +175,16 @@ def validate_schema_audit(result, packet): for o in case["outputs"]), ]: evidence_sources = [source] + if "method" in review and review["verdict"] == "correct": + native = methods[review["method"]] + if native.get("http_status") != 200 or not isinstance( + native.get("normalized_fields"), dict + ): + raise ValueError("Absent or failed output cannot be reviewed correct") + if any(field["status"] in {"incorrect", "partial", "missing", "unsupported"} + or any(f["kind"] == "extraction_error" for f in field["findings"]) + for field in review["fields"]): + raise ValueError("Correct verdict contradicts field assessments") if "method" in review and methods[review["method"]].get("normalized_fields") is None: def strings(value): if isinstance(value, str): @@ -574,6 +603,22 @@ def literal_citation_span(source, fragments, *, limit=4096): def reconcile_citations(result, packet): """Retain raw opinions and select literal excerpts without changing judgments.""" if any(c.get("audit_kind") == "schema-json-audit/1" for c in packet["cases"]): + if not isinstance(result, dict) or not isinstance(result.get("cases"), list): + raise ValueError("Invalid schema audit containers") + for case in result["cases"]: + if (not isinstance(case, dict) or not isinstance(case.get("reference"), dict) + or not isinstance(case.get("outputs"), list)): + raise ValueError("Invalid schema audit containers") + for group in [case["reference"], *case["outputs"]]: + if not isinstance(group, dict) or not isinstance(group.get("fields"), list): + raise ValueError("Invalid schema audit field containers") + for field in group["fields"]: + if (not isinstance(field, dict) + or ("findings" in field and ( + not isinstance(field["findings"], list) + or any(not isinstance(f, dict) for f in field["findings"]) + ))): + raise ValueError("Invalid schema audit findings") result = copy.deepcopy(result) for case in result.get("cases", []): reference = case.get("reference", {}) @@ -1438,6 +1483,7 @@ def assemble_schema_audit(home, content, work, output, *, profile): for selection in model["evidence_runs"]: run = Path(selection["path"]) contract, rows = load_run(run) + require_schema_dataset(home, contract) if len(contract["methods"]) != 1 or len(rows) != len(truth): raise ValueError("Schema audit requires complete single-method source runs") method = contract["methods"][0] @@ -1457,9 +1503,9 @@ def assemble_schema_audit(home, content, work, output, *, profile): opinion = next(o for o in opinions[cid]["outputs"] if o["method"] == method) kinds = {finding["kind"] for field in opinion["fields"] for finding in field["findings"]} - if not isinstance(row.get("normalized_fields"), dict): + if row.get("http_status") != 200 or not isinstance(row.get("normalized_fields"), dict): cause = "delivery_failure" - elif opinion["verdict"] in {"partial", "incorrect"}: + elif opinion["verdict"] in {"partial", "incorrect", "no_output"}: cause = "extraction_error" elif opinion["verdict"] == "ambiguous": cause = "source_ambiguous" @@ -1475,6 +1521,8 @@ def assemble_schema_audit(home, content, work, output, *, profile): "cause": cause, "finding_kinds": sorted(kinds), } distribution[method] = dict(counts) + if not model.get("participants") or set(methods) != set(model["participants"]): + raise ValueError("Schema audit omits a declared participant") if set(identities) != {(m, cid) for m in methods for cid in truth}: raise ValueError("Schema audit coverage is incomplete or contains extra results") value = { @@ -1510,6 +1558,8 @@ def run(work, *, model, workers=4, limit=None, backend="codex", credential_env_f for name in ("logs", "raw", "reviews", "reviewer.json", "schema.json"): require_external_output(work / name) manifest = read_manifest(work) + if backend == "litellm" and manifest["schema"] != "schema-json-audit/1": + raise ValueError("Gateway review requires the complete frozen Markdown schema contract") for item in manifest["packets"]: name = Path(item["path"]).name for target in ( diff --git a/tests/web_extract/test_human_eval.py b/tests/web_extract/test_human_eval.py index 2d3cd97a3..8d9288f86 100644 --- a/tests/web_extract/test_human_eval.py +++ b/tests/web_extract/test_human_eval.py @@ -37,6 +37,8 @@ def dataset(tmp_path, monkeypatch): ' return result\n' 'def _compare_values(pred,gold):\n' ' if isinstance(gold,list): return float(sorted(pred,key=str)==sorted(gold,key=str))\n' + ' if isinstance(gold,str) and gold: return len(set(str(pred).split()) & ' + 'set(gold.split()))/len(set(gold.split()))\n' ' return float(pred==gold)\n' 'def magic_metric_failed(): return {"key_f1":0.,"value_score":0.}\n' 'def run_all(pred,schema,gold):\n' @@ -230,3 +232,79 @@ def test_source_reference_revision_preserves_original_and_rejects_unbound_edits( p.save_json(revision, candidate) with pytest.raises(ValueError): he.load_reference_revision(home, revision) + + +@pytest.mark.parametrize('defect', [ + 'missing_case', 'empty_cases', 'empty_mean', 'missing_leaf_metric', + 'missing_mean_metric', 'population', 'delivery', 'extra_case', +]) +def test_predecessor_scores_reject_incomplete_or_changed_evidence(dataset, defect): + import copy + + home, _, _ = dataset + truth = p.read_json(home / 'dataset/dataset/ground-truth.json') + rows = [{'method': 'cloudflare-json', 'case_id': cid, 'http_status': 200, + 'normalized_fields': case['response']} for cid, case in truth.items()] + current = p.score_rows(home, truth, rows, ['cloudflare-json']) + saved = copy.deepcopy(current) + for group in saved.values(): + for scores in [group['mean'], *group['per_case'].values()]: + for key in ('item_precision', 'item_recall', 'item_f1'): + scores.pop(key) + assert he.verify_predecessor_scores(saved, current, list(truth)) + group = saved['cloudflare-json'] + if defect == 'missing_case': + group['per_case'].pop(next(iter(truth))) + elif defect == 'empty_cases': + group['per_case'] = {} + elif defect == 'empty_mean': + group['mean'] = {} + elif defect == 'missing_leaf_metric': + group['per_case'][next(iter(truth))].pop('value_score') + elif defect == 'missing_mean_metric': + group['mean'].pop('value_score') + elif defect == 'population': + group['population'] += 1 + elif defect == 'delivery': + group['delivery_count'] -= 1 + else: + group['per_case']['not-declared'] = copy.deepcopy(next(iter(group['per_case'].values()))) + assert not he.verify_predecessor_scores(saved, current, list(truth)) + + +def test_nested_array_numeric_serializers_preserve_order_and_multiplicity(): + gold = {'items': [{'values': [100000000000000000000, 11]}]} + assert he.item_scores(gold, {'items': [{'values': [11, 1e20]}]})['item_f1'] == 1 + assert he.item_scores({'items': [{'values': [1, 1, 2]}]}, + {'items': [{'values': [1, 2, 2]}]})['item_f1'] == 0 + + +def test_reference_rescoring_rejects_a_corrupt_predecessor_envelope(dataset, monkeypatch, tmp_path): + home, _, _ = dataset + truth = p.read_json(home / 'dataset/dataset/ground-truth.json') + rows = [{'method': 'cloudflare-json', 'case_id': cid, 'http_status': 200, + 'normalized_fields': case['response']} for cid, case in truth.items()] + contract = {'kind': 'full_archived_extraction', 'case_ids': list(truth), + 'methods': ['cloudflare-json'], + 'dataset_manifest_sha256': p.digest((home / 'dataset/manifest.json').read_bytes())} + run = tmp_path / 'run' + p.save_json(run / 'manifest.json', {'synthetic_calls': list(truth)}) + monkeypatch.setattr(p, 'load_run', lambda path: (contract, rows)) + p.score_run(home, run) + original = home / 'dataset/dataset/ground-truth.json' + source = home / 'dataset/markdown/human-eval-000.md' + authority = tmp_path / 'revision.json' + p.save_json(authority, {'schema': 'source-json-reference-revision/1', + 'original_reference_sha256': p.digest(original.read_bytes()), + 'root_cause': 'Keep literal heading punctuation.', + 'regression_ids': he.check_scoring_controls(home), 'changes': [{ + 'case_id': 'human-eval-000', 'path': '/name', 'before': 'A & B', 'after': '# A & B', + 'source_sha256': p.digest(source.read_bytes()), 'source_quote': '# A & B', + 'reason': 'The literal source heading includes the marker.'}]}) + p.score_run(home, run, reference_revision=authority, output=run / 'revised.json') + predecessor = p.read_json(run / 'scores.json') + predecessor['cases'] = 0 + p.save_json(run / 'scores.json', predecessor) + with pytest.raises(ValueError, match='Original scores do not reproduce'): + p.score_run(home, run, reference_revision=authority, output=run / 'invalid.json') + assert not (run / 'invalid.json').exists() diff --git a/tests/web_extract/test_schema_audit.py b/tests/web_extract/test_schema_audit.py index 1674ef1b0..5d837fccc 100644 --- a/tests/web_extract/test_schema_audit.py +++ b/tests/web_extract/test_schema_audit.py @@ -24,7 +24,7 @@ def audit(tmp_path): "reference": reference, "reference_leaf_paths": json_leaf_paths(reference), "reference_field_inventory": ["items"], "outputs": [{ "method": "native", "leaf_paths": json_leaf_paths(native), - "field_inventory": ["items"], + "field_inventory": ["items"], "normalized_fields": native, "http_status": 200, }], } group = {"field": "items", "status": "supported", "source_quote": "Alice: red mug, 12 EUR.", @@ -113,3 +113,82 @@ def test_runtime_error_is_evidence_only_for_an_absent_output(audit): native['normalized_fields'] = {} with pytest.raises(ValueError, match='literal'): validate(result, packet) + + +@pytest.mark.parametrize('defect', ['absent', 'http_error', 'incorrect_field', 'error_finding']) +def test_correct_verdict_cannot_contradict_native_evidence(audit, defect): + packet, result = audit + output = result['cases'][0]['outputs'][0] + if defect == 'absent': + packet['cases'][0]['outputs'][0]['normalized_fields'] = None + elif defect == 'http_error': + packet['cases'][0]['outputs'][0]['http_status'] = 400 + elif defect == 'incorrect_field': + output['fields'][0]['status'] = 'incorrect' + else: + output['fields'][0]['findings'] = [{ + 'path': '/items', 'kind': 'extraction_error', + 'reason': 'A requested record is missing.', + 'source_quote': 'Bob: blue mug, 15 EUR.', 'proposed_json': '', + }] + with pytest.raises(ValueError): + validate(result, packet) + + +@pytest.mark.parametrize('defect', ['outputs', 'reference', 'fields', 'findings']) +def test_malformed_cached_model_containers_are_retryable_value_errors(audit, defect): + from harness.web_extract.review import reconcile_citations + + packet, result = audit + case = result['cases'][0] + if defect in {'outputs', 'reference'}: + case.pop(defect) + elif defect == 'fields': + case['outputs'][0]['fields'] = None + else: + case['outputs'][0]['fields'][0]['findings'] = [None] + with pytest.raises(ValueError): + reconcile_citations(result, packet) + + +def test_resume_quarantines_a_malformed_raw_reply_and_retries_once(tmp_path, monkeypatch, audit): + from harness.web_extract import review + from harness.web_extract.pipeline import digest, read_json + + packet, valid = audit + work = tmp_path / 'work' + path = work / 'packets/one.json' + review.save_packet(path, packet, tmp_path) + review.save_manifest(work, {'schema': 'schema-json-audit/1', 'case_count': 1, + 'result_count': 1, 'bindings': {}, 'packets': [{'path': str(path), + 'sha256': digest(path.read_bytes())}]}, tmp_path) + save_json(work / 'schema.json', review.review_schema(packet['cases'])) + malformed = copy.deepcopy(valid) + malformed['cases'][0].pop('outputs') + replies = iter([malformed, valid]) + calls = [] + + def chat(**kwargs): + calls.append(kwargs) + return copy.deepcopy(next(replies)), {'requested_model': 'one-model'} + + monkeypatch.setattr('harness.review_runner.run_chat_reviewer', chat) + credentials = tmp_path / 'gateway.env' + credentials.write_text('OPENAI_BASE_URL=https://example.org/v1\nOPENAI_API_KEY=test-only\n') + options = {'model': 'one-model', 'workers': 1, 'backend': 'litellm', + 'credential_env_file': credentials} + assert len(review.run(work, **options)['needs_correction']) == 1 + assert review.run(work, **options)['packets'] == 1 + assert len(list((work / 'invalid').glob('*.json'))) == 1 + assert read_json(work / 'reviews/one.json') == valid + assert review.run(work, **options)['packets'] == 1 + assert len(calls) == 2 + + +def test_gateway_rejects_an_html_audit_without_a_complete_inline_source_contract(tmp_path): + from harness.web_extract.review import run + + work = tmp_path / 'html-review' + save_json(work / 'manifest.json', {'schema': 'product-audit/1', 'packets': [], 'bindings': {}}) + with pytest.raises(ValueError, match='complete frozen Markdown'): + run(work, model='one-model', backend='litellm') diff --git a/tests/web_extract/test_schema_boundaries.py b/tests/web_extract/test_schema_boundaries.py new file mode 100644 index 000000000..a233960e8 --- /dev/null +++ b/tests/web_extract/test_schema_boundaries.py @@ -0,0 +1,130 @@ +"""Full reviews and official comparisons reject mixed or incomplete evidence.""" + +import json +from pathlib import Path + +import pytest + +from harness.web_extract import pipeline as p +from harness.web_extract import report, review + + +@pytest.fixture +def evidence(tmp_path, monkeypatch): + home, run, work = tmp_path / 'home', tmp_path / 'run', tmp_path / 'review' + source = home / 'dataset/markdown/one.md' + source.parent.mkdir(parents=True) + source.write_text('Amber mug: 12 EUR.') + (home / 'dataset/html').mkdir() + (home / 'dataset/html/one.html').write_text('Amber mug: 12 EUR.') + schema = {'type': 'object', 'properties': {'name': {'type': 'string'}}} + native = {'name': 'Amber mug'} + p.save_json(home / 'dataset/manifest.json', {'benchmark': 'human-eval'}) + manifest_sha = p.digest((home / 'dataset/manifest.json').read_bytes()) + p.save_json(home / 'dataset/dataset/ground-truth.json', { + 'one': {'response': native, 'schema_text': json.dumps(schema)}, + }) + (home / 'dataset/evaluate.py').write_text('# A synthetic external scorer dependency.\n') + row = {'method': 'native', 'case_id': 'one', 'normalized_fields': native, + 'http_status': 200, 'response': {'json': native}} + contract = {'methods': ['native'], 'case_ids': ['one'], + 'dataset_manifest_sha256': manifest_sha} + p.save_json(run / 'manifest.json', {'synthetic_calls': ['one']}) + p.save_json(run / 'contract.json', contract) + call_sha = p.digest((run / 'manifest.json').read_bytes()) + p.save_json(run / 'certificate.json', {'manifest_sha256': call_sha}) + score = {'population': 1, 'delivery_count': 1, 'mean': {}, 'per_case': {'one': {}}} + p.save_json(run / 'scores.json', {'scores': {'native': score}, + 'call_manifest_sha256': call_sha, + 'scorer_sha256': p.digest((home / 'dataset/evaluate.py').read_bytes())}) + field = {'field': 'name', 'status': 'supported', 'source_quote': 'Amber mug', + 'reason': 'The source names the mug.', 'findings': []} + opinion = {'method': 'native', 'verdict': 'correct', 'summary': 'Correct mug.', + 'fields': [field], 'checked_paths': ['/name']} + reference = {'checked_paths': ['/name'], 'fields': [field], 'summary': 'Supported mug.'} + audit = {'review_scope': 'all', 'accepted_unreviewed_passes': 0, + 'dataset_manifest_sha256': manifest_sha, + 'results': {'native:one': {**opinion, + 'execution_identity': p.execution_identity(row)}}, + 'reference_reviews': {'one': reference}} + p.save_json(tmp_path / 'audit.json', audit) + model = {'participants': {'native': {'label': 'Native'}}, + 'audit': {'path': str(tmp_path / 'audit.json'), + 'sha256': p.digest((tmp_path / 'audit.json').read_bytes())}, + 'evidence_runs': [{'path': str(run), 'scores': {'path': str(run / 'scores.json'), + 'sha256': p.digest((run / 'scores.json').read_bytes())}}]} + monkeypatch.setattr(p, 'validate_dataset', lambda *a, **kw: None) + monkeypatch.setattr(report, 'validate_dataset', lambda *a, **kw: None) + monkeypatch.setattr(p, 'load_run', lambda path: (contract, [row])) + monkeypatch.setattr(report, 'load_run', lambda path: (contract, [row])) + monkeypatch.setattr(report, 'score_rows', lambda *a, **kw: {'native': score}) + monkeypatch.setattr(review, 'completed', lambda path: ( + {'one': {'outputs': [{'method': 'native', + 'execution_identity': p.execution_identity(row)}]}}, + {'one': {'reference': reference, 'outputs': [opinion]}}, [], + )) + p.save_json(work / 'manifest.json', {}) + p.save_json(work / 'reviewer.json', {'model': 'synthetic'}) + return home, run, work, model, contract + + +def assemble(evidence, tmp_path): + home, _, work, model, _ = evidence + content, profile = tmp_path / 'content.json', tmp_path / 'profile.json' + p.save_json(content, model) + p.save_json(profile, {'independent_result_review_scope': 'all', + 'research_report': {'content': {'sha256': p.digest(content.read_bytes())}}}) + return review.assemble_schema_audit(home, content, work, tmp_path / 'assembled.json', + profile=profile) + + +@pytest.mark.parametrize('boundary', ['prepare', 'assemble', 'report']) +def test_same_case_ids_from_another_dataset_revision_are_rejected(evidence, tmp_path, boundary): + home, _, work, model, contract = evidence + contract['dataset_manifest_sha256'] = 'f' * 64 + with pytest.raises(ValueError, match='different dataset'): + if boundary == 'prepare': + content = tmp_path / 'content.json' + p.save_json(content, model) + review.prepare_schema_audit(home, content, work) + elif boundary == 'assemble': + assemble(evidence, tmp_path) + else: + report.schema_report_inputs(home, model) + + +@pytest.mark.parametrize('boundary', ['prepare', 'assemble']) +def test_full_audit_requires_every_declared_participant(evidence, tmp_path, boundary): + home, _, work, model, _ = evidence + model['participants']['unselected'] = {'label': 'Unselected'} + with pytest.raises(ValueError, match='declared participant'): + if boundary == 'prepare': + content = tmp_path / 'content.json' + p.save_json(content, model) + review.prepare_schema_audit(home, content, work) + else: + assemble(evidence, tmp_path) + assert not (tmp_path / 'assembled.json').exists() + + +@pytest.mark.parametrize('revision', ['reference_revision', 'projection_revision']) +def test_official_columns_reject_revised_reference_or_projection_scores(evidence, revision): + home, run, _, model, _ = evidence + assert report.schema_report_inputs(home, model)[2]['native']['population'] == 1 + scored = p.read_json(run / 'scores.json') + scored[revision] = {} + p.save_json(run / 'scores.json', scored) + model['evidence_runs'][0]['scores']['sha256'] = p.digest((run / 'scores.json').read_bytes()) + with pytest.raises(ValueError, match='original-reference'): + report.schema_report_inputs(home, model) + + +def test_report_revalidates_field_and_verdict_consistency(evidence): + home, _, _, model, _ = evidence + audit_path = Path(model['audit']['path']) + audit = p.read_json(audit_path) + audit['results']['native:one']['fields'][0]['status'] = 'incorrect' + p.save_json(audit_path, audit) + model['audit']['sha256'] = p.digest(audit_path.read_bytes()) + with pytest.raises(ValueError, match='contradicts'): + report.schema_report_inputs(home, model) From b3d50eec82b1fe5fc41910f5c9d948c2503be504 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:12:40 +0800 Subject: [PATCH 007/103] fix(web-extract): reject type collisions and inconsistent reviews --- .../benchmarks/scoring-regressions.json | 56 +++++++++++++++++++ src/harness/web_extract/human_eval.py | 11 ++-- src/harness/web_extract/report.py | 4 +- src/harness/web_extract/review.py | 8 ++- tests/web_extract/test_human_eval.py | 11 ++-- tests/web_extract/test_schema_audit.py | 26 ++++++--- tests/web_extract/test_schema_boundaries.py | 7 ++- tests/web_extract/test_schema_report.py | 12 ++-- 8 files changed, 107 insertions(+), 28 deletions(-) diff --git a/src/harness/benchmarks/scoring-regressions.json b/src/harness/benchmarks/scoring-regressions.json index 5d91d9e25..6188868f3 100644 --- a/src/harness/benchmarks/scoring-regressions.json +++ b/src/harness/benchmarks/scoring-regressions.json @@ -80392,6 +80392,62 @@ "expected": { "item_f1": 0 } + }, + { + "id": "human-eval-scalar-versus-array", + "reference": { + "price": 12 + }, + "prediction": { + "price": [ + 12 + ] + }, + "expected": { + "item_f1": 0 + } + }, + { + "id": "human-eval-array-versus-scalar", + "reference": { + "price": [ + 12 + ] + }, + "prediction": { + "price": 12 + }, + "expected": { + "item_f1": 0 + } + }, + { + "id": "human-eval-empty-versus-nested-array", + "reference": { + "price": [] + }, + "prediction": { + "price": [ + [] + ] + }, + "expected": { + "item_f1": 0 + } + }, + { + "id": "human-eval-nested-versus-empty-array", + "reference": { + "price": [ + [] + ] + }, + "prediction": { + "price": [] + }, + "expected": { + "item_f1": 0 + } } ] } diff --git a/src/harness/web_extract/human_eval.py b/src/harness/web_extract/human_eval.py index d2ed65e25..78432eed8 100644 --- a/src/harness/web_extract/human_eval.py +++ b/src/harness/web_extract/human_eval.py @@ -20,14 +20,15 @@ SCORE_REQUIRES_HOME = True MISSING_OUTPUT_NONE = True SCORING_REVISION = { - "id": "human-eval-record-items/3", + "id": "human-eval-record-items/4", "root_cause": ( "The upstream value_score sums scores over a hash-randomized key set. " "Use the same upstream leaf comparisons with math.fsum in sorted key order; " "aggregate cases with math.fsum too. The official wildcard scorer loses record " "associations and scores complete array fields all-or-nothing. Add a separate " "duplicate-preserving exact item F1, including numeric-equivalent nested array " - "multisets; keep every official metric unchanged." + "multisets and distinct array-member versus field identities; " + "keep every official metric unchanged." ), "regression_ids": [ "human-eval-missing-empty-reference", "human-eval-delivered-empty-reference", @@ -36,6 +37,8 @@ "human-eval-item-partial-recall", "human-eval-item-extra-record", "human-eval-item-number-equivalence", "human-eval-item-boolean-type", "human-eval-nested-number-equivalence", "human-eval-nested-array-multiplicity", + "human-eval-scalar-versus-array", "human-eval-array-versus-scalar", + "human-eval-empty-versus-nested-array", "human-eval-nested-versus-empty-array", ], "provider_calls_added": 0, } @@ -181,8 +184,8 @@ def exact_items(value, path=()): return Counter(item for key, child in value.items() for item in exact_items(child, (*path, key)).elements()) if isinstance(value, list) and value: - return Counter((path, item_identity(item)) for item in value) - return Counter({(path, item_identity(value)): 1}) + return Counter(("array-member", path, item_identity(item)) for item in value) + return Counter({("field", path, item_identity(value)): 1}) def item_scores(reference, prediction): diff --git a/src/harness/web_extract/report.py b/src/harness/web_extract/report.py index ac240ef53..7915d4530 100644 --- a/src/harness/web_extract/report.py +++ b/src/harness/web_extract/report.py @@ -4851,11 +4851,11 @@ def check_schema_cohort(output, *, release=False, visual_receipt=None): if relative and not resolved.is_file(): raise ValueError("Schema report has a broken evidence or navigation link: " + target) for lang in manifest["languages"]: - results = BeautifulSoup((output / lang / "results.html").read_text(), "html.parser") + results = BeautifulSoup((output / lang / "results.html").read_text(encoding="utf-8"), "html.parser") if len(results.select("tr[data-result]")) != expected: raise ValueError("Schema inventory omits results") for route in ("index", "interpretation"): - document = (output / lang / (route + ".html")).read_text() + document = (output / lang / (route + ".html")).read_text(encoding="utf-8") check_metric_guide(document, schema_metric_definitions(lang, manifest["case_count"]), title=message("schema_metric_guide", lang), diff --git a/src/harness/web_extract/review.py b/src/harness/web_extract/review.py index 845bc688a..8976548d6 100644 --- a/src/harness/web_extract/review.py +++ b/src/harness/web_extract/review.py @@ -181,7 +181,8 @@ def validate_schema_audit(result, packet): native.get("normalized_fields"), dict ): raise ValueError("Absent or failed output cannot be reviewed correct") - if any(field["status"] in {"incorrect", "partial", "missing", "unsupported"} + if any(field["status"] in { + "incorrect", "partial", "missing", "unsupported", "ambiguous"} or any(f["kind"] == "extraction_error" for f in field["findings"]) for field in review["fields"]): raise ValueError("Correct verdict contradicts field assessments") @@ -655,7 +656,12 @@ def reconcile_citations(result, packet): validate_structure(result, packet) expected = {c["case_id"]: c for c in packet["cases"]} for case in result["cases"]: + if case["case_id"] not in expected: + raise ValueError("Unknown schema audit case") item = expected[case["case_id"]] + methods = {o["method"]: o for o in item["outputs"]} + if any(o["method"] not in methods for o in case["outputs"]): + raise ValueError("Unknown schema audit method") source = Path(item["source_outline"]).read_text(encoding="utf-8") for group in [case["reference"], *case["outputs"]]: paths = item["reference_leaf_paths"] if group is case["reference"] else next( diff --git a/tests/web_extract/test_human_eval.py b/tests/web_extract/test_human_eval.py index 8d9288f86..43b4508b7 100644 --- a/tests/web_extract/test_human_eval.py +++ b/tests/web_extract/test_human_eval.py @@ -36,7 +36,8 @@ def dataset(tmp_path, monkeypatch): ' else: result[key]=item\n' ' return result\n' 'def _compare_values(pred,gold):\n' - ' if isinstance(gold,list): return float(sorted(pred,key=str)==sorted(gold,key=str))\n' + ' if isinstance(gold,list): return float(isinstance(pred,list) and ' + 'sorted(pred,key=str)==sorted(gold,key=str))\n' ' if isinstance(gold,str) and gold: return len(set(str(pred).split()) & ' 'set(gold.split()))/len(set(gold.split()))\n' ' return float(pred==gold)\n' @@ -46,7 +47,7 @@ def dataset(tmp_path, monkeypatch): ' except ValueError: return {"is_valid_json":0.,"key_f1":1.,"value_score":1.}\n' ' return {"is_valid_json":1.,"key_f1":float(value==gold),' '"value_score":float(value==gold)}\n' - ) + , encoding='utf-8') card = tmp_path / "card.json" p.save_json(card, {"dataset": {"commit": "synthetic", "cases": 2, "files": { "source/human_eval.parquet": p.digest((source / "human_eval.parquet").read_bytes()), @@ -66,12 +67,12 @@ def test_frozen_content_and_schema_roundtrip(dataset): truth = p.read_json(home / "dataset/dataset/ground-truth.json") for index, case in enumerate(truth.values()): cid = case["id"] - document = (home / "dataset/html" / (cid + ".html")).read_text() + document = (home / "dataset/html" / (cid + ".html")).read_text(encoding='utf-8') assert he.unwrap_markdown(document) == rows[index]["content"] assert "\n[x](/link) &\n", - "response": '{"name":"A & B"}'}, - {"schema": '{"type":"object","properties":{}}', "content": "Empty", "response": "{}"}, - ] - pq.write_table(pa.Table.from_pylist(rows), source / "human_eval.parquet") - # A deliberately simple synthetic evaluator tests orchestration, without - # pretending to reproduce the official BLEU implementation in a unit test. - (source / "metrics.py").write_text( - 'import json\n' - 'def _flatten_dict(value):\n' - ' result={}\n' - ' for key,item in value.items():\n' - ' if isinstance(item,list) and item and isinstance(item[0],dict):\n' - ' for field in item[0]: result[key+"[*]."+field]=[r[field] for r in item]\n' - ' else: result[key]=item\n' - ' return result\n' - 'def _compare_values(pred,gold):\n' - ' if isinstance(gold,list): return float(isinstance(pred,list) and ' - 'sorted(pred,key=str)==sorted(gold,key=str))\n' - ' if isinstance(gold,str) and gold: return len(set(str(pred).split()) & ' - 'set(gold.split()))/len(set(gold.split()))\n' - ' return float(pred==gold)\n' - 'def magic_metric_failed(): return {"key_f1":0.,"value_score":0.}\n' - 'def run_all(pred,schema,gold):\n' - ' try: value=json.loads(pred)\n' - ' except ValueError: return {"is_valid_json":0.,"key_f1":1.,"value_score":1.}\n' - ' return {"is_valid_json":1.,"key_f1":float(value==gold),' - '"value_score":float(value==gold)}\n' - , encoding='utf-8') - card = tmp_path / "card.json" - p.save_json(card, {"dataset": {"commit": "synthetic", "cases": 2, "files": { - "source/human_eval.parquet": p.digest((source / "human_eval.parquet").read_bytes()), - "evaluate.py": p.digest((source / "metrics.py").read_bytes()), - }}, "source": {"dataset_url": "https://example.org/pinned"}}) - monkeypatch.setattr(he, "CARD", card) - monkeypatch.setitem(BENCHMARKS, "human-eval", { - "profile": str(card), "module": "harness.web_extract.human_eval", - }) - home = tmp_path / "home" - p.prepare(home, benchmark="human-eval", source=source) - return home, source, rows - - -def test_frozen_content_and_schema_roundtrip(dataset): - home, source, rows = dataset - truth = p.read_json(home / "dataset/dataset/ground-truth.json") - for index, case in enumerate(truth.values()): - cid = case["id"] - document = (home / "dataset/html" / (cid + ".html")).read_text(encoding='utf-8') - assert he.unwrap_markdown(document) == rows[index]["content"] - assert "\n" - schema = {"type": "object", "properties": {"name": {"type": "string"}}} - received = [] - - class Response: - status = 200 - headers = {} - - def text(self): - return '{"json":{"name":"A & B"}}' - - def request(url, **kwargs): - received.append(kwargs) - return Response() - - monkeypatch.setattr(p, "request", request) - for method in ("scrapegraph-extract", "firecrawl-parse-json"): - p.provider_call(method, he.wrap_markdown(content), "https://example.org/", - schema=schema, native_options=he.PARTICIPANT_OPTIONS[method]) - assert received[0]["json_body"]["markdown"] == content - assert received[0]["json_body"]["schema"] == schema - assert b'text/html' in received[1]["body"] - assert he.wrap_markdown(content).encode() in received[1]["body"] - assert b'snapshot.html' in received[1]["body"] - with pytest.raises(ValueError, match="Unsupported native input route"): - p.validate_native_options("cloudflare-json", {"input_route": "markdown"}) - - -def test_missing_output_stays_zero_even_for_empty_gold(dataset): - home, _, _ = dataset - truth = p.read_json(home / "dataset/dataset/ground-truth.json") - scores = he.score(truth, {"human-eval-000": {"name": "A & B"}}, home=home) - assert scores["population"] == 2 - assert scores["delivery_count"] == 1 - assert scores["mean"]["value_score"] == .5 - assert scores["per_case"]["human-eval-001"]["value_score"] == 0 - rows = [{"method": "scrapegraph-extract", "case_id": cid, - "http_status": 200, "normalized_fields": None} for cid in truth] - result = p.score_rows(home, truth, rows, ["scrapegraph-extract"]) - assert result["scrapegraph-extract"]["delivery_count"] == 0 - - -def test_record_identity_preserves_pairs_duplicates_and_types(): - gold = [{"name": "A", "price": 1}, {"name": "B", "price": 2}] - assert he.record_identity(gold) == he.record_identity(list(reversed(gold))) - assert he.record_identity(gold) != he.record_identity( - [{"name": "A", "price": 2}, {"name": "B", "price": 1}]) - assert he.record_identity(["A", "A"]) != he.record_identity(["A"]) - assert he.record_identity(True) != he.record_identity(1) - assert he.record_identity(None) != he.record_identity("") - - -def test_value_score_is_independent_of_case_and_field_order(dataset): - home, _, _ = dataset - truth, predictions = {}, {} - for size in range(2, 80): - cid = str(size) - gold = {str(key): key for key in range(size)} - truth[cid] = {"response": gold, "schema_text": '{"type":"object"}'} - predictions[cid] = {**gold, "0": -1} - rows = [{"method": "cloudflare-json", "case_id": cid, - "http_status": 200, "normalized_fields": predictions[cid]} for cid in truth] - first = p.score_rows(home, truth, rows, ["cloudflare-json"]) - second = p.score_rows(home, dict(reversed(list(truth.items()))), rows, ["cloudflare-json"]) - assert first == second - expected = math.fsum((size - 1) / size for size in range(2, 80)) / 78 - assert first["cloudflare-json"]["mean"]["value_score"] == expected - - -def test_registered_controls_execute_through_public_scoring(dataset): - home, _, _ = dataset - # The scenario data stay in the shared registry. This fixture substitutes - # only the external evaluator dependency; requests and scoring use public APIs. - assert he.check_scoring_controls(home) == he.SCORING_REVISION["regression_ids"] - - -def test_source_changes_fail_before_materialization(dataset, tmp_path): - _, source, _ = dataset - (source / "metrics.py").write_text("changed", encoding='utf-8') - with pytest.raises(ValueError, match="Pinned human_eval source differs"): - he.prepare(tmp_path / "other", source) - assert not (tmp_path / "other/dataset").exists() - - -def test_certified_cohort_keeps_original_task_and_complete_population( - dataset, tmp_path, monkeypatch -): - from harness.web_extract.certification import run - - home, _, _ = dataset - profile = tmp_path / "plan.json" - create_plan(home, ["scrapegraph-extract"], profile) - monkeypatch.setattr(p, "load_runtime_env", lambda: None) - monkeypatch.setenv("SGAI_API_KEY", "synthetic") - dispatched = [] - - def call(method, document, url, fixture=None, **kwargs): - dispatched.append((document, kwargs)) - assert kwargs["prompt"] == "" - assert kwargs["native_options"] == {"input_route": "markdown"} - value = {"name": "A & B"} if len(dispatched) == 1 else {} - return {"http_status": 200, "elapsed_seconds": .01, "response": {"json": value}} - - monkeypatch.setattr(p, "provider_call", call) - result = run(home, profile, "scrapegraph-extract", tmp_path / "run") - assert result["cases"] == 2 - assert result["scores"]["scrapegraph-extract"]["delivery_count"] == 2 - assert (tmp_path / "run/certificate.json").is_file() - contract = p.read_json(tmp_path / "run/contract.json") - assert contract["extraction_instructions"] == "" - assert contract["extraction_instruction_delivery"] == "original-schema/1" - run(home, profile, "scrapegraph-extract", tmp_path / "run") - assert len(dispatched) == 2 - - -def test_source_reference_revision_preserves_original_and_rejects_unbound_edits( - dataset, monkeypatch, -): - home, _, _ = dataset - monkeypatch.setattr(he, 'check_scoring_controls', lambda home: ['paired-control']) - original = home / 'dataset/dataset/ground-truth.json' - source = home / 'dataset/markdown/human-eval-000.md' - revision = home / 'reference-revision.json' - value = {'schema': 'source-json-reference-revision/1', - 'original_reference_sha256': p.digest(original.read_bytes()), - 'root_cause': 'The original answer omitted the source heading punctuation.', - 'regression_ids': ['paired-control'], 'changes': [{ - 'case_id': 'human-eval-000', 'path': '/name', 'before': 'A & B', - 'after': '# A & B', 'source_sha256': p.digest(source.read_bytes()), - 'source_quote': '# A & B', 'reason': 'Retain the exact source heading.'}]} - p.save_json(revision, value) - revised, policy = he.load_reference_revision(home, revision) - assert revised['human-eval-000']['response']['name'] == '# A & B' - assert p.read_json(original)['human-eval-000']['response']['name'] == 'A & B' - assert policy == {} - for key, bad in [('before', 'wrong predecessor'), ('source_quote', 'invented words'), - ('source_sha256', '0' * 64)]: - candidate = json.loads(json.dumps(value)) - candidate['changes'][0][key] = bad - p.save_json(revision, candidate) - with pytest.raises(ValueError): - he.load_reference_revision(home, revision) - - -@pytest.mark.parametrize('defect', [ - 'missing_case', 'empty_cases', 'empty_mean', 'missing_leaf_metric', - 'missing_mean_metric', 'population', 'delivery', 'extra_case', -]) -def test_predecessor_scores_reject_incomplete_or_changed_evidence(dataset, defect): - import copy - - home, _, _ = dataset - truth = p.read_json(home / 'dataset/dataset/ground-truth.json') - rows = [{'method': 'cloudflare-json', 'case_id': cid, 'http_status': 200, - 'normalized_fields': case['response']} for cid, case in truth.items()] - current = p.score_rows(home, truth, rows, ['cloudflare-json']) - saved = copy.deepcopy(current) - for group in saved.values(): - for scores in [group['mean'], *group['per_case'].values()]: - for key in ('item_precision', 'item_recall', 'item_f1'): - scores.pop(key) - assert he.verify_predecessor_scores(saved, current, list(truth)) - group = saved['cloudflare-json'] - if defect == 'missing_case': - group['per_case'].pop(next(iter(truth))) - elif defect == 'empty_cases': - group['per_case'] = {} - elif defect == 'empty_mean': - group['mean'] = {} - elif defect == 'missing_leaf_metric': - group['per_case'][next(iter(truth))].pop('value_score') - elif defect == 'missing_mean_metric': - group['mean'].pop('value_score') - elif defect == 'population': - group['population'] += 1 - elif defect == 'delivery': - group['delivery_count'] -= 1 - else: - group['per_case']['not-declared'] = copy.deepcopy(next(iter(group['per_case'].values()))) - assert not he.verify_predecessor_scores(saved, current, list(truth)) - - -def test_nested_array_numeric_serializers_preserve_order_and_multiplicity(): - gold = {'items': [{'values': [100000000000000000000, 11]}]} - assert he.item_scores(gold, {'items': [{'values': [11, 1e20]}]})['item_f1'] == 1 - assert he.item_scores({'items': [{'values': [1, 1, 2]}]}, - {'items': [{'values': [1, 2, 2]}]})['item_f1'] == 0 - - -def test_reference_rescoring_rejects_a_corrupt_predecessor_envelope(dataset, monkeypatch, tmp_path): - home, _, _ = dataset - truth = p.read_json(home / 'dataset/dataset/ground-truth.json') - rows = [{'method': 'cloudflare-json', 'case_id': cid, 'http_status': 200, - 'normalized_fields': case['response']} for cid, case in truth.items()] - contract = {'kind': 'full_archived_extraction', 'case_ids': list(truth), - 'methods': ['cloudflare-json'], - 'dataset_manifest_sha256': p.digest((home / 'dataset/manifest.json').read_bytes())} - run = tmp_path / 'run' - p.save_json(run / 'manifest.json', {'synthetic_calls': list(truth)}) - monkeypatch.setattr(p, 'load_run', lambda path: (contract, rows)) - p.score_run(home, run) - original = home / 'dataset/dataset/ground-truth.json' - source = home / 'dataset/markdown/human-eval-000.md' - authority = tmp_path / 'revision.json' - p.save_json(authority, {'schema': 'source-json-reference-revision/1', - 'original_reference_sha256': p.digest(original.read_bytes()), - 'root_cause': 'Keep literal heading punctuation.', - 'regression_ids': he.check_scoring_controls(home), 'changes': [{ - 'case_id': 'human-eval-000', 'path': '/name', 'before': 'A & B', 'after': '# A & B', - 'source_sha256': p.digest(source.read_bytes()), 'source_quote': '# A & B', - 'reason': 'The literal source heading includes the marker.'}]}) - p.score_run(home, run, reference_revision=authority, output=run / 'revised.json') - predecessor = p.read_json(run / 'scores.json') - predecessor['cases'] = 0 - p.save_json(run / 'scores.json', predecessor) - with pytest.raises(ValueError, match='Original scores do not reproduce'): - p.score_run(home, run, reference_revision=authority, output=run / 'invalid.json') - assert not (run / 'invalid.json').exists() diff --git a/tests/web_extract/test_schema_audit.py b/tests/web_extract/test_schema_audit.py deleted file mode 100644 index 879df796d..000000000 --- a/tests/web_extract/test_schema_audit.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Nested JSON reviews must cover all retained results and cite the frozen source.""" - -import copy - -import pytest - -from harness.web_extract.pipeline import save_json -from harness.web_extract.review import ( - json_leaf_paths, - review_schema, - validate, -) - - -@pytest.fixture -def audit(tmp_path): - source = tmp_path / "source.md" - source.write_text("Alice: red mug, 12 EUR. Bob: blue mug, 15 EUR.\n", encoding='utf-8') - reference = {"items": [{"name": "Alice", "price": 12}, {"name": "Bob", "price": 15}]} - native = {"items": [{"name": "Bob", "price": 15}, {"name": "Alice", "price": 12}]} - item = { - "audit_kind": "schema-json-audit/1", "case_id": "one", - "source_outline": str(source), "source_html": str(source), - "reference": reference, "reference_leaf_paths": json_leaf_paths(reference), - "reference_field_inventory": ["items"], "outputs": [{ - "method": "native", "leaf_paths": json_leaf_paths(native), - "field_inventory": ["items"], "normalized_fields": native, "http_status": 200, - }], - } - group = {"field": "items", "status": "supported", "source_quote": "Alice: red mug, 12 EUR.", - "reason": "Both named records and their associated prices are supported.", - "findings": []} - result = {"cases": [{"case_id": "one", "reference": { - "checked_paths": item["reference_leaf_paths"], "fields": [group], "summary": "Supported", - }, "outputs": [{"method": "native", "verdict": "correct", "summary": "Reordered records", - "checked_paths": item["outputs"][0]["leaf_paths"], "fields": [copy.deepcopy(group)]}]}]} - return {"cases": [item]}, result - - -def test_nested_leaf_paths_preserve_empty_values_and_escaped_keys(): - assert json_leaf_paths({"a/b": [None, {}, []], "~": False}) == [ - "/a~1b/0", "/a~1b/1", "/a~1b/2", "/~0", - ] - - -def test_complete_source_grounded_review_is_accepted(audit): - packet, result = audit - assert review_schema(packet["cases"])["properties"]["cases"] - assert validate(result, packet) == result - - -@pytest.mark.parametrize("change", ["missing_method", "missing_leaf", "duplicate_leaf", - "missing_field", "invented_quote"]) -def test_incomplete_or_ungrounded_reviews_fail_closed(audit, change): - packet, result = audit - value = copy.deepcopy(result) - output = value["cases"][0]["outputs"][0] - if change == "missing_method": - value["cases"][0]["outputs"] = [] - elif change == "missing_leaf": - output["checked_paths"].pop() - elif change == "duplicate_leaf": - output["checked_paths"][1] = output["checked_paths"][0] - elif change == "missing_field": - output["fields"] = [] - else: - output["fields"][0]["source_quote"] = "The source never said this" - with pytest.raises(ValueError): - validate(value, packet) - - -def test_packet_persistence_does_not_embed_machine_scores(tmp_path, audit): - from harness.web_extract.review import read_packet, save_packet - - packet, _ = audit - save_json(tmp_path / "manifest.json", {"root": "."}) - path = tmp_path / "packets/one.json" - save_packet(path, packet, tmp_path) - restored = read_packet(path) - assert restored == packet - assert "value_score" not in path.read_text(encoding='utf-8') - - -def test_reconciliation_recovers_only_unambiguous_structure_and_literal_text(audit): - from harness.web_extract.review import reconcile_citations - - packet, result = audit - result = copy.deepcopy(result) - original = copy.deepcopy(result) - output = result['cases'][0]['outputs'][0] - output['checked_paths'][0] = '/0/name' - output['fields'][0]['source_quote'] = 'Alice: red mug, 12 EUR.' - restored = reconcile_citations(result, packet) - assert validate(restored, packet) == original - assert output['checked_paths'][0] == '/0/name' # Raw review remains untouched. - output['checked_paths'][0] = '/name' - with pytest.raises(ValueError, match='leaf path'): - validate(reconcile_citations(result, packet), packet) - - -def test_runtime_error_is_evidence_only_for_an_absent_output(audit): - packet, result = audit - native = packet['cases'][0]['outputs'][0] - native['normalized_fields'] = None - native['runtime_error'] = 'Schema generation rejected' - native['leaf_paths'] = [] - output = result['cases'][0]['outputs'][0] - output['verdict'] = 'no_output' - output['checked_paths'] = [] - output['fields'][0]['status'] = 'missing' - output['fields'][0]['source_quote'] = 'Schema generation rejected' - assert validate(result, packet) - native['normalized_fields'] = {} - with pytest.raises(ValueError, match='literal'): - validate(result, packet) - - -@pytest.mark.parametrize('defect', [ - 'absent', 'http_error', 'incorrect_field', 'ambiguous', 'error_finding']) -def test_correct_verdict_cannot_contradict_native_evidence(audit, defect): - packet, result = audit - output = result['cases'][0]['outputs'][0] - if defect == 'absent': - packet['cases'][0]['outputs'][0]['normalized_fields'] = None - elif defect == 'http_error': - packet['cases'][0]['outputs'][0]['http_status'] = 400 - elif defect in {'incorrect_field', 'ambiguous'}: - output['fields'][0]['status'] = 'ambiguous' if defect == 'ambiguous' else 'incorrect' - else: - output['fields'][0]['findings'] = [{ - 'path': '/items', 'kind': 'extraction_error', - 'reason': 'A requested record is missing.', - 'source_quote': 'Bob: blue mug, 15 EUR.', 'proposed_json': '', - }] - with pytest.raises(ValueError): - validate(result, packet) - - -@pytest.mark.parametrize('defect', ['outputs', 'reference', 'fields', 'findings']) -def test_malformed_cached_model_containers_are_retryable_value_errors(audit, defect): - from harness.web_extract.review import reconcile_citations - - packet, result = audit - case = result['cases'][0] - if defect in {'outputs', 'reference'}: - case.pop(defect) - elif defect == 'fields': - case['outputs'][0]['fields'] = None - else: - case['outputs'][0]['fields'][0]['findings'] = [None] - with pytest.raises(ValueError): - reconcile_citations(result, packet) - - -@pytest.mark.parametrize('defect', ['outputs', 'case', 'method', 'finding_kind']) -def test_resume_quarantines_a_malformed_raw_reply_and_retries_once( - tmp_path, monkeypatch, audit, defect -): - from harness.web_extract import review - from harness.web_extract.pipeline import digest, read_json - - packet, valid = audit - work = tmp_path / 'work' - path = work / 'packets/one.json' - review.save_packet(path, packet, tmp_path) - review.save_manifest(work, {'schema': 'schema-json-audit/1', 'case_count': 1, - 'result_count': 1, 'bindings': {}, 'packets': [{'path': str(path), - 'sha256': digest(path.read_bytes())}]}, tmp_path) - save_json(work / 'schema.json', review.review_schema(packet['cases'])) - malformed = copy.deepcopy(valid) - if defect == 'outputs': - malformed['cases'][0].pop('outputs') - elif defect == 'case': - malformed['cases'][0]['case_id'] = 'unknown-case' - elif defect == 'method': - malformed['cases'][0]['outputs'][0]['method'] = 'unknown-method' - else: - malformed['cases'][0]['outputs'][0]['fields'][0]['findings'] = [{ - 'kind': [], 'path': '/items', 'reason': 'Malformed reply.', - 'source_quote': 'Alice', 'proposed_json': '', - }] - replies = iter([malformed, valid]) - calls = [] - - def chat(**kwargs): - calls.append(kwargs) - return copy.deepcopy(next(replies)), {'requested_model': 'one-model'} - - monkeypatch.setattr('harness.review_runner.run_chat_reviewer', chat) - credentials = tmp_path / 'gateway.env' - credentials.write_text( - 'OPENAI_BASE_URL=https://example.org/v1\nOPENAI_API_KEY=test-only\n', encoding='utf-8') - options = {'model': 'one-model', 'workers': 1, 'backend': 'litellm', - 'credential_env_file': credentials} - assert len(review.run(work, **options)['needs_correction']) == 1 - assert review.run(work, **options)['packets'] == 1 - assert len(list((work / 'invalid').glob('*.json'))) == 1 - assert read_json(work / 'reviews/one.json') == valid - assert review.run(work, **options)['packets'] == 1 - assert len(calls) == 2 - - -def test_gateway_rejects_an_html_audit_without_a_complete_inline_source_contract(tmp_path): - from harness.web_extract.review import run - - work = tmp_path / 'html-review' - save_json(work / 'manifest.json', {'schema': 'product-audit/1', 'packets': [], 'bindings': {}}) - with pytest.raises(ValueError, match='complete frozen Markdown'): - run(work, model='one-model', backend='litellm') diff --git a/tests/web_extract/test_schema_boundaries.py b/tests/web_extract/test_schema_boundaries.py deleted file mode 100644 index 0e0805499..000000000 --- a/tests/web_extract/test_schema_boundaries.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Full reviews and official comparisons reject mixed or incomplete evidence.""" - -import json -from pathlib import Path - -import pytest - -from harness.web_extract import pipeline as p -from harness.web_extract import report, review - - -@pytest.fixture -def evidence(tmp_path, monkeypatch): - home, run, work = tmp_path / 'home', tmp_path / 'run', tmp_path / 'review' - source = home / 'dataset/markdown/one.md' - source.parent.mkdir(parents=True) - source.write_text('Amber mug: 12 EUR.', encoding='utf-8') - (home / 'dataset/html').mkdir() - (home / 'dataset/html/one.html').write_text('
Amber mug: 12 EUR.', encoding='utf-8') - schema = {'type': 'object', 'properties': {'name': {'type': 'string'}}} - native = {'name': 'Amber mug'} - p.save_json(home / 'dataset/manifest.json', {'benchmark': 'human-eval'}) - manifest_sha = p.digest((home / 'dataset/manifest.json').read_bytes()) - p.save_json(home / 'dataset/dataset/ground-truth.json', { - 'one': {'response': native, 'schema_text': json.dumps(schema)}, - }) - (home / 'dataset/evaluate.py').write_text( - '# A synthetic external scorer dependency.\n', encoding='utf-8') - row = {'method': 'native', 'case_id': 'one', 'normalized_fields': native, - 'http_status': 200, 'response': {'json': native}} - contract = {'methods': ['native'], 'case_ids': ['one'], - 'kind': 'full_archived_extraction', 'pipeline_sha256': 'frozen-pipeline', - 'dataset_manifest_sha256': manifest_sha} - p.save_json(run / 'manifest.json', {'synthetic_calls': ['one']}) - p.save_json(run / 'contract.json', contract) - call_sha = p.digest((run / 'manifest.json').read_bytes()) - plan = tmp_path / 'plan.json' - case_sha = p.digest(b'one\n') - p.save_json(plan, {'case_count': 1, 'case_ids_sha256': case_sha, - 'participants': {'native': {}}}) - plan_sha = p.digest(plan.read_bytes()) - p.save_json(run / 'certification-precall.json', { - 'profile_sha256': plan_sha, 'method': 'native', 'case_ids_sha256': case_sha, - 'dataset_manifest_sha256': manifest_sha, 'pipeline_sha256': 'frozen-pipeline'}) - score = {'population': 1, 'delivery_count': 1, 'mean': {}, 'per_case': {'one': {}}} - p.save_json(run / 'scores.json', {'scores': {'native': score}, - 'call_manifest_sha256': call_sha, - 'scorer_sha256': p.digest((home / 'dataset/evaluate.py').read_bytes())}) - p.save_json(run / 'certificate.json', { - 'manifest_sha256': call_sha, 'scores_sha256': p.digest((run / 'scores.json').read_bytes()), - 'precall_sha256': p.digest((run / 'certification-precall.json').read_bytes()), - 'profile_sha256': plan_sha, 'case_count': 1, 'method': 'native', - 'dataset_manifest_sha256': manifest_sha}) - field = {'field': 'name', 'status': 'supported', 'source_quote': 'Amber mug', - 'reason': 'The source names the mug.', 'findings': []} - opinion = {'method': 'native', 'verdict': 'correct', 'summary': 'Correct mug.', - 'fields': [field], 'checked_paths': ['/name']} - reference = {'checked_paths': ['/name'], 'fields': [field], 'summary': 'Supported mug.'} - audit = {'review_scope': 'all', 'accepted_unreviewed_passes': 0, - 'dataset_manifest_sha256': manifest_sha, - 'results': {'native:one': {**opinion, - 'execution_identity': p.execution_identity(row)}}, - 'reference_reviews': {'one': reference}} - p.save_json(tmp_path / 'audit.json', audit) - model = {'participants': {'native': {'label': 'Native'}}, - 'audit': {'path': str(tmp_path / 'audit.json'), - 'sha256': p.digest((tmp_path / 'audit.json').read_bytes())}, - 'evidence_runs': [{'path': str(run), 'plan': str(plan), 'plan_sha256': plan_sha, - 'scores': {'path': str(run / 'scores.json'), - 'sha256': p.digest((run / 'scores.json').read_bytes())}}]} - monkeypatch.setattr(p, 'validate_dataset', lambda *a, **kw: None) - monkeypatch.setattr(report, 'validate_dataset', lambda *a, **kw: None) - monkeypatch.setattr(p, 'load_run', lambda path: (contract, [row])) - monkeypatch.setattr(report, 'load_run', lambda path: (contract, [row])) - monkeypatch.setattr(report, 'score_rows', lambda *a, **kw: {'native': score}) - monkeypatch.setattr(review, 'completed', lambda path: ( - {'one': {'outputs': [{'method': 'native', - 'execution_identity': p.execution_identity(row)}]}}, - {'one': {'reference': reference, 'outputs': [opinion]}}, [], - )) - p.save_json(work / 'manifest.json', {}) - p.save_json(work / 'reviewer.json', {'model': 'synthetic'}) - return home, run, work, model, contract - - -def assemble(evidence, tmp_path): - home, _, work, model, _ = evidence - content, profile = tmp_path / 'content.json', tmp_path / 'profile.json' - p.save_json(content, model) - p.save_json(profile, {'independent_result_review_scope': 'all', - 'research_report': {'content': {'sha256': p.digest(content.read_bytes())}}}) - return review.assemble_schema_audit(home, content, work, tmp_path / 'assembled.json', - profile=profile) - - -@pytest.mark.parametrize('boundary', ['prepare', 'assemble', 'report']) -def test_same_case_ids_from_another_dataset_revision_are_rejected(evidence, tmp_path, boundary): - home, _, work, model, contract = evidence - contract['dataset_manifest_sha256'] = 'f' * 64 - with pytest.raises(ValueError, match='different dataset'): - if boundary == 'prepare': - content = tmp_path / 'content.json' - p.save_json(content, model) - review.prepare_schema_audit(home, content, work) - elif boundary == 'assemble': - assemble(evidence, tmp_path) - else: - report.schema_report_inputs(home, model) - - -@pytest.mark.parametrize('boundary', ['prepare', 'assemble']) -def test_full_audit_requires_every_declared_participant(evidence, tmp_path, boundary): - home, _, work, model, _ = evidence - model['participants']['unselected'] = {'label': 'Unselected'} - with pytest.raises(ValueError, match='declared participant'): - if boundary == 'prepare': - content = tmp_path / 'content.json' - p.save_json(content, model) - review.prepare_schema_audit(home, content, work) - else: - assemble(evidence, tmp_path) - assert not (tmp_path / 'assembled.json').exists() - - -@pytest.mark.parametrize('revision', ['reference_revision', 'projection_revision']) -def test_official_columns_reject_revised_reference_or_projection_scores(evidence, revision): - home, run, _, model, _ = evidence - assert report.schema_report_inputs(home, model)[2]['native']['population'] == 1 - scored = p.read_json(run / 'scores.json') - scored[revision] = {} - selected = run / 'scores-revised.json' - p.save_json(selected, scored) - model['evidence_runs'][0]['scores'] = { - 'path': str(selected), 'sha256': p.digest(selected.read_bytes())} - with pytest.raises(ValueError, match='original-reference'): - report.schema_report_inputs(home, model) - - -def test_report_revalidates_field_and_verdict_consistency(evidence): - home, _, _, model, _ = evidence - audit_path = Path(model['audit']['path']) - audit = p.read_json(audit_path) - audit['results']['native:one']['fields'][0]['status'] = 'incorrect' - p.save_json(audit_path, audit) - model['audit']['sha256'] = p.digest(audit_path.read_bytes()) - with pytest.raises(ValueError, match='contradicts'): - report.schema_report_inputs(home, model) - - -@pytest.mark.parametrize('defect', ['missing_precall', 'precall_profile', 'certificate_method', - 'certificate_population', 'certificate_profile']) -def test_schema_report_requires_complete_original_certification(evidence, defect): - home, run, _, model, _ = evidence - assert report.schema_report_inputs(home, model)[2]['native']['population'] == 1 - if defect == 'missing_precall': - (run / 'certification-precall.json').unlink() - elif defect == 'precall_profile': - path = run / 'certification-precall.json' - value = p.read_json(path) - value['profile_sha256'] = 'wrong' - p.save_json(path, value) - else: - path = run / 'certificate.json' - value = p.read_json(path) - key = {'certificate_method': 'method', 'certificate_population': 'case_count', - 'certificate_profile': 'profile_sha256'}[defect] - value[key] = 'wrong' - p.save_json(path, value) - with pytest.raises((ValueError, FileNotFoundError)): - report.schema_report_inputs(home, model) diff --git a/tests/web_extract/test_schema_report.py b/tests/web_extract/test_schema_report.py deleted file mode 100644 index d01f430d7..000000000 --- a/tests/web_extract/test_schema_report.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Arbitrary-schema reports retain measured values and inspectable evidence.""" - -import pytest -from bs4 import BeautifulSoup - -from harness.web_extract.example import create_schema -from harness.web_extract.pipeline import digest, read_json, save_json -from harness.web_extract.report import check - - -@pytest.fixture -def report(tmp_path): - output = tmp_path / 'report' - create_schema(output, language='en') - return output - - -def test_schema_report_has_complete_matrix_and_source_only_dataset(report): - manifest = check(report) - assert manifest['observations'] == manifest['reviewed_results'] == 6 - assert 'Relations compare model-reviewed correctness' in ( - report / 'en/results.html' - ).read_text(encoding='utf-8') - for language in ('en', 'zh'): - results = BeautifulSoup( - (report / language / 'results.html').read_text(encoding='utf-8'), 'html.parser') - assert len(results.select('tr[data-result]')) == 6 - detail = BeautifulSoup( - (report / language / 'c-two.html').read_text(encoding='utf-8'), 'html.parser') - assert len(detail.select('[data-result-panel]')) == 3 - assert detail.select_one('[data-result-logs] [data-kind="error"]').get_text() == ( - 'Synthetic schema rejection' - ) - task = (report / language / 't-one.html').read_text(encoding='utf-8') - assert 'Synthetic schema rejection' not in task - assert 'Amber mug' in task - with pytest.raises(ValueError, match='Synthetic'): - check(report, release=True) - - -def mutate(report, route, transform): - path = report / route - path.write_text(transform(path.read_text(encoding='utf-8')), encoding='utf-8') - manifest = read_json(report / 'report-manifest.json') - manifest['files'][route] = digest(path.read_bytes()) - save_json(report / 'report-manifest.json', manifest) - - -@pytest.mark.parametrize('defect', ['evidence', 'summary', 'logs', 'highlight']) -def test_schema_semantics_reject_invalid_reports_even_with_rebound_files(report, defect): - if defect == 'evidence': - mutate(report, 'en/c-two.html', lambda text: text.replace( - 'public-evidence/r-cloudflare-json-two.json', 'public-evidence/absent.json')) - elif defect == 'summary': - mutate(report, 'en/c-two.html', lambda text: text.replace( - 'data-failure-diagnosis="">', - 'data-failure-diagnosis="">
Product
", + "https://original.example/item", + "https://snapshots.example/hash/case", + native_options=OPTIONS, + schema=SCHEMA, + prompt="Copy the main record.", + ) + args = request.call_args + assert args.args == ("https://extract.example/extract/crawl4ai",) + body = args.kwargs["json_body"] + assert body["url"] == "https://snapshots.example/hash/case" + assert body["expected_runtime"] == RUNTIME + assert "Copy the main record." in body["schema"]["properties"]["price"]["description"] + assert "synthetic-secret" not in str(result) + assert normalize("crawl4ai-json", result["response"])["price"] == "12.99" + + +@pytest.mark.parametrize( + "options", + [ + {}, + {"endpoint": "https://extract.example"}, + {"endpoint": "https://extract.example", "runtime_identity": {"engine": "crawl4ai"}}, + ], +) +def test_run_requires_a_frozen_model_and_implementation(options): + with pytest.raises(ValueError, match="runtime identity"): + validate_native_options("crawl4ai-json", options) diff --git a/tests/web_extract/test_snapshots.py b/tests/web_extract/test_snapshots.py index 297407a9f..ce42e3d9c 100644 --- a/tests/web_extract/test_snapshots.py +++ b/tests/web_extract/test_snapshots.py @@ -179,7 +179,29 @@ def test_new_plan_freezes_one_url_for_every_participant(home, tmp_path): methods = [ name for name in METHODS if name not in {"extruct", "firecrawl-parse-json", "zyte-custom"} ] - plan = create_plan(home, methods, tmp_path / "plan.json", snapshots=source) + options = { + method: { + "endpoint": "https://extract.example", + "runtime_identity": { + "engine": engine, + "version": "synthetic", + "model": "test-model", + "model_base": "https://model.example", + "implementation_sha256": "a" * 64, + "environment_sha256": "b" * 64, + }, + } + for method, engine in { + "crawl4ai-json": "crawl4ai", + "scrapegraph-oss-json": "scrapegraph", + "stagehand-local-json": "stagehand", + }.items() + } + plan = create_plan( + home, methods, tmp_path / "plan.json", snapshots=source, native_options=options + ) + for method, native in options.items(): + assert plan["participants"][method]["native_options"] == native assert all( row["input"] == "url" and row["fixture_suffix"] == "" for row in plan["participants"].values() diff --git a/tests/web_extract/test_swde.py b/tests/web_extract/test_swde.py index 7609fa1f3..bc2050e10 100644 --- a/tests/web_extract/test_swde.py +++ b/tests/web_extract/test_swde.py @@ -1353,6 +1353,9 @@ def test_partial_labels_need_field_role_adjudication_not_suffix_matching(): def test_custom_schema_projection_retains_fields_and_invalid_types(method): value = {"author": ["Ada"], "year": "invalid scalar", "extra": 123} payload = { + "crawl4ai-json": {"data": value}, + "scrapegraph-oss-json": {"data": value}, + "stagehand-local-json": {"data": value}, "cloudflare-json": {"result": value}, "firecrawl-json": {"data": {"json": value}}, "firecrawl-parse-json": {"data": {"json": value}}, From f1a81f0f38a255a76f424db355d0949335ae6318 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:46:18 +0800 Subject: [PATCH 013/103] fix(web-extract): use the configured model context window --- src/harness/web_extract/self_hosted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/harness/web_extract/self_hosted.py b/src/harness/web_extract/self_hosted.py index 026adc8d0..308749433 100644 --- a/src/harness/web_extract/self_hosted.py +++ b/src/harness/web_extract/self_hosted.py @@ -118,7 +118,7 @@ def graph(value): source=value["url"], schema=create_model(value["schema"]), config={ - "llm": {"model_instance": model, "model_tokens": 65536}, + "llm": {"model_instance": model, "model_tokens": 1_000_000}, "verbose": False, "headless": True, "reattempt": False, From 8954794d386b7b5640cd6c065091d8b50cd37284 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:53:30 +0800 Subject: [PATCH 014/103] feat(web-extract): add authenticated Firecrawl self-hosting --- .env.example | 3 +- Dockerfile | 2 +- examples/web_extract/README.md | 12 +++-- src/harness/web_extract/pipeline.py | 15 +++++- src/harness/web_extract/self_hosted.py | 69 +++++++++++++++++++++++--- tests/web_extract/test_snapshots.py | 1 + tests/web_extract/test_swde.py | 1 + 7 files changed, 90 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 74029012b..4942d8bad 100644 --- a/.env.example +++ b/.env.example @@ -144,4 +144,5 @@ WEB_EXTRACT_PUBLICATION_HOSTS= # Self-hosted open-source Web Extract service (caller authentication). EXTRACTION_API_KEY= # Service-only environment: EXTRACTION_MODEL, EXTRACTION_MODEL_BASE, -# EXTRACTION_MODEL_API_KEY, SNAPSHOT_ORIGINS and optional CHROME_PATH. +# EXTRACTION_MODEL_API_KEY, EXTRACTION_MODEL_CONTEXT_TOKENS, SNAPSHOT_ORIGINS +# and optional CHROME_PATH. Firecrawl also needs FIRECRAWL_BACKEND_URL and FIRECRAWL_IMAGE. diff --git a/Dockerfile b/Dockerfile index 6564e249e..472066ee8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.10-slim-bookworm +FROM docker.1ms.run/library/python:3.12.10-slim-bookworm RUN apt-get update && apt-get install -y --no-install-recommends chromium libgeos-c1v5 libspatialindex6 && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir uv==0.7.8 WORKDIR /app diff --git a/examples/web_extract/README.md b/examples/web_extract/README.md index 5d31b60c3..5ba3c0e7b 100644 --- a/examples/web_extract/README.md +++ b/examples/web_extract/README.md @@ -492,8 +492,8 @@ resolve from `" ) interpretation += ( - f'' + f'' + (message("representative_result_and_complete_response", lang)) + "
" ) @@ -2015,6 +2015,7 @@ def build(output: Path, content: Path, *, home: Path, run=None, language=None, p "