From 852b1cc6f47e653b871ff56a241f624de3d32b3c Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Mon, 28 Sep 2026 16:49:44 +0300
Subject: [PATCH 1/7] feat(search): search the web for claims that cite no
source
A pasted post or plain text with no link, marker or bibliography used to
end in a parse error. It is now searched: up to five check-worthy
sentences go to Tavily or a SearXNG instance, and the pages found are
read and scored through the usual fetch and NLI path. Each finding is
marked "found by proofpath", and an empty search is reported as NO
EVIDENCE FOUND (searched), never as REFUTED. A judge writes the queries
and translates non-English claims; when it is limited, the sentence
itself is searched and the run says so.
The judge now falls back from Groq to a local qwen3.5:9b in Ollama the
moment Groq is rate-limited or down. The request that failed is sent
again to the local model, the switch is announced (stderr, TUI line,
report), and the next run starts on Groq again. Nothing is ever pulled.
Also: --no-search, the search section in /config, footer lines, and an
AVeriTeC --search eval mode that excludes the claim's own fact-check site.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
README.md | 3 +-
docs/superpowers/OPEN-ITEMS.md | 1 +
.../plans/2026-09-28-evidence-search.md | 2700 +++++++++++++++++
.../2026-09-28-evidence-search-design.md | 225 ++
scripts/eval_averitec.py | 74 +-
scripts/live_judge_fallback.py | 199 ++
src/proofpath/claims.py | 83 +
src/proofpath/cli.py | 16 +-
src/proofpath/config.py | 61 +-
src/proofpath/document.py | 11 +
src/proofpath/eval/averitec.py | 3 +
src/proofpath/events.py | 4 +
src/proofpath/judge.py | 637 +++-
src/proofpath/polite.py | 19 +-
src/proofpath/prompts/queries.md | 40 +
src/proofpath/report.py | 97 +-
src/proofpath/sarif.py | 19 +
src/proofpath/search/__init__.py | 79 +
src/proofpath/search/language.py | 49 +
src/proofpath/search/providers.py | 126 +
src/proofpath/search/queries.py | 95 +
src/proofpath/settings_hints.py | 6 +
src/proofpath/tui/app.py | 46 +-
src/proofpath/tui/widgets/__init__.py | 2 +
src/proofpath/tui/widgets/config_panel.py | 46 +-
src/proofpath/tui/widgets/footer.py | 19 +-
src/proofpath/tui/widgets/judge_notice.py | 45 +
src/proofpath/ui.py | 24 +-
src/proofpath/verify.py | 431 ++-
tests/data/verify-report-golden.json | 55 +-
tests/fixtures/search/searxng.json | 8 +
tests/fixtures/search/tavily.json | 11 +
tests/test_averitec.py | 13 +
tests/test_cache.py | 2 +-
tests/test_check_cli.py | 20 +
tests/test_claims.py | 60 +
tests/test_config.py | 52 +
tests/test_config_cli.py | 2 +-
tests/test_eval_averitec.py | 62 +
tests/test_judge.py | 51 +-
tests/test_judge_fallback.py | 859 ++++++
tests/test_polite.py | 23 +
tests/test_report.py | 85 +-
tests/test_sarif.py | 17 +
tests/test_search.py | 144 +
tests/test_search_queries.py | 215 ++
tests/test_tui_app.py | 273 +-
tests/test_ui.py | 23 +
tests/test_verify_search.py | 477 +++
49 files changed, 7472 insertions(+), 140 deletions(-)
create mode 100644 docs/superpowers/plans/2026-09-28-evidence-search.md
create mode 100644 docs/superpowers/specs/2026-09-28-evidence-search-design.md
create mode 100644 scripts/live_judge_fallback.py
create mode 100644 src/proofpath/prompts/queries.md
create mode 100644 src/proofpath/search/__init__.py
create mode 100644 src/proofpath/search/language.py
create mode 100644 src/proofpath/search/providers.py
create mode 100644 src/proofpath/search/queries.py
create mode 100644 src/proofpath/tui/widgets/judge_notice.py
create mode 100644 tests/fixtures/search/searxng.json
create mode 100644 tests/fixtures/search/tavily.json
create mode 100644 tests/test_judge_fallback.py
create mode 100644 tests/test_search.py
create mode 100644 tests/test_search_queries.py
create mode 100644 tests/test_verify_search.py
diff --git a/README.md b/README.md
index 101f0c7..393b25f 100644
--- a/README.md
+++ b/README.md
@@ -348,7 +348,8 @@ one per citation.
**Providers.** Default is Groq `openai/gpt-oss-120b` (free without a card, no training
on submitted data). `proofpath config set judge.provider gemini` switches to Gemini —
note that Google trains on free-tier prompts outside the EEA/UK/CH, and proofpath prints
-that warning once per run. `judge.provider ollama` runs fully offline. Gemini and Ollama are
+that warning once per run. `judge.provider ollama` runs fully offline on `qwen3.5:9b`,
+which must already be installed (proofpath never pulls a model). Gemini and Ollama are
fixture-tested and were not exercised live in v0.3.0. All three speak
the OpenAI `chat/completions` shape. The key comes from `GROQ_API_KEY` / `GEMINI_API_KEY`
in the environment or a `.env` file, never from config, and is never printed.
diff --git a/docs/superpowers/OPEN-ITEMS.md b/docs/superpowers/OPEN-ITEMS.md
index ec4522c..ca2f253 100644
--- a/docs/superpowers/OPEN-ITEMS.md
+++ b/docs/superpowers/OPEN-ITEMS.md
@@ -542,6 +542,7 @@ against. "Paste the text" is only an answer for a tweet that links its source.
| # | Item | Note |
|---|---|---|
| 17.1 | **Evidence search for a source-less claim** (decided 2026-09-18: this is the fix, path B) | An opt-in stage for a document that cites nothing: query a web search provider with the user's own key (Tavily / Brave / SearXNG; consent per §7.1, never on by default), take the top N pages and run them through the existing fetch → retrieval → entailment path. The report must carry a **distinct state** for such a source — `found by proofpath, not cited by the author` — on every finding and in coverage; rules 1–2 stay: passage or no verdict, unreachable stays unreachable. **Gate:** AVeriTeC (§15.1) measures exactly this scenario at 0.270 vs a 0.708 majority baseline, so the stage ships only once a web-claim eval is above baseline, or it ships flagged experimental with that number printed in the report. Not the judge: a judge answering with no source is the truth oracle §3 forbids |
+| 17.1a | **How 17.1 searches** (decided 2026-09-28) | Tavily + SearXNG behind one `Searcher`; query = the cleaned sentence, or 1–2 judge-written queries when `--judge` is set, falling back to the sentence with a visible notice on a rate limit / quota; heuristic check-worthiness filter, cap 5 claims × 3 pages. **Revises "never on by default":** runs automatically on source-less text once a provider is configured (configuring it is the consent); `--no-search` / `permissions.web_search = deny` turn it off. Design: `specs/2026-09-28-evidence-search-design.md` |
| 17.2 | README:280 and the `check --url` X message overstate "paste the text" | both should say a tweet with no link cannot be checked, until 17.1 lands |
| 17.3 | The "cites nothing" finding fires for a whole document, never per paragraph | a long pasted post where only some paragraphs link is fine today (it has references); a `.txt` mixing cited and uncited claims reports nothing about the uncited ones — that is the existing per-claim coverage's job, but worth checking it says so |
diff --git a/docs/superpowers/plans/2026-09-28-evidence-search.md b/docs/superpowers/plans/2026-09-28-evidence-search.md
new file mode 100644
index 0000000..0553a58
--- /dev/null
+++ b/docs/superpowers/plans/2026-09-28-evidence-search.md
@@ -0,0 +1,2700 @@
+# Evidence Search Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** A text that cites no source is searched on the web (Tavily or SearXNG), the pages found are read and checked like cited sources, and the report says what was found and that proofpath (not the author) found it.
+
+**Architecture:** A new `Searching` stage in `verify.prepare()`, between Claims and Resolving, runs only when the document cites nothing and a searcher is configured. It picks check-worthy sentences (`claims.checkworthy`), builds queries (the sentence text, or judge-written with a noticed fallback), asks a `Searcher` for addresses, and turns the kept hits into synthetic `Reference(origin="search")` entries. From there, Resolving, Fetching and `decide_all` run unchanged. The report gains three things:
+- two kinds, `EVIDENCE_FOUND` and `NO_EVIDENCE`;
+- a provenance note on every finding about a found page;
+- a `SearchSummary` block.
+
+**Tech Stack:** Python 3.10+, httpx via `polite.PoliteClient`, typer, textual, pytest + respx.
+
+**Spec:** `docs/superpowers/specs/2026-09-28-evidence-search-design.md` (read it first).
+
+## Global Constraints
+
+- Type annotations are mandatory on every function signature. `ruff check` and `ruff format --check` must be clean.
+- `pathlib` everywhere; every file read and write passes `encoding="utf-8"`.
+- **No network in unit tests.** Use `respx` plus `tests/fixtures/search/*.json`, or stub searchers.
+- Secrets: the Tavily key is read only through `secrets.resolve_api_key`. It is never logged, never put in an exception, never in a `repr`, never in config.
+- Rule 1: a provider's snippet or `content` is never evidence. Only pages fetched by the ladder are.
+- Rule 2: `NO EVIDENCE FOUND (searched)` is never REFUTED; an unreadable found page keeps its own §15 state.
+- Rule 4: search never prompts. `permissions.web_search` of `ask` or `deny` turns it off.
+- `cli` and `tui` hold no logic: they pass `search=` through to `Engine.default`.
+- Code, identifiers and comments in English. Match the surrounding comment style: comments say *why*.
+- **Git:** do not commit. Each task ends by staging its files (`git add …`). The user commits.
+- Run tests with `uv run pytest …`.
+
+---
+
+### Task 1: `[search]` config section and `permissions.web_search`
+
+**Files:**
+- Modify: `src/proofpath/config.py`
+- Modify: `src/proofpath/settings_hints.py`
+- Test: `tests/test_config.py`
+
+**Interfaces:**
+- Produces:
+ - `config.SearchProvider = Literal["off", "tavily", "searxng"]` and `config.SEARCH_PROVIDERS: tuple[str, ...]`.
+ - `config.SearchConfig(provider: SearchProvider = "off", api_key_env: str = "TAVILY_API_KEY", base_url: str = "", max_claims: int = 5, results_per_claim: int = 3)`.
+ - `Config.search: SearchConfig`.
+ - `Permissions.web_search: Permission = "allow"`.
+ - `settings_hints.SEARCH_SETTING = "search.provider tavily"`.
+
+- [ ] **Step 1: Write the failing tests** (append to `tests/test_config.py`; add any missing imports: `SearchConfig`, `set_value`, `load_config`, `ConfigError`, `pytest`, `Path`)
+
+```python
+def test_search_is_off_by_default_and_web_search_is_allowed() -> None:
+ config = Config()
+ assert config.search == SearchConfig()
+ assert config.search.provider == "off"
+ assert config.search.api_key_env == "TAVILY_API_KEY"
+ assert (config.search.max_claims, config.search.results_per_claim) == (5, 3)
+ assert config.permissions.web_search == "allow"
+
+
+def test_the_search_section_loads_from_toml(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ path.write_text(
+ '[search]\nprovider = "searxng"\nbase_url = "http://localhost:8888"\nmax_claims = 2\n',
+ encoding="utf-8",
+ )
+ config = load_config(path)
+ assert config.search.provider == "searxng"
+ assert config.search.base_url == "http://localhost:8888"
+ assert config.search.max_claims == 2
+
+
+@pytest.mark.parametrize(
+ "body",
+ ['provider = "bing"', "max_claims = 0", "max_claims = true", 'max_claims = "5"'],
+)
+def test_bad_search_values_are_refused_by_name(tmp_path: Path, body: str) -> None:
+ path = tmp_path / "config.toml"
+ path.write_text(f"[search]\n{body}\n", encoding="utf-8")
+ with pytest.raises(ConfigError, match=r"search\."):
+ load_config(path)
+
+
+def test_set_value_parses_an_integer_and_round_trips_it(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ updated = set_value("search.max_claims", "3", path)
+ assert updated.search.max_claims == 3
+ assert "max_claims = 3" in path.read_text(encoding="utf-8")
+ assert load_config(path).search.max_claims == 3
+ with pytest.raises(ConfigError, match="positive integer"):
+ set_value("search.max_claims", "many", path)
+
+
+def test_set_value_takes_a_search_provider(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ assert set_value("search.provider", "tavily", path).search.provider == "tavily"
+ with pytest.raises(ConfigError, match="search.provider"):
+ set_value("search.provider", "bing", path)
+```
+
+- [ ] **Step 2: Run them to see them fail**
+
+Run: `uv run pytest tests/test_config.py -k "search" -v`
+Expected: FAIL with `ImportError: cannot import name 'SearchConfig'`.
+
+- [ ] **Step 3: Implement in `src/proofpath/config.py`**
+
+Change the typing import to `from typing import Any, Literal, get_args, get_origin`. Add `web_search` to `Permissions`:
+
+```python
+@dataclass(frozen=True)
+class Permissions:
+ # Step 3 of the fetch ladder: ~280 MB browser engine. Never installed silently.
+ install_browser: Permission = "ask"
+ network: Permission = "allow"
+ # Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). Only matters
+ # once ``search.provider`` is set: configuring a provider is the consent. The
+ # search never prompts, so ``ask`` is read as ``deny`` (rule 4).
+ web_search: Permission = "allow"
+```
+
+After `JudgeConfig`, add:
+
+```python
+SearchProvider = Literal["off", "tavily", "searxng"]
+SEARCH_PROVIDERS: tuple[str, ...] = get_args(SearchProvider)
+
+
+@dataclass(frozen=True)
+class SearchConfig:
+ """Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). Off until a
+ provider is named. The key is never stored here: ``api_key_env`` names the
+ variable that holds it, exactly as ``JudgeConfig`` does."""
+
+ provider: SearchProvider = "off"
+ api_key_env: str = "TAVILY_API_KEY"
+ base_url: str = "" # SearXNG only: the user's own instance
+ max_claims: int = 5
+ results_per_claim: int = 3
+```
+
+Add `search: SearchConfig = field(default_factory=SearchConfig)` to `Config` after `judge`, and `"search": SearchConfig,` to `_SECTIONS` after `"judge"`.
+
+In `_coerce`, after the `Permission` branch, add:
+
+```python
+ if get_origin(expected) is Literal:
+ choices = get_args(expected)
+ if isinstance(value, str) and value in choices:
+ return value
+ raise ConfigError(f"{path}: {where} must be one of {', '.join(choices)}, got {value!r}")
+ if expected is int:
+ # ``bool`` is an ``int`` to Python and a different answer to a reader.
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 1:
+ return value
+ raise ConfigError(f"{path}: {where} must be a positive integer, got {value!r}")
+```
+
+In `_resolve_type`, extend the map:
+
+```python
+ return {
+ "Permission": Permission,
+ "SearchProvider": SearchProvider,
+ "bool": bool,
+ "int": int,
+ "str": str,
+ }[annotation]
+```
+
+In `_toml_value`, after the `bool` check (the order matters, because a bool is an int):
+
+```python
+ if isinstance(value, int):
+ return str(value)
+```
+
+In `set_value`, after the `bool` block:
+
+```python
+if expected is int:
+ try:
+ value = int(raw_value.strip())
+ except ValueError:
+ raise ConfigError(f"{dotted_key} must be a positive integer, got {raw_value!r}") from None
+```
+
+In `src/proofpath/settings_hints.py`, append:
+
+```python
+#: ``search.provider`` set to ``tavily``: a text that cites nothing is searched with
+#: the user's own Tavily key (OPEN-ITEMS 17.1a).
+SEARCH_SETTING = "search.provider tavily"
+```
+
+- [ ] **Step 4: Run the config tests**
+
+Run: `uv run pytest tests/test_config.py tests/test_config_cli.py -v`
+Expected: PASS. If a test pins `render_config`'s exact output or the list of sections, extend its expectation with the new `[search]` block and `web_search = "allow"`. Do not change the renderer.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add src/proofpath/config.py src/proofpath/settings_hints.py tests/test_config.py tests/test_config_cli.py
+```
+
+---
+
+### Task 2: JSON POST on `PoliteClient`, and Tavily's interval
+
+**Files:**
+- Modify: `src/proofpath/polite.py` (`MIN_INTERVAL`, `post`, `_send`)
+- Test: `tests/test_polite.py`
+
+**Interfaces:**
+- Produces: `PoliteClient.post(url, *, data=None, json_body: dict[str, Any] | None = None, headers=None, auth=None) -> httpx.Response`.
+
+- [ ] **Step 1: Write the failing test** (append to `tests/test_polite.py`)
+
+```python
+@respx.mock
+def test_post_sends_a_json_body_and_keeps_headers_out_of_the_url() -> None:
+ route = respx.post("https://api.test/search").mock(
+ return_value=httpx.Response(200, json={"results": []})
+ )
+ client = pl.PoliteClient(throttle=pl.HostThrottle())
+ response = client.post(
+ "https://api.test/search",
+ json_body={"query": "a claim", "max_results": 3},
+ headers={"Authorization": "Bearer secret"},
+ )
+ assert response.status_code == 200
+ sent = route.calls.last.request
+ assert json.loads(sent.content) == {"query": "a claim", "max_results": 3}
+ assert sent.headers["Authorization"] == "Bearer secret"
+ assert "secret" not in str(sent.url)
+
+
+def test_tavily_has_a_polite_interval() -> None:
+ assert pl.MIN_INTERVAL["api.tavily.com"] == 0.5
+```
+
+(Add `import json` to the imports if it is missing.)
+
+- [ ] **Step 2: Run it to see it fail**
+
+Run: `uv run pytest tests/test_polite.py -k "json_body or tavily" -v`
+Expected: FAIL with `TypeError: ... unexpected keyword argument 'json_body'`.
+
+- [ ] **Step 3: Implement**
+
+In `MIN_INTERVAL`, after `"oauth.reddit.com": 1.0,`:
+
+```python
+ # Evidence search (OPEN-ITEMS 17.1a): a run asks at most a handful of queries, and
+ # the user's own key is what they spend. SearXNG is the user's own instance, so it
+ # is not listed and gets the default.
+ "api.tavily.com": 0.5,
+```
+
+Change `post` to accept a JSON body, and pass it through `_send`:
+
+```python
+ def post(
+ self,
+ url: str,
+ *,
+ data: dict[str, str] | None = None,
+ json_body: dict[str, Any] | None = None,
+ headers: dict[str, str] | None = None,
+ auth: tuple[str, str] | None = None,
+ ) -> httpx.Response:
+ """A POST, throttled and retried like :meth:`get`.
+
+ Two callers: the OAuth token exchange a user's own Reddit app makes (spec
+ section 6.2), which sends a form in ``data``, and the evidence search, which
+ sends JSON in ``json_body``. A credential rides in ``auth`` or ``headers`` and
+ never in the URL: a query string is the part of a request that gets logged.
+ """
+ return self._send("POST", url, data=data, json_body=json_body, headers=headers, auth=auth)
+```
+
+In `_send`, add the parameter `json_body: dict[str, Any] | None = None` after `data`, and pass `json=json_body,` to `self._client.request(...)` after `data=data,`.
+
+- [ ] **Step 4: Run the polite tests**
+
+Run: `uv run pytest tests/test_polite.py -v`
+Expected: PASS.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add src/proofpath/polite.py tests/test_polite.py
+```
+
+---
+
+### Task 3: The `search` package (the protocol, Tavily, SearXNG, hit filtering)
+
+**Files:**
+- Create: `src/proofpath/search/__init__.py`
+- Create: `src/proofpath/search/providers.py`
+- Create: `tests/fixtures/search/tavily.json`
+- Create: `tests/fixtures/search/searxng.json`
+- Test: `tests/test_search.py`
+
+**Interfaces:**
+- Consumes: `PoliteClient.post(json_body=…)` (Task 2), `SearchConfig` (Task 1), `secrets.resolve_api_key`, `secrets.missing_hint`, `polite.ProviderError`, `providers.is_social`.
+- Produces:
+ - `search.SearchHit(url: str, title: str, rank: int)`, frozen.
+ - `search.Searcher` Protocol: `name: str`; `search(query: str, max_results: int) -> list[SearchHit]`, which raises `ProviderError` when the provider does not answer.
+ - `search.canonical(url: str) -> str` and `search.keep_hits(hits: Sequence[SearchHit], *, own_host: str | None, limit: int) -> list[SearchHit]`.
+ - `search.providers.TavilySearcher(client: PoliteClient, key: ApiKey)` and `search.providers.SearxngSearcher(client: PoliteClient, base_url: str)`.
+ - `search.providers.SearchSetup(searcher: Searcher | None, problem: str)` and `search.providers.build_searcher(config: SearchConfig, client: PoliteClient, *, resolve_key: Callable[[str], ApiKey | None] = resolve_api_key) -> SearchSetup`.
+ - `search.providers.TAVILY_URL = "https://api.tavily.com/search"` and `TAVILY_QUERY_LIMIT = 400`.
+
+- [ ] **Step 1: Write the fixtures**
+
+`tests/fixtures/search/tavily.json`:
+
+```json
+{
+ "query": "ChatGPT shut down 2025",
+ "results": [
+ {"title": "OpenAI statement", "url": "https://news.test/openai-statement?utm_source=x", "content": "SNIPPET-MUST-NOT-BE-EVIDENCE", "score": 0.91},
+ {"title": "Duplicate", "url": "https://news.test/openai-statement", "content": "dup", "score": 0.8},
+ {"title": "A post", "url": "https://x.com/someone/status/1", "content": "post", "score": 0.7},
+ {"title": "Encyclopedia", "url": "https://wiki.test/ChatGPT", "content": "wiki", "score": 0.6},
+ {"title": "No address", "content": "broken entry", "score": 0.5}
+ ],
+ "response_time": 0.8
+}
+```
+
+`tests/fixtures/search/searxng.json`:
+
+```json
+{
+ "query": "ChatGPT shut down 2025",
+ "number_of_results": 2,
+ "results": [
+ {"url": "https://wiki.test/ChatGPT", "title": "ChatGPT", "content": "wiki", "engine": "wikipedia"},
+ {"url": "ftp://files.test/x", "title": "Not the web", "content": "", "engine": "x"}
+ ]
+}
+```
+
+- [ ] **Step 2: Write the failing tests** in `tests/test_search.py`:
+
+```python
+"""Evidence search providers, offline: respx over fixtures, no socket (OPEN-ITEMS 17.1a)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+import respx
+
+from proofpath import polite as pl
+from proofpath.config import SearchConfig
+from proofpath.polite import ProviderError
+from proofpath.search import SearchHit, canonical, keep_hits
+from proofpath.search.providers import (
+ TAVILY_QUERY_LIMIT,
+ TAVILY_URL,
+ SearxngSearcher,
+ TavilySearcher,
+ build_searcher,
+)
+from proofpath.secrets import ApiKey
+
+FIX = Path(__file__).parent / "fixtures" / "search"
+SEARXNG = "http://localhost:8888"
+
+
+def fixture(name: str) -> Any:
+ return json.loads((FIX / name).read_text(encoding="utf-8"))
+
+
+def client() -> pl.PoliteClient:
+ return pl.PoliteClient(throttle=pl.HostThrottle(), retries=0)
+
+
+@respx.mock
+def test_tavily_sends_the_key_as_a_bearer_header_and_reads_addresses_only() -> None:
+ route = respx.post(TAVILY_URL).mock(
+ return_value=httpx.Response(200, json=fixture("tavily.json"))
+ )
+ searcher = TavilySearcher(client(), ApiKey("tvly-secret", source="test"))
+
+ hits = searcher.search("ChatGPT shut down 2025", 5)
+
+ sent = route.calls.last.request
+ assert sent.headers["Authorization"] == "Bearer tvly-secret"
+ body = json.loads(sent.content)
+ assert body["query"] == "ChatGPT shut down 2025"
+ assert body["max_results"] == 5
+ assert body["include_raw_content"] is False
+ # The entry with no address is dropped; nothing of ``content`` survives (rule 1).
+ assert [hit.url for hit in hits] == [
+ "https://news.test/openai-statement?utm_source=x",
+ "https://news.test/openai-statement",
+ "https://x.com/someone/status/1",
+ "https://wiki.test/ChatGPT",
+ ]
+ assert [hit.rank for hit in hits] == [1, 2, 3, 4]
+ assert "SNIPPET" not in repr(hits)
+
+
+@respx.mock
+def test_tavily_cuts_a_long_query_to_its_limit() -> None:
+ route = respx.post(TAVILY_URL).mock(return_value=httpx.Response(200, json={"results": []}))
+ TavilySearcher(client(), ApiKey("k", source="test")).search("word " * 200, 3)
+ assert len(json.loads(route.calls.last.request.content)["query"]) <= TAVILY_QUERY_LIMIT
+
+
+@respx.mock
+def test_a_provider_that_refuses_is_a_provider_error_without_the_body() -> None:
+ respx.post(TAVILY_URL).mock(return_value=httpx.Response(401, json={"detail": "tvly-secret"}))
+ with pytest.raises(ProviderError) as caught:
+ TavilySearcher(client(), ApiKey("tvly-secret", source="test")).search("q", 3)
+ assert "tvly-secret" not in str(caught.value)
+
+
+@respx.mock
+def test_a_non_json_answer_is_a_provider_error() -> None:
+ respx.get(f"{SEARXNG}/search").mock(return_value=httpx.Response(200, text=""))
+ with pytest.raises(ProviderError, match="non-JSON"):
+ SearxngSearcher(client(), SEARXNG).search("q", 3)
+
+
+@respx.mock
+def test_searxng_asks_for_json_and_keeps_only_web_addresses() -> None:
+ route = respx.get(f"{SEARXNG}/search").mock(
+ return_value=httpx.Response(200, json=fixture("searxng.json"))
+ )
+ hits = SearxngSearcher(client(), SEARXNG + "/").search("ChatGPT shut down 2025", 3)
+ params = route.calls.last.request.url.params
+ assert params["format"] == "json"
+ assert params["q"] == "ChatGPT shut down 2025"
+ assert "mailto" not in params
+ assert hits == [SearchHit("https://wiki.test/ChatGPT", "ChatGPT", 1)]
+
+
+def test_canonical_drops_the_fragment_and_tracking_parameters() -> None:
+ assert canonical("HTTPS://News.Test/a?utm_source=x&id=2#top") == "https://news.test/a?id=2"
+ assert canonical("https://news.test") == "https://news.test/"
+
+
+def test_keep_hits_dedupes_and_drops_posts_and_the_documents_own_host() -> None:
+ hits = [
+ SearchHit("https://news.test/a?utm_source=x", "", 1),
+ SearchHit("https://news.test/a", "", 2),
+ SearchHit("https://x.com/u/status/1", "", 3),
+ SearchHit("https://www.blog.test/post", "", 4),
+ SearchHit("https://wiki.test/A", "", 5),
+ SearchHit("https://other.test/B", "", 6),
+ ]
+ kept = keep_hits(hits, own_host="blog.test", limit=2)
+ assert [hit.url for hit in kept] == ["https://news.test/a?utm_source=x", "https://wiki.test/A"]
+
+
+def test_build_searcher_is_off_by_default() -> None:
+ setup = build_searcher(SearchConfig(), client())
+ assert setup.searcher is None
+ assert setup.problem == ""
+
+
+def test_build_searcher_names_the_missing_key_and_never_a_value() -> None:
+ setup = build_searcher(SearchConfig(provider="tavily"), client(), resolve_key=lambda _: None)
+ assert setup.searcher is None
+ assert setup.problem == "set TAVILY_API_KEY in .env"
+
+
+def test_build_searcher_builds_tavily_with_a_key() -> None:
+ setup = build_searcher(
+ SearchConfig(provider="tavily"),
+ client(),
+ resolve_key=lambda _: ApiKey("tvly-SECRET", source="t"),
+ )
+ assert setup.searcher is not None and setup.searcher.name == "tavily"
+ assert "tvly-SECRET" not in repr(setup)
+
+
+def test_build_searcher_needs_a_searxng_address() -> None:
+ assert build_searcher(SearchConfig(provider="searxng"), client()).problem == (
+ "set search.base_url to your SearXNG instance"
+ )
+ built = build_searcher(SearchConfig(provider="searxng", base_url=SEARXNG), client())
+ assert built.searcher is not None and built.searcher.name == "searxng"
+```
+
+- [ ] **Step 3: Run them to see them fail**
+
+Run: `uv run pytest tests/test_search.py -v`
+Expected: FAIL with `ModuleNotFoundError: No module named 'proofpath.search'`.
+
+- [ ] **Step 4: Implement `src/proofpath/search/__init__.py`**
+
+```python
+"""Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a).
+
+A searcher turns a query into addresses and nothing else. What a page says is read by
+the fetch ladder, exactly like a cited source, so a provider's snippet can never
+become the passage a verdict rests on (product rule 1).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from proofpath.providers import is_social
+
+
+@dataclass(frozen=True)
+class SearchHit:
+ """One address a provider answered with, in its own order."""
+
+ url: str
+ title: str
+ rank: int # 1-based position in the provider's answer
+
+
+class Searcher(Protocol):
+ """A web search provider. ``search`` raises ``polite.ProviderError`` when the
+ provider does not answer; an empty list means it answered with nothing."""
+
+ name: str # "tavily" | "searxng": what the Searching stage is attributed to
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]: ...
+
+
+# Parameters that name the campaign a link came from, not the page it points at.
+_TRACKING = re.compile(r"^(utm_\w+|fbclid|gclid|mc_cid|mc_eid)$")
+
+
+def canonical(url: str) -> str:
+ """One spelling per page: lower-cased scheme and host, no fragment, no tracking."""
+ parts = urlsplit(url)
+ host = (parts.hostname or "").lower()
+ if parts.port:
+ host = f"{host}:{parts.port}"
+ query = urlencode(
+ [
+ (k, v)
+ for k, v in parse_qsl(parts.query, keep_blank_values=True)
+ if not _TRACKING.match(k)
+ ]
+ )
+ return urlunsplit((parts.scheme.lower(), host, parts.path or "/", query, ""))
+
+
+def _bare_host(url: str) -> str:
+ return (urlsplit(url).hostname or "").lower().removeprefix("www.")
+
+
+def keep_hits(hits: Sequence[SearchHit], *, own_host: str | None, limit: int) -> list[SearchHit]:
+ """The hits worth reading, in order: one per page, no posts, not the document itself.
+
+ A post is dropped because a platform post is a claim, not a source. X, the case
+ that brought this feature, cannot be read at all (spec section 3). The document's
+ own host is dropped so a page is never found to support itself.
+ """
+ own = (own_host or "").lower().removeprefix("www.")
+ kept: list[SearchHit] = []
+ seen: set[str] = set()
+ for hit in hits:
+ key = canonical(hit.url)
+ if key in seen or is_social(hit.url) or (own and _bare_host(hit.url) == own):
+ continue
+ seen.add(key)
+ kept.append(hit)
+ if len(kept) >= limit:
+ break
+ return kept
+```
+
+- [ ] **Step 5: Implement `src/proofpath/search/providers.py`**
+
+```python
+"""The two search providers of OPEN-ITEMS 17.1a: Tavily (a key) and SearXNG (an address)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from proofpath.config import SearchConfig
+from proofpath.polite import PoliteClient, ProviderError
+from proofpath.search import SearchHit, Searcher
+from proofpath.secrets import ApiKey, missing_hint, resolve_api_key
+
+TAVILY_URL = "https://api.tavily.com/search"
+# Tavily refuses a longer query (API reference, checked 2026-09-28).
+TAVILY_QUERY_LIMIT = 400
+_WEB = ("http://", "https://")
+SEARXNG_NEEDS_URL = "set search.base_url to your SearXNG instance"
+
+
+class TavilySearcher:
+ name = "tavily"
+
+ def __init__(self, client: PoliteClient, key: ApiKey) -> None:
+ self._client = client
+ self._key = key
+
+ def __repr__(self) -> str:
+ return f"TavilySearcher(key={self._key!r})" # ApiKey's repr never shows the value
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ response = self._client.post(
+ TAVILY_URL,
+ json_body={
+ "query": query[:TAVILY_QUERY_LIMIT],
+ "max_results": max_results,
+ "search_depth": "basic",
+ "include_answer": False,
+ "include_raw_content": False,
+ "include_images": False,
+ },
+ headers={"Authorization": f"Bearer {self._key.value}"},
+ )
+ return _hits(_payload(response, "api.tavily.com").get("results"), max_results)
+
+
+class SearxngSearcher:
+ name = "searxng"
+
+ def __init__(self, client: PoliteClient, base_url: str) -> None:
+ self._client = client
+ self._url = base_url.rstrip("/") + "/search"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ # ``format=json`` must be enabled in the instance's settings.yml; an instance
+ # that has not enabled it answers 403, which is reported, not guessed around.
+ response = self._client.get(self._url, {"q": query, "format": "json"}, mailto=False)
+ host = httpx.URL(self._url).host
+ return _hits(_payload(response, host).get("results"), max_results)
+
+
+def _payload(response: httpx.Response, host: str) -> dict[str, Any]:
+ """The answer's JSON object. The body is never quoted: a provider echoes keys."""
+ if response.status_code != 200:
+ raise ProviderError(f"HTTP {response.status_code}", response.status_code, None)
+ try:
+ body = response.json()
+ except ValueError:
+ raise ProviderError(
+ f"{host} answered with a non-JSON body", response.status_code, None
+ ) from None
+ if not isinstance(body, dict):
+ raise ProviderError(f"{host} answered with a non-JSON body", response.status_code, None)
+ return body
+
+
+def _hits(results: Any, limit: int) -> list[SearchHit]:
+ """Addresses and titles, in the provider's order. ``content`` is never read."""
+ hits: list[SearchHit] = []
+ if not isinstance(results, list):
+ return hits
+ for entry in results:
+ if not isinstance(entry, dict):
+ continue
+ url = entry.get("url")
+ if not isinstance(url, str) or not url.lower().startswith(_WEB):
+ continue
+ title = entry.get("title")
+ hits.append(
+ SearchHit(url=url, title=title if isinstance(title, str) else "", rank=len(hits) + 1)
+ )
+ if len(hits) >= limit:
+ break
+ return hits
+
+
+@dataclass(frozen=True)
+class SearchSetup:
+ """The run's searcher, or why a configured one could not be built.
+
+ ``problem`` is empty when search is simply off. A named provider that cannot run
+ (no key, no address) is reported rather than skipped (rule 2).
+ """
+
+ searcher: Searcher | None
+ problem: str
+
+
+def build_searcher(
+ config: SearchConfig,
+ client: PoliteClient,
+ *,
+ resolve_key: Callable[[str], ApiKey | None] = resolve_api_key,
+) -> SearchSetup:
+ if config.provider == "tavily":
+ key = resolve_key(config.api_key_env)
+ if key is None:
+ return SearchSetup(None, missing_hint(config.api_key_env))
+ return SearchSetup(TavilySearcher(client, key), "")
+ if config.provider == "searxng":
+ if not config.base_url:
+ return SearchSetup(None, SEARXNG_NEEDS_URL)
+ return SearchSetup(SearxngSearcher(client, config.base_url), "")
+ return SearchSetup(None, "")
+```
+
+Check `ProviderError.__init__` in `polite.py` before this step. It is raised as `ProviderError(last, status, code)`. If its parameters are keyword-only or named differently, match them.
+
+- [ ] **Step 6: Run the tests**
+
+Run: `uv run pytest tests/test_search.py -v`
+Expected: PASS.
+
+- [ ] **Step 7: Stage**
+
+```bash
+git add src/proofpath/search tests/fixtures/search tests/test_search.py
+```
+
+---
+
+### Task 4: Check-worthy sentences (`claims.checkworthy`)
+
+**Files:**
+- Modify: `src/proofpath/claims.py` (next to `pair_links`)
+- Test: `tests/test_claims.py`
+
+**Interfaces:**
+- Produces:
+ - `claims.is_checkworthy(text: str) -> bool`.
+ - `claims.Checkworthy(claims: tuple[Claim, ...], eligible: int, sentences: int)`, frozen.
+ - `claims.checkworthy(doc: Document, *, limit: int) -> Checkworthy`. The claims it returns have `cited_refs=()` and `marker=None`.
+
+- [ ] **Step 1: Write the failing tests** (append to `tests/test_claims.py`; import `from proofpath import ingest` if it is missing)
+
+```python
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("ChatGPT was shut down yesterday.", True), # a name with an inner capital
+ ("The unemployment rate fell to 3.9% in 2023.", True), # a figure
+ ("Istanbul is the largest city in Turkey.", True), # a name past the first word
+ ('He said "we will never raise taxes" on stage.', True), # a quotation
+ ("I think this is really bad for everyone.", False), # opinion, nothing to find
+ ("Did Tesla really sell 2 million cars?", False), # a question
+ ("It rose 5%.", False), # too short to state anything searchable
+ ],
+)
+def test_is_checkworthy(text: str, expected: bool) -> None:
+ assert claims.is_checkworthy(text) is expected
+
+
+def test_checkworthy_keeps_document_order_up_to_the_cap_and_counts_the_rest() -> None:
+ doc = ingest.from_text(
+ "NASA landed on the Moon in 1969. I love this so much honestly. "
+ "Paris is the capital of France. The rate was 4% in 2020 overall.",
+ name="pasted text",
+ kind="text",
+ )
+ found = claims.checkworthy(doc, limit=2)
+
+ assert [claim.text for claim in found.claims] == [
+ "NASA landed on the Moon in 1969.",
+ "Paris is the capital of France.",
+ ]
+ assert found.eligible == 3
+ assert found.sentences == 4
+ assert all(claim.cited_refs == () and claim.marker is None for claim in found.claims)
+```
+
+(Adjust the `claims` alias to how the test module already imports the module; it may be `from proofpath import claims` or `claims_mod`.)
+
+- [ ] **Step 2: Run them to see them fail**
+
+Run: `uv run pytest tests/test_claims.py -k checkworthy -v`
+Expected: FAIL with `AttributeError: module 'proofpath.claims' has no attribute 'is_checkworthy'`.
+
+- [ ] **Step 3: Implement** (in `claims.py`, after `pair_links`; `dataclass` is already imported there because `Claims` is one, so check)
+
+```python
+# What makes a sentence worth searching for (OPEN-ITEMS 17.1a): long enough to state
+# something, not a question, and carrying something a page could confirm -- a figure,
+# a name, or a quotation. Opinion ("I think this is bad") has nothing to find.
+CHECKWORTHY_MIN_WORDS = 5
+_WORDS = re.compile(r"\w+")
+_FIGURE = re.compile(r"\d")
+_QUOTED = re.compile(r"[\"“”«»]")
+
+
+@dataclass(frozen=True)
+class Checkworthy:
+ """The sentences a search will be run for, and what the cap left out."""
+
+ claims: tuple[Claim, ...] # in document order, at most the cap
+ eligible: int # check-worthy sentences before the cap
+ sentences: int # every sentence with a word in it: the coverage denominator
+
+
+def is_checkworthy(text: str) -> bool:
+ stripped = text.strip()
+ if stripped.endswith("?"):
+ return False
+ words = _WORDS.findall(stripped)
+ if len(words) < CHECKWORTHY_MIN_WORDS:
+ return False
+ if _FIGURE.search(stripped) or _QUOTED.search(stripped):
+ return True
+ # A capital past the first word is a name ("Turkey"); a capital inside a word is
+ # one too ("ChatGPT", "NASA"), even as the sentence's first word.
+ return any(word[:1].isupper() for word in words[1:]) or any(
+ any(char.isupper() for char in word[1:]) for word in words
+ )
+
+
+def checkworthy(doc: Document, *, limit: int) -> Checkworthy:
+ """Claims for a document that cites nothing, to be searched for.
+
+ They cite no reference yet. The Searching stage points each at the pages it finds,
+ and a claim it finds nothing for keeps an empty ``cited_refs`` and is reported
+ as NO EVIDENCE FOUND, never dropped (product rule 6).
+ """
+ kept: list[Claim] = []
+ eligible = 0
+ sentences = 0
+ for paragraph in doc.paragraphs:
+ for position, sentence in enumerate(paragraph.sentences):
+ text = strip_links(sentence.text)
+ if not _WORD.search(text):
+ continue
+ sentences += 1
+ if not is_checkworthy(text):
+ continue
+ eligible += 1
+ if len(kept) < limit:
+ kept.append(
+ Claim(
+ text=text,
+ locator=sentence.locator,
+ cited_refs=(),
+ paragraph=paragraph.index,
+ sentence=position,
+ )
+ )
+ return Checkworthy(claims=tuple(kept), eligible=eligible, sentences=sentences)
+```
+
+If `ingest.from_text` splits the test paragraph into sentences differently than expected, check `ingest`'s sentence splitter. Adjust the **test input** (for example, put each sentence on its own line), not the splitter.
+
+- [ ] **Step 4: Run the claims tests**
+
+Run: `uv run pytest tests/test_claims.py -v`
+Expected: PASS.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add src/proofpath/claims.py tests/test_claims.py
+```
+
+#### Amendment A (2026-09-28): review fixes, decided with the user
+
+The heuristic above was revised in a fix round; the authoritative code is the one in the
+tree. Summary of the changes:
+- `#tags` and `@handles` are stripped before counting.
+- A word keeps its apostrophe suffix ("Türkiye'nin").
+- The minimum drops to 3 words.
+- A capitalised first word counts as a name unless it is a common English or Turkish opener.
+- When **no** sentence qualifies, every sentence is taken, up to the cap, and `eligible` then equals `sentences`. This is why "OpenAI battı" is searched.
+
+---
+
+### Task 5: Queries (the sentence text, `Judge.queries`, the LLM fallback notice)
+
+**Files:**
+- Create: `src/proofpath/prompts/queries.md`
+- Create: `src/proofpath/search/queries.py`
+- Modify: `src/proofpath/judge.py` (`Judge.__init__`, a new `Judge.queries`, `_QUERIES_SCHEMA`, `_queries_from`)
+- Test: `tests/test_search_queries.py`
+
+**Interfaces:**
+- Consumes: `Judge`, `JudgeUnavailable`, `Completion`, and `JudgeCost` from `judge.py`.
+- Produces:
+ - `Judge.queries(claims: Sequence[str]) -> dict[int, tuple[str, ...]]`. It never raises; on trouble it sets `unavailable` and `detail`.
+ - `search.queries.QUERY_LIMIT = 400`, `SENTENCE_BY = "sentence"`, `LLM_LIMIT = "LLM limit reached — searched with the sentence text"`, `LLM_UNAVAILABLE = "LLM did not answer — searched with the sentence text"`.
+ - `search.queries.sentence_query(text: str) -> str`.
+ - `search.queries.QueryPlan(queries: tuple[tuple[str, ...], ...], by: str, notice: str | None)`.
+ - `search.queries.plan_queries(texts: Sequence[str], judge: Judge | None) -> QueryPlan`.
+
+- [ ] **Step 1: Write the prompt** `src/proofpath/prompts/queries.md`
+
+```markdown
+## System
+
+You write web search queries for a fact-checking tool. You never say whether a claim
+is true: another stage reads the pages your queries find and decides that from their
+own words. Your only job is queries that would find a page stating the facts the
+claim is about.
+
+## User
+
+For each claim below, write one or two search queries.
+
+Rules:
+
+- Make each query stand alone: replace pronouns and vague references ("he", "the
+ company", "yesterday") with what the claim itself names, when it names it.
+- Keep the claim's names, numbers and dates; drop opinion words and emotion.
+- Plain keywords or a short sentence, at most 25 words, in the claim's language.
+- Add no fact, answer or source that is not in the claim.
+
+Answer with JSON only: {"items": [{"id": , "queries": ["...", "..."]}]}
+
+Claims:
+
+$items
+```
+
+- [ ] **Step 2: Write the failing tests** in `tests/test_search_queries.py`:
+
+```python
+"""Query building for the evidence search: the sentence, the judge, and the fallback."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from proofpath.judge import Completion, Judge, JudgeCost, JudgeUnavailable
+from proofpath.search.queries import (
+ LLM_LIMIT,
+ LLM_UNAVAILABLE,
+ QUERY_LIMIT,
+ SENTENCE_BY,
+ plan_queries,
+ sentence_query,
+)
+
+
+class ScriptedClient:
+ """``JudgeClient`` without a socket: one scripted answer or exception."""
+
+ provider = "fake"
+ model = "q-1"
+
+ def __init__(self, answer: str | Exception) -> None:
+ self.answer = answer
+ self.cost = JudgeCost(model=self.model)
+ self.prompts: list[str] = []
+
+ def complete(self, messages: list[dict[str, str]], **_: Any) -> Completion:
+ self.prompts.append(messages[-1]["content"])
+ if isinstance(self.answer, Exception):
+ raise self.answer
+ return Completion(text=self.answer, prompt_tokens=1, completion_tokens=1, model=self.model)
+
+ def close(self) -> None:
+ pass
+
+
+def judge(answer: str | Exception) -> Judge:
+ return Judge(ScriptedClient(answer)) # type: ignore[arg-type]
+
+
+def test_sentence_query_strips_links_handles_hashtag_signs_and_emoji() -> None:
+ text = "🚨 @newsbot says #ChatGPT was shut down https://t.co/xyz today!"
+ assert sentence_query(text) == "says ChatGPT was shut down today!"
+
+
+def test_sentence_query_cuts_at_a_word_boundary() -> None:
+ query = sentence_query("word " * 200)
+ assert len(query) <= QUERY_LIMIT
+ assert not query.endswith(" ")
+
+
+def test_without_a_judge_the_sentence_is_the_query() -> None:
+ plan = plan_queries(["ChatGPT was shut down in 2025."], None)
+ assert plan.queries == (("ChatGPT was shut down in 2025.",),)
+ assert plan.by == SENTENCE_BY
+ assert plan.notice is None
+
+
+def test_the_judge_writes_decontextualised_queries() -> None:
+ answer = json.dumps({"items": [{"id": 0, "queries": ["OpenAI ChatGPT shutdown 2025"]}]})
+ plan = plan_queries(["He shut it down in 2025."], judge(answer))
+ assert plan.queries == (("OpenAI ChatGPT shutdown 2025",),)
+ assert plan.by == "fake q-1"
+ assert plan.notice is None
+
+
+def test_a_rate_limited_judge_falls_back_to_the_sentence_and_says_so() -> None:
+ plan = plan_queries(
+ ["ChatGPT was shut down in 2025."],
+ judge(JudgeUnavailable("HTTP 429 from https://api.test/v1/chat/completions")),
+ )
+ assert plan.queries == (("ChatGPT was shut down in 2025.",),)
+ assert plan.by == SENTENCE_BY
+ assert plan.notice is not None and plan.notice.startswith(LLM_LIMIT)
+
+
+def test_a_judge_that_is_down_falls_back_with_its_own_words() -> None:
+ plan = plan_queries(["ChatGPT was shut down in 2025."], judge(JudgeUnavailable("HTTP 503")))
+ assert plan.notice is not None and plan.notice.startswith(LLM_UNAVAILABLE)
+
+
+def test_a_claim_the_judge_skipped_keeps_its_sentence() -> None:
+ answer = json.dumps({"items": [{"id": 1, "queries": ["Paris capital France"]}]})
+ plan = plan_queries(["NASA landed in 1969.", "Paris is the capital."], judge(answer))
+ assert plan.queries == (("NASA landed in 1969.",), ("Paris capital France",))
+
+
+def test_judge_queries_never_raises_on_an_unreadable_answer() -> None:
+ built = judge("not json at all")
+ assert built.queries(["a claim"]) == {}
+ assert built.unavailable is False
+```
+
+- [ ] **Step 3: Run them to see them fail**
+
+Run: `uv run pytest tests/test_search_queries.py -v`
+Expected: FAIL with `ModuleNotFoundError: No module named 'proofpath.search.queries'`.
+
+- [ ] **Step 4: Implement `Judge.queries`** in `judge.py`
+
+Add near `_REVIEW_SCHEMA`:
+
+```python
+# Evidence search (OPEN-ITEMS 17.1a): the judge writes *queries*, never an answer.
+# Deciding a claim without a passage is the truth oracle spec section 3 forbids.
+_QUERIES_TOKENS = 1500
+_QUERIES_PER_CLAIM = 2
+_QUERY_CHARS = 400
+_QUERIES_SCHEMA: dict[str, Any] = {
+ "type": "object",
+ "properties": {
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {"type": "integer"},
+ "queries": {"type": "array", "items": {"type": "string"}},
+ },
+ "required": ["id", "queries"],
+ "additionalProperties": False,
+ },
+ }
+ },
+ "required": ["items"],
+ "additionalProperties": False,
+}
+```
+
+In `Judge.__init__`, after `self._summary_template = …`:
+
+```python
+ self._queries_template = load_prompt("queries")
+```
+
+Add this method after `summarize`:
+
+```python
+ def queries(self, claims: Sequence[str]) -> dict[int, tuple[str, ...]]:
+ """One or two search queries per claim, keyed by its position; ``{}`` on trouble.
+
+ It never raises, like ``review``: a provider that is down or out of quota costs
+ the written queries, and ``unavailable``/``detail`` say why, so the run can fall
+ back to the sentence itself and tell the reader (OPEN-ITEMS 17.1a).
+ """
+ self.unavailable = False
+ self.detail = ""
+ if not claims:
+ return {}
+ items = "\n".join(
+ f"- id: {index}\n claim: {_one_line(text)}" for index, text in enumerate(claims)
+ )
+ system, user = _split_prompt(self._queries_template.substitute(items=items))
+ messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+ try:
+ completion = self._client.complete(
+ messages,
+ json_schema=_QUERIES_SCHEMA,
+ max_tokens=_QUERIES_TOKENS,
+ reasoning_effort=_REASONING_EFFORT,
+ )
+ except JudgeUnavailable as exc:
+ self._give_up(str(exc))
+ return {}
+ except Exception as exc:
+ # The type is named, the message is not: it can carry the key.
+ self._give_up(f"{type(exc).__name__} from the judge client")
+ return {}
+ return _queries_from(completion.text, len(claims))
+```
+
+After `_loads`, add:
+
+```python
+def _queries_from(text: str, count: int) -> dict[int, tuple[str, ...]]:
+ """The ``items`` of a queries answer, read leniently; an id out of range is dropped."""
+ for candidate in _json_candidates(text):
+ try:
+ payload = json.loads(candidate)
+ except ValueError:
+ continue
+ if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
+ continue
+ found: dict[int, tuple[str, ...]] = {}
+ for entry in payload["items"]:
+ if not isinstance(entry, dict):
+ continue
+ try:
+ ident = int(entry.get("id"))
+ except (TypeError, ValueError):
+ continue
+ raw = entry.get("queries")
+ if not 0 <= ident < count or not isinstance(raw, list):
+ continue
+ written = tuple(
+ _one_line(query)[:_QUERY_CHARS]
+ for query in raw
+ if isinstance(query, str) and query.strip()
+ )[:_QUERIES_PER_CLAIM]
+ if written:
+ found[ident] = written
+ return found
+ return {}
+```
+
+- [ ] **Step 5: Implement `src/proofpath/search/queries.py`**
+
+```python
+"""What the evidence search asks: the sentence itself, or what the judge wrote.
+
+The default makes zero LLM calls (spec section 11). With ``--judge``, the judge
+rewrites each claim as a query that stands on its own. When it cannot -- a rate limit,
+an exhausted quota, an outage -- the sentence is used instead and the run says so,
+because a weaker query is a weaker search and the reader has to know which one they
+got (product rule 6).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from proofpath.judge import Judge
+
+QUERY_LIMIT = 400
+SENTENCE_BY = "sentence"
+LLM_LIMIT = "LLM limit reached — searched with the sentence text"
+LLM_UNAVAILABLE = "LLM did not answer — searched with the sentence text"
+
+_URL = re.compile(r"https?://\S+")
+_HANDLE = re.compile(r"(? str:
+ """The sentence as a query: no addresses, handles or emoji, at most 400 characters."""
+ cleaned = _EMOJI.sub(" ", _HANDLE.sub(" ", _URL.sub(" ", text)))
+ cleaned = " ".join(_HASH.sub(r"\1", cleaned).split())
+ if len(cleaned) <= QUERY_LIMIT:
+ return cleaned
+ cut = cleaned[:QUERY_LIMIT]
+ return cut.rsplit(" ", 1)[0] if " " in cut else cut
+
+
+@dataclass(frozen=True)
+class QueryPlan:
+ queries: tuple[tuple[str, ...], ...] # per claim, in claim order; never empty
+ by: str # SENTENCE_BY, or the judge's name
+ notice: str | None # set when the judge was asked and the sentence was used instead
+
+
+def fallback_notice(detail: str) -> str:
+ head = LLM_LIMIT if "HTTP 429" in detail else LLM_UNAVAILABLE
+ return f"{head} ({detail})"
+
+
+def plan_queries(texts: Sequence[str], judge: Judge | None) -> QueryPlan:
+ fallback = tuple((sentence_query(text),) for text in texts)
+ if judge is None or not texts:
+ return QueryPlan(fallback, SENTENCE_BY, None)
+ written = judge.queries(texts)
+ if judge.unavailable:
+ return QueryPlan(fallback, SENTENCE_BY, fallback_notice(judge.detail))
+ queries = tuple(written.get(index) or fallback[index] for index in range(len(texts)))
+ return QueryPlan(queries, judge.name, None)
+```
+
+- [ ] **Step 6: Run the tests**
+
+Run: `uv run pytest tests/test_search_queries.py tests/test_judge.py -v`
+Expected: PASS. Check that `prompts/queries.md` ships in the wheel: `grep -n "prompts" pyproject.toml`. If the prompts are listed file by file, add `queries.md` beside `review.md`.
+
+- [ ] **Step 7: Stage**
+
+```bash
+git add src/proofpath/prompts/queries.md src/proofpath/search/queries.py src/proofpath/judge.py tests/test_search_queries.py pyproject.toml
+```
+
+#### Amendment A (2026-09-28): claims that are not in English
+
+The models are English-only (`cross-encoder/nli-deberta-v3-base`, `bge-small-en-v1.5`). The user decided: a claim that does not look English is **translated by the judge** when there is one. The English text drives both the queries and the check. With no judge, or a judge that did not answer, the claim has no English text and Task 7 reports it as `UNVERIFIED (language not supported)`. This amendment changes the interfaces above:
+
+- **Produces (new):**
+ - `search.language.looks_english(text: str) -> bool`.
+ - `search.queries.QueryPlan` gains `hypotheses: tuple[str | None, ...]`: per claim, the English text the models check, or `None` when the claim is not English and nothing translated it.
+- **Changes:** `Judge.queries(claims) -> dict[int, JudgedClaim]`, where `judge.JudgedClaim(queries: tuple[str, ...], english: str)`, frozen. `english` is `""` when the model gave none.
+
+Create `src/proofpath/search/language.py`:
+
+```python
+"""Whether a claim reads as English, the only language the models were trained on.
+
+Deliberately small and one-sided: English is the default, and a claim is called
+something else only on positive evidence -- a letter outside ASCII (ç, ğ, ı, ö, ş,
+ü, Cyrillic, CJK...) or a common function word of another language. A wrong
+"English" costs a weaker check; a wrong "not English" would cost the claim its
+search without a judge, so the doubt goes to English.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Function words of languages a reader is likely to paste, none of them English words.
+_FOREIGN_WORDS = frozenset(
+ {
+ # Turkish
+ "ve", "bir", "bu", "için", "ile", "değil", "çok", "daha", "gibi", "olan",
+ "olarak", "ama", "şu", "mi", "mı", "mu", "mü", "da", "de", "ki", "yok", "var",
+ # Spanish / Portuguese / Italian
+ "el", "los", "las", "que", "y", "es", "por", "con", "una", "para", "del",
+ "não", "uma", "il", "della", "che", "sono",
+ # German / Dutch
+ "der", "das", "und", "ist", "nicht", "ein", "eine", "mit", "den", "het", "een",
+ # French
+ "le", "les", "et", "est", "une", "des", "du", "pas", "pour", "dans",
+ }
+) # fmt: skip
+_WORDS = re.compile(r"[^\W\d_]+")
+
+
+def looks_english(text: str) -> bool:
+ letters = [char for char in text if char.isalpha()]
+ if any(ord(char) > 127 for char in letters):
+ return False
+ return not any(word.casefold() in _FOREIGN_WORDS for word in _WORDS.findall(text))
+```
+
+In `judge.py`:
+- Add `JudgedClaim`, next to `JudgeItem`:
+
+ ```python
+ @dataclass(frozen=True)
+ class JudgedClaim:
+ """What the judge wrote for one claim: search queries, and the claim in English."""
+
+ queries: tuple[str, ...]
+ english: str # "" when the model gave none
+ ```
+- Extend `_QUERIES_SCHEMA`'s item with `"english": {"type": "string"}` and add `"english"` to `required`.
+- `_queries_from` returns `dict[int, JudgedClaim]`. Keep an entry when it has queries **or** a non-empty `english`. `english = _one_line(str(entry.get("english", "")))[:_QUERY_CHARS]`.
+- `queries()`'s return annotation and docstring change to match.
+
+In `prompts/queries.md`:
+- Change the user instruction to: "For each claim below, write one or two search queries **in English**, and the claim itself in English."
+- Add this rule: "- `english`: the claim in English, as close to word for word as English allows; if it is already English, repeat it unchanged. Add nothing, soften nothing."
+- The JSON line becomes: `Answer with JSON only: {"items": [{"id": , "english": "...", "queries": ["...", "..."]}]}`
+
+In `search/queries.py`:
+- `QueryPlan` gains `hypotheses: tuple[str | None, ...]` after `queries`.
+- `plan_queries` becomes:
+
+```python
+def plan_queries(texts: Sequence[str], judge: Judge | None) -> QueryPlan:
+ english = tuple(looks_english(text) for text in texts)
+ fallback = tuple((sentence_query(text),) for text in texts)
+ # An English claim is checked as written; another language only once translated.
+ as_written = tuple(text if eng else None for text, eng in zip(texts, english, strict=True))
+ if judge is None or not texts:
+ return QueryPlan(fallback, as_written, SENTENCE_BY, None)
+ written = judge.queries(texts)
+ if judge.unavailable:
+ return QueryPlan(fallback, as_written, SENTENCE_BY, fallback_notice(judge.detail))
+ queries = tuple(
+ (written[i].queries if i in written and written[i].queries else fallback[i])
+ for i in range(len(texts))
+ )
+ hypotheses = tuple(
+ texts[i] if english[i] else ((written[i].english or None) if i in written else None)
+ for i in range(len(texts))
+ )
+ return QueryPlan(queries, hypotheses, judge.name, None)
+```
+
+Tests to add in `tests/test_search_queries.py`. Answers now carry `english`. Update the existing judge answers to include `"english": ""`.
+
+```python
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("OpenAI shut down ChatGPT in 2025.", True),
+ ("NASA confirms water on Mars", True),
+ ("OpenAI battı.", False), # ı
+ ("OpenAI batti ve kapandi.", False), # ASCII Turkish, "ve"
+ ("El gobierno cerró la empresa.", False),
+ ("Die Firma ist pleite.", False), # "ist"
+ ],
+)
+def test_looks_english(text: str, expected: bool) -> None:
+ assert looks_english(text) is expected
+
+
+def test_an_english_claim_is_checked_as_written() -> None:
+ plan = plan_queries(["OpenAI shut down ChatGPT in 2025."], None)
+ assert plan.hypotheses == ("OpenAI shut down ChatGPT in 2025.",)
+
+
+def test_without_a_judge_a_turkish_claim_has_nothing_to_check() -> None:
+ plan = plan_queries(["OpenAI battı."], None)
+ assert plan.hypotheses == (None,)
+
+
+def test_the_judge_translates_a_turkish_claim_for_the_check_and_the_queries() -> None:
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": "OpenAI went bankrupt.", "queries": ["OpenAI bankruptcy"]}]}
+ )
+ plan = plan_queries(["OpenAI battı."], judge(answer))
+ assert plan.hypotheses == ("OpenAI went bankrupt.",)
+ assert plan.queries == (("OpenAI bankruptcy",),)
+
+
+def test_a_rate_limited_judge_leaves_a_turkish_claim_untranslated() -> None:
+ plan = plan_queries(
+ ["OpenAI battı."], judge(JudgeUnavailable("HTTP 429 from https://api.test"))
+ )
+ assert plan.hypotheses == (None,)
+ assert plan.notice is not None and plan.notice.startswith(LLM_LIMIT)
+```
+
+(Import `pytest` and `looks_english`. `src/proofpath/search/language.py` is a new file: `git add -N` it.)
+
+---
+
+### Task 6: The report model (origin, the two kinds, `SearchSummary`, renderers)
+
+**Files:**
+- Modify: `src/proofpath/document.py` (`Reference.origin`)
+- Modify: `src/proofpath/report.py`
+- Modify: `src/proofpath/sarif.py` (`RULE_DESCRIPTIONS`, `_run_properties`)
+- Modify: `tests/data/verify-report-golden.json` (regenerated)
+- Test: `tests/test_report.py`, `tests/test_sarif.py`
+
+**Interfaces:**
+- Produces:
+ - `document.Origin = Literal["author", "search"]` and `Reference.origin: Origin = "author"`.
+ - `Kind.EVIDENCE_FOUND = "evidence-found"`: level `note`, state `"SUPPORTED (found by proofpath)"`, asserting (needs a passage).
+ - `Kind.NO_EVIDENCE = "no-evidence-found"`: level `warning`, state `"NO EVIDENCE FOUND (searched)"`.
+ - `report.FOUND_BY_PROOFPATH = "FOUND BY PROOFPATH (not cited by the author)"`, `report.SEARCH_UNAVAILABLE = "UNVERIFIED (search unavailable)"`, `report.AVERITEC_SEARCH_SCORE = "not measured yet"`, and `report.search_experimental() -> str`.
+ - `report.SearchSummary(by, queries_by, sentences, eligible, searched, pages_found, pages_read, notices=())`.
+ - `Report.search: SearchSummary | None = None`.
+ - `report.search_lines(report: Report) -> tuple[str, ...]` and `Footer.search: tuple[str, ...] = ()`.
+
+- [ ] **Step 1: Write the failing tests** (append to `tests/test_report.py`, which already has `report_with(*findings)` to build a `Report`)
+
+```python
+def test_the_two_search_kinds_carry_their_section_15_words() -> None:
+ assert STATE_WORDS[Kind.EVIDENCE_FOUND] == "SUPPORTED (found by proofpath)"
+ assert LEVELS[Kind.EVIDENCE_FOUND] == "note"
+ assert STATE_WORDS[Kind.NO_EVIDENCE] == "NO EVIDENCE FOUND (searched)"
+ assert LEVELS[Kind.NO_EVIDENCE] == "warning"
+
+
+def test_evidence_found_needs_a_passage() -> None:
+ with pytest.raises(ValueError, match="passage"):
+ Finding(
+ kind=Kind.EVIDENCE_FOUND, level="note", locator=Locator(line=1),
+ title="t", state=STATE_WORDS[Kind.EVIDENCE_FOUND], reference=None, claim=None,
+ verdict=None, source_id=None, fetch_step=None, tier=None,
+ ) # fmt: skip
+
+
+def test_a_reference_is_the_authors_unless_the_search_found_it() -> None:
+ assert Reference(1, "x", Locator(line=1)).origin == "author"
+ assert Reference(1, "x", Locator(line=1), origin="search").origin == "search"
+
+
+SUMMARY = SearchSummary(
+ by="tavily", queries_by="sentence", sentences=7, eligible=4, searched=3,
+ pages_found=9, pages_read=6, notices=("LLM limit reached — searched with the sentence text",),
+) # fmt: skip
+
+
+def test_search_lines_state_what_was_searched_and_what_was_not() -> None:
+ lines = search_lines(replace(report_with(), search=SUMMARY))
+ assert lines[0] == search_experimental()
+ assert "evidence search by tavily, queries by sentence" in lines
+ assert "claims searched 3 of 4 check-worthy (7 sentences)" in lines
+ assert "claims not searched 4" in lines
+ assert "pages found / read 9 / 6" in lines
+ assert lines[-1].startswith("LLM limit reached")
+
+
+def test_a_run_that_did_not_search_has_no_search_lines() -> None:
+ assert search_lines(report_with()) == ()
+ assert render_footer(report_with()).search == ()
+
+
+def test_the_footer_and_the_markdown_carry_the_search_block() -> None:
+ report = replace(report_with(), search=SUMMARY)
+ assert render_footer(report).search == search_lines(report)
+ markdown = render_markdown(report)
+ assert "## Evidence search" in markdown
+ assert "- claims not searched 4" in markdown
+```
+
+(Import `replace` from `dataclasses`, and `SearchSummary`, `search_lines`, `search_experimental`, `render_footer`, `render_markdown`, `LEVELS`, `STATE_WORDS`, `Finding`, `Kind` from `proofpath.report`, and `Reference`, `Locator` from `proofpath.document`. Add any that the module does not import yet.)
+
+Append to `tests/test_sarif.py`:
+
+```python
+def test_every_kind_has_a_rule_description() -> None:
+ assert set(RULE_DESCRIPTIONS) == set(Kind)
+
+
+def test_a_search_run_states_its_search_in_the_run_properties() -> None:
+ report = replace(report_with(), search=SearchSummary("tavily", "sentence", 2, 2, 2, 3, 1))
+ props = to_sarif(report, artifact="draft.md")["runs"][0]["properties"]
+ assert props["evidenceSearch"]["pagesRead"] == 1
+ assert props["evidenceSearch"]["experimental"] is True
+```
+
+(`report_with` is already imported from `tests.test_report`; add `replace`, `RULE_DESCRIPTIONS`, `to_sarif` and `SearchSummary` to the imports if they are missing.)
+
+Also add `Kind.EVIDENCE_FOUND` to the two `@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH])` lists in `tests/test_report.py`. They pin which kinds must carry a passage, and `test_an_asserting_finding_with_a_passage_is_valid` must then pass for it too.
+
+- [ ] **Step 2: Run them to see them fail**
+
+Run: `uv run pytest tests/test_report.py tests/test_sarif.py -v`
+Expected: FAIL with `AttributeError: EVIDENCE_FOUND` and import errors.
+
+- [ ] **Step 3: Implement `document.py`**
+
+```python
+# Who put a reference in the document: its author, or proofpath's evidence search
+# (OPEN-ITEMS 17.1a). Carried on the reference so every finding about a found page
+# can say so, whichever stage wrote it.
+Origin = Literal["author", "search"]
+
+
+@dataclass(frozen=True)
+class Reference:
+ """One bibliography entry, kept verbatim; resolve.py does the parsing."""
+
+ number: int # 1-based bibliography position, what "[12]" refers to
+ raw: str # the entry verbatim, whitespace collapsed
+ locator: Locator
+ origin: Origin = "author"
+```
+
+- [ ] **Step 4: Implement `report.py`**
+
+- Append to `Kind`, after `PROVIDER_UNAVAILABLE`:
+
+```python
+ EVIDENCE_FOUND = "evidence-found" # a page the search found supports the claim
+ NO_EVIDENCE = "no-evidence-found" # the search found nothing that decides the claim
+```
+
+- Add `Kind.NO_EVIDENCE: "warning",` to `LEVELS` beside the other warnings, and `Kind.EVIDENCE_FOUND: "note",` beside the notes.
+- Add to `STATE_WORDS`:
+
+```python
+ Kind.EVIDENCE_FOUND: "SUPPORTED (found by proofpath)",
+ Kind.NO_EVIDENCE: "NO EVIDENCE FOUND (searched)",
+```
+
+- Change `_ASSERTING` to `frozenset({Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND})`, and extend its comment: "EVIDENCE_FOUND asserts support, and support needs its passage just as much (rule 1)."
+- Below `UNVERIFIED_PREFIX`, add:
+
+```python
+# Evidence search (OPEN-ITEMS 17.1a). The provenance note every finding about a found
+# page carries, so no renderer can show one as a source the author cited.
+FOUND_BY_PROOFPATH = "FOUND BY PROOFPATH (not cited by the author)"
+SEARCH_UNAVAILABLE = f"{UNVERIFIED_PREFIX} (search unavailable)"
+# The §17.1 gate: printed on every run that searched until the eval beats the
+# majority baseline. Updated by hand from docs/eval/*-averitec-search.md.
+AVERITEC_SEARCH_SCORE = "not measured yet"
+AVERITEC_BASELINE = "0.708"
+
+
+def search_experimental() -> str:
+ return (
+ f"evidence search is experimental — AVeriTeC search-mode {AVERITEC_SEARCH_SCORE} "
+ f"vs {AVERITEC_BASELINE} majority baseline"
+ )
+```
+
+- After `Coverage`, add:
+
+```python
+@dataclass(frozen=True)
+class SearchSummary:
+ """What the evidence search did: the block beneath the coverage block (rule 6)."""
+
+ by: str # the searcher: "tavily" | "searxng"
+ queries_by: str # "sentence", or the judge that wrote them
+ sentences: int # sentences with a word in them
+ eligible: int # of those, check-worthy
+ searched: int # of those, actually searched
+ pages_found: int
+ pages_read: int # full text or abstract
+ notices: tuple[str, ...] = () # the LLM fallback, said once
+```
+
+- Add `search: SearchSummary | None = None` as the last field of `Report`, with the comment `# Set only on a run that searched (OPEN-ITEMS 17.1a).`
+- Add `search: tuple[str, ...] = ()` as the last field of `Footer`, with the comment `# ``search_lines``: empty unless the run searched.`, and set `search=search_lines(report),` in `render_footer`.
+- Add the functions:
+
+```python
+def search_lines(report: Report) -> tuple[str, ...]:
+ """The evidence-search block, one line each, for every surface to print as is."""
+ item = report.search
+ if item is None:
+ return ()
+ return (
+ search_experimental(),
+ f"evidence search by {item.by}, queries by {item.queries_by}",
+ f"claims searched {item.searched} of {item.eligible} check-worthy "
+ f"({item.sentences} sentences)",
+ f"claims not searched {item.sentences - item.searched}",
+ f"pages found / read {item.pages_found} / {item.pages_read}",
+ *item.notices,
+ )
+
+
+def _markdown_search(report: Report) -> list[str]:
+ lines = search_lines(report)
+ if not lines:
+ return []
+ return ["## Evidence search", "", *(f"- {line}" for line in lines), ""]
+```
+
+- In `render_markdown`, call `lines.extend(_markdown_search(report))` right after `lines.extend(_markdown_coverage(report))`.
+
+- [ ] **Step 5: Implement `sarif.py`**
+
+Add to `RULE_DESCRIPTIONS`:
+
+```python
+ Kind.EVIDENCE_FOUND: "a page proofpath found, not one the author cited, supports the claim",
+ Kind.NO_EVIDENCE: "the evidence search found no page that supports or contradicts the claim",
+```
+
+In `_run_properties`, add after `"cancelled": report.cancelled,`:
+
+```python
+ "evidenceSearch": None
+ if report.search is None
+ else {
+ "experimental": True,
+ "by": report.search.by,
+ "queriesBy": report.search.queries_by,
+ "sentences": report.search.sentences,
+ "eligible": report.search.eligible,
+ "searched": report.search.searched,
+ "pagesFound": report.search.pages_found,
+ "pagesRead": report.search.pages_read,
+ "notices": list(report.search.notices),
+ },
+```
+
+- [ ] **Step 6: Regenerate the golden report, and check that only additions changed**
+
+Look at how `tests/data/verify-report-golden.json` is indented (`head -5`), then match it in `indent=` below:
+
+```bash
+uv run python - <<'EOF'
+import json
+from proofpath.verify import verify
+from tests.test_verify_golden import BODY, CLOSED, GHOSTLY, GOLDEN_PATH, REAL, WEB, built_engine, draft, stable
+
+report = verify(draft(BODY, [REAL, GHOSTLY, WEB, CLOSED]), built_engine())
+GOLDEN_PATH.write_text(json.dumps(stable(report), indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+EOF
+git diff tests/data/verify-report-golden.json
+```
+
+Expected diff: only `+ "origin": "author"` inside each reference, and `+ "search": null` on the report. Anything else means the regeneration changed formatting. Revert and match the file's original formatting.
+
+- [ ] **Step 7: Run the report, SARIF, golden and UI tests**
+
+Run: `uv run pytest tests/test_report.py tests/test_sarif.py tests/test_verify_golden.py tests/test_ui.py tests/test_tui_rich.py -v`
+Expected: PASS. A test that iterates `for kind in Kind` and expects a colour, a style or a description per kind must be satisfied by adding the entry for the two new kinds where that test says, never by skipping them.
+
+- [ ] **Step 8: Stage**
+
+```bash
+git add src/proofpath/document.py src/proofpath/report.py src/proofpath/sarif.py tests/data/verify-report-golden.json tests/test_report.py tests/test_sarif.py
+```
+
+#### Amendment A (2026-09-28): the language state
+
+Add a constant next to `SEARCH_UNAVAILABLE` in `report.py`:
+
+```python
+# A claim in another language that no judge translated: the models read English only,
+# so it is not searched, and it is not called NEI either (OPEN-ITEMS 17.1a).
+LANGUAGE_UNSUPPORTED = f"{UNVERIFIED_PREFIX} (language not supported)"
+```
+
+Test it in `tests/test_report.py`:
+`assert LANGUAGE_UNSUPPORTED == "UNVERIFIED (language not supported)"`.
+
+---
+
+### Task 7: The Searching stage in `verify`
+
+**Files:**
+- Modify: `src/proofpath/verify.py`
+- Test: `tests/test_verify_search.py` (new); `tests/test_verify.py` (only if the hint wording is pinned)
+
+**Interfaces:**
+- Consumes: everything from Tasks 1–6.
+- Produces:
+ - `verify.SEARCHING = "Searching"`.
+ - `Engine.searcher: Searcher | None = None`, `Engine.search_problem: str = ""`, `Engine.search: bool = True`.
+ - `Engine.default(…, search: bool = True)`.
+ - `Prepared.search: SearchSummary | None = None`.
+ - `verify.NO_EVIDENCE_TITLE`, `verify.NO_EVIDENCE_NOTE`, `verify.EVIDENCE_FOUND_TITLE`, `verify.SEARCH_CANNOT_RUN`.
+
+- [ ] **Step 1: Write the failing tests** in `tests/test_verify_search.py`:
+
+```python
+"""The Searching stage end to end, offline (OPEN-ITEMS 17.1a, spec 2026-09-28).
+
+A stub searcher answers by substring, pages come from ``StubFetcher``, and both models
+are the fakes ``tests/test_verify.py`` uses. Nothing here opens a socket.
+"""
+
+from __future__ import annotations
+
+from proofpath import oa
+from proofpath.config import Config, Permissions, SearchConfig
+from proofpath.judge import Judge, JudgeUnavailable
+from proofpath.polite import ProviderError
+from proofpath.report import (
+ FOUND_BY_PROOFPATH,
+ SEARCH_UNAVAILABLE,
+ Kind,
+)
+from proofpath.search import SearchHit
+from proofpath.search.queries import LLM_LIMIT, sentence_query
+from proofpath.secrets import CREDENTIALS_MISSING
+from proofpath.verify import (
+ NO_SOURCE_HINT,
+ NO_SOURCE_IN_TEXT,
+ SEARCHING,
+ Engine,
+ prepare,
+ verify,
+)
+from tests.fakes import TableScorer, WordEmbedder
+from tests.test_verify import (
+ NEI_ROW,
+ SUPPORTED_ROW,
+ FakeJudgeClient,
+ StubFetcher,
+ engine,
+ fetched,
+ words,
+)
+
+CLAIM = "OpenAI shut down ChatGPT in March 2025."
+PAGE_A = "https://news.test/openai"
+PAGE_B = "https://wiki.test/ChatGPT"
+BACKING = "OpenAI shut down ChatGPT in March 2025 after a vote."
+FILLER = words(oa.FULLTEXT_MIN_WORDS + 10)
+
+
+class StubSearcher:
+ name = "stub"
+
+ def __init__(
+ self, answers: dict[str, list[str]] | None = None, *, error: ProviderError | None = None
+ ) -> None:
+ self.answers = answers or {}
+ self.error = error
+ self.queries: list[str] = []
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ self.queries.append(query)
+ if self.error is not None:
+ raise self.error
+ for key, urls in self.answers.items():
+ if key in query:
+ return [SearchHit(url, "", rank) for rank, url in enumerate(urls, start=1)]
+ return []
+
+
+def searching(
+ searcher: StubSearcher | None,
+ *,
+ pages: dict[str, str] | None = None,
+ table: dict[str, tuple[float, float, float]] | None = None,
+ config: Config | None = None,
+ network_allowed: bool = True,
+) -> Engine:
+ fetcher = StubFetcher(
+ {url: fetched(url, text) for url, text in (pages or {}).items()},
+ network_allowed=network_allowed,
+ network_note="" if network_allowed else "network not permitted",
+ )
+ built = engine(
+ fetcher=fetcher,
+ config=config,
+ embedder=WordEmbedder(),
+ scorer=TableScorer(table or {BACKING: SUPPORTED_ROW}),
+ )
+ built.searcher = searcher
+ return built
+
+
+def test_a_bare_claim_is_searched_and_the_backing_page_is_reported_as_found() -> None:
+ searcher = StubSearcher({"ChatGPT": [PAGE_A]})
+ report = verify(CLAIM, searching(searcher, pages={PAGE_A: f"{BACKING} {FILLER}"}))
+
+ assert searcher.queries == [sentence_query(CLAIM)]
+ assert report.counts() == {Kind.EVIDENCE_FOUND: 1}
+ found = report.findings[0]
+ assert found.verdict is not None and found.verdict.passage is not None
+ assert found.verdict.passage.text == BACKING
+ assert found.detail[-1] == FOUND_BY_PROOFPATH
+ assert report.sources[0].reference.origin == "search"
+ assert report.sources[0].reference.raw == PAGE_A
+ assert SEARCHING in [stage.name for stage in report.stages]
+ assert report.search is not None
+ assert (report.search.searched, report.search.pages_found, report.search.pages_read) == (
+ 1,
+ 1,
+ 1,
+ )
+
+
+def test_a_search_that_finds_nothing_is_no_evidence_never_refuted() -> None:
+ report = verify(CLAIM, searching(StubSearcher()))
+
+ assert report.counts() == {Kind.NO_EVIDENCE: 1}
+ assert report.findings[0].claim is not None
+ assert report.findings[0].claim.text == CLAIM
+
+
+def test_an_unreadable_found_page_keeps_its_state_and_says_who_found_it() -> None:
+ report = verify(CLAIM, searching(StubSearcher({"ChatGPT": [PAGE_A]})))
+
+ assert report.counts() == {Kind.UNVERIFIED: 1, Kind.NO_EVIDENCE: 1}
+ unread = next(f for f in report.findings if f.kind is Kind.UNVERIFIED)
+ assert unread.detail[-1] == FOUND_BY_PROOFPATH
+
+
+def test_a_page_that_says_nothing_either_way_is_nei_and_no_evidence() -> None:
+ report = verify(
+ CLAIM,
+ searching(
+ StubSearcher({"ChatGPT": [PAGE_A]}),
+ pages={PAGE_A: f"{BACKING} {FILLER}"},
+ table={BACKING: NEI_ROW},
+ ),
+ )
+ assert Kind.NOT_SUPPORTED not in report.counts()
+ assert report.counts()[Kind.NO_EVIDENCE] == 1
+
+
+def test_a_provider_that_fails_is_reported_and_the_claim_is_not_called_empty() -> None:
+ searcher = StubSearcher(error=ProviderError("HTTP 503", 503, None))
+ report = verify(CLAIM, searching(searcher))
+
+ assert report.counts() == {Kind.UNVERIFIED: 1}
+ assert report.findings[0].state == SEARCH_UNAVAILABLE
+ assert report.search is not None and report.search.searched == 0
+
+
+def test_without_a_searcher_the_old_parse_error_stands_with_the_setup_hint() -> None:
+ ready = prepare(CLAIM, searching(None))
+ errors = [f for f in ready.findings if f.kind is Kind.PARSE_ERROR]
+ assert [f.title for f in errors] == [NO_SOURCE_IN_TEXT]
+ assert errors[0].detail == (NO_SOURCE_HINT,)
+ assert "search.provider" in NO_SOURCE_HINT
+
+
+def test_no_search_and_a_denied_network_search_nothing() -> None:
+ off = StubSearcher({"ChatGPT": [PAGE_A]})
+ built = searching(off)
+ built.search = False
+ prepare(CLAIM, built)
+ denied = StubSearcher({"ChatGPT": [PAGE_A]})
+ prepare(CLAIM, searching(denied, network_allowed=False))
+ assert off.queries == [] and denied.queries == []
+
+
+def test_a_configured_provider_that_cannot_run_says_why() -> None:
+ built = searching(None)
+ built.search_problem = "set TAVILY_API_KEY in .env"
+ ready = prepare(CLAIM, built)
+ [item] = ready.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == CREDENTIALS_MISSING
+ assert item.detail == ("set TAVILY_API_KEY in .env",)
+
+
+def test_a_text_that_cites_something_is_never_searched() -> None:
+ searcher = StubSearcher({"ChatGPT": [PAGE_A]})
+ prepare(f"{CLAIM} See {PAGE_B}", searching(searcher))
+ assert searcher.queries == []
+
+
+def test_the_cap_limits_the_search_and_the_rest_is_counted() -> None:
+ body = " ".join(f"Company{i} sold {i} million units in 2020." for i in range(7))
+ searcher = StubSearcher()
+ report = verify(body, searching(searcher, config=Config(search=SearchConfig(max_claims=5))))
+ assert len(searcher.queries) == 5
+ assert report.search is not None
+ assert (report.search.eligible, report.search.searched) == (7, 5)
+
+
+def test_a_rate_limited_judge_falls_back_to_the_sentence_and_the_report_says_so() -> None:
+ searcher = StubSearcher()
+ built = searching(searcher)
+ built.judge = Judge(FakeJudgeClient([JudgeUnavailable("HTTP 429 from https://api.test")])) # type: ignore[arg-type]
+ report = verify(CLAIM, built)
+ assert searcher.queries == [sentence_query(CLAIM)]
+ assert report.search is not None
+ assert report.search.notices[0].startswith(LLM_LIMIT)
+
+
+def test_ask_or_deny_turns_the_search_off_in_the_default_engine() -> None:
+ config = Config(
+ permissions=Permissions(web_search="ask"),
+ search=SearchConfig(provider="searxng", base_url="http://localhost:8888"),
+ )
+ with Engine.default(config, interactive=True, no_cache=True) as built:
+ assert built.searcher is not None
+ assert built.search is False
+ allowed = Config(search=SearchConfig(provider="searxng", base_url="http://localhost:8888"))
+ with Engine.default(allowed, interactive=False, no_cache=True) as built:
+ assert built.search is True
+```
+
+- [ ] **Step 2: Run them to see them fail**
+
+Run: `uv run pytest tests/test_verify_search.py -v`
+Expected: FAIL with `ImportError: cannot import name 'SEARCHING'`.
+
+- [ ] **Step 3: Implement `Engine` fields and `Engine.default`**
+
+Imports to add in `verify.py`:
+
+```python
+from proofpath.report import FOUND_BY_PROOFPATH, SEARCH_UNAVAILABLE, SearchSummary
+from proofpath.search import SearchHit, Searcher, canonical, keep_hits
+from proofpath.search.providers import build_searcher
+from proofpath.search.queries import plan_queries
+from proofpath.secrets import CREDENTIALS_MISSING
+from proofpath.settings_hints import SEARCH_SETTING
+```
+
+(Merge them into the existing `from proofpath.report import (…)` and `settings_hints` import lines.)
+
+Stage constant, beside the others: `SEARCHING = "Searching"`.
+
+`Engine` fields, after `escalate`:
+
+```python
+ # Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). ``None`` when
+ # no provider is configured; ``search_problem`` then says why a configured one
+ # could not be built (a missing key), and ``search`` is this run's own switch
+ # (``--no-search``, or ``permissions.web_search`` other than ``allow``).
+ searcher: Searcher | None = None
+ search_problem: str = ""
+ search: bool = True
+```
+
+`Engine.default`: add the parameter `search: bool = True`. After `client = PoliteClient(contact_email=email)` and its closer, add:
+
+```python
+ setup = build_searcher(config.search, client)
+```
+
+Then pass to `cls(...)`:
+
+```python
+searcher = (setup.searcher,)
+search_problem = (setup.problem,)
+# The search never prompts, so ``ask`` is ``deny`` here (rule 4).
+search = (search and config.permissions.web_search == "allow",)
+```
+
+- [ ] **Step 4: Rewrite the no-source message**
+
+```python
+# What the parsing stage says of a text that neither links nor cites anything: a bare
+# claim. Without a search provider there is nothing to check it against, and ending
+# with every stage at zero would read as a clean run (product rule 6). The hint names
+# the one setting that makes proofpath look for evidence itself (OPEN-ITEMS 17.1a).
+NO_SOURCE_IN_TEXT = "the text links or cites no source; nothing to verify against"
+NO_SOURCE_HINT = (
+ "proofpath checks a claim against the sources it cites; to have it search the web "
+ f"for evidence instead: proofpath config set {SEARCH_SETTING}, with TAVILY_API_KEY in .env"
+)
+# A provider is configured but cannot run (no key, no address). Reported as the
+# credential state, naming what to set -- never as a text that simply cites nothing.
+SEARCH_CANNOT_RUN = "the text cites no source, and the evidence search could not run"
+EVIDENCE_FOUND_TITLE = "a page proofpath found supports the claim"
+NO_EVIDENCE_TITLE = "no page proofpath found supports or contradicts the claim"
+NO_EVIDENCE_NOTE = "absence of evidence is not evidence against the claim"
+```
+
+- [ ] **Step 5: Decide in the parsing stage whether the run will search**
+
+Replace the `if document.kind in ("post", "page") and not document.references:` … `elif …:` block in `prepare` with:
+
+```python
+ sourceless = not document.references and (
+ document.kind in ("post", "page")
+ or (bool(document.paragraphs) and not claims_mod.find_markers(document))
+ )
+ will_search = (
+ sourceless
+ and bool(document.paragraphs)
+ and allowed
+ and engine.search
+ and engine.searcher is not None
+ )
+ if sourceless and not will_search:
+ if engine.search and engine.search_problem and allowed:
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ Locator(line=1),
+ SEARCH_CANNOT_RUN,
+ state=CREDENTIALS_MISSING,
+ detail=(engine.search_problem,),
+ )
+ )
+ elif document.kind in ("post", "page"):
+ # (keep the existing comment)
+ add(
+ _finding(
+ Kind.PARSE_ERROR,
+ Locator(line=1),
+ f"{document.kind} carries no links; nothing to verify against",
+ )
+ )
+ else:
+ # (keep the existing comment)
+ pasted = not _is_file(target)
+ add(
+ _finding(
+ Kind.PARSE_ERROR,
+ Locator(line=1),
+ NO_SOURCE_IN_TEXT
+ if pasted
+ else f"{document.kind} cites nothing; nothing to verify against",
+ detail=(NO_SOURCE_HINT,) if pasted else (),
+ )
+ )
+```
+
+A missing SearXNG address (`SEARXNG_NEEDS_URL`) is not a credential, but it is fixed the same way, with one setting. It shares `CREDENTIALS_MISSING`, so the report needs no new state (spec §4).
+
+- [ ] **Step 6: Add the Searching stage and `_search`**
+
+In `prepare`, right after `if not allowed and denied_note: emit(Note(denied_note))` and before `# --- 3. resolving`:
+
+```python
+ # --- 2b. searching (OPEN-ITEMS 17.1a) ------------------------------------
+ # Only for a document that cites nothing. Its claims are the check-worthy
+ # sentences, and its references are the pages found for them, each marked as
+ # found (``origin="search"``). From here on every stage runs exactly as it does
+ # for a cited source.
+ search: SearchSummary | None = None
+ if will_search:
+ found = _search(document, engine, add=add, emit=emit, check=check, opened=opened, closed=closed)
+ document, claims, search = found.document, found.claims, found.summary
+```
+
+Before `return Prepared(...)`, count what was read, and pass it:
+
+```python
+ if search is not None:
+ search = replace(
+ search,
+ pages_read=sum(1 for status in sources.values() if status.text_kind != "none"),
+ )
+```
+
+Add `search=search,` to `Prepared(...)`, and the field `search: SearchSummary | None = None` to `Prepared`, after `started`.
+
+Module-level helpers, placed after `_nothing_read` at the end of the file:
+
+```python
+@dataclass(frozen=True)
+class _Searched:
+ document: Document
+ claims: Claims
+ summary: SearchSummary
+
+
+def _search(
+ document: Document,
+ engine: Engine,
+ *,
+ add: Callable[[Finding], None],
+ emit: Listener,
+ check: Callable[[], None],
+ opened: Callable[[str, str], float],
+ closed: Callable[[str, str, str, float], None],
+) -> _Searched:
+ """Search for each check-worthy sentence and turn the kept hits into references.
+
+ A provider that fails is reported once and the search stops there: the claims it
+ did not reach count as not searched rather than as searched and empty, which
+ would read as "nothing out there" (product rule 2).
+ """
+ searcher = engine.searcher
+ assert searcher is not None # guarded by ``will_search``
+ settings = engine.config.search
+ began = opened(SEARCHING, searcher.name)
+ chosen = claims_mod.checkworthy(document, limit=settings.max_claims)
+ plan = plan_queries(
+ [claim.text for claim in chosen.claims], engine.judge if engine.escalate else None
+ )
+ if plan.notice:
+ emit(Note(plan.notice))
+ own = _own_host(document)
+ numbers: dict[str, int] = {}
+ references: list[Reference] = []
+ searched: list[Claim] = []
+ for index, (claim, queries) in enumerate(
+ zip(chosen.claims, plan.queries, strict=True), start=1
+ ):
+ hits: list[SearchHit] = []
+ try:
+ for query in queries:
+ hits.extend(searcher.search(query, settings.results_per_claim))
+ except ProviderError as exc:
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ claim.locator,
+ "the evidence search did not answer",
+ state=SEARCH_UNAVAILABLE,
+ detail=(f"{searcher.name}: {exc}",),
+ )
+ )
+ break
+ cited: list[int] = []
+ for hit in keep_hits(hits, own_host=own, limit=settings.results_per_claim):
+ key = canonical(hit.url)
+ if key not in numbers:
+ numbers[key] = len(references) + 1
+ references.append(Reference(numbers[key], hit.url, claim.locator, origin="search"))
+ cited.append(numbers[key])
+ searched.append(replace(claim, cited_refs=tuple(cited)))
+ emit(
+ Progress(
+ name=SEARCHING, done=index, total=len(chosen.claims), detail=claim.locator.label()
+ )
+ )
+ check()
+ closed(
+ SEARCHING,
+ searcher.name,
+ f"{len(searched)} of {chosen.eligible} claims, {len(references)} pages",
+ began,
+ )
+ summary = SearchSummary(
+ by=searcher.name,
+ queries_by=plan.by,
+ sentences=chosen.sentences,
+ eligible=chosen.eligible,
+ searched=len(searched),
+ pages_found=len(references),
+ pages_read=0, # filled in once the fetching stage has run
+ notices=(plan.notice,) if plan.notice else (),
+ )
+ return _Searched(
+ document=replace(document, references=tuple(references)),
+ claims=Claims(claims=tuple(searched), markers=(), unsupported=(), unresolved=()),
+ summary=summary,
+ )
+
+
+def _own_host(document: Document) -> str | None:
+ """The host of a page or post read as the document: never found to back itself."""
+ match = _BARE_URL.fullmatch(document.name.strip())
+ return urlsplit(match.group(0)).hostname if match else None
+```
+
+Add `from urllib.parse import urlsplit` to the imports. `markers=()` is deliberate: the author made no citation, and the report's marker count must not say they did.
+
+- [ ] **Step 7: Put provenance on every finding about a found page, and give a supported found page its own finding**
+
+```python
+def _provenance(reference: Reference | None, detail: tuple[str, ...]) -> tuple[str, ...]:
+ """A found page says so on every finding about it (OPEN-ITEMS 17.1a). Last, not
+ first: a numeric mismatch's reason is read back out of ``detail[0]``."""
+ if reference is not None and reference.origin == "search":
+ return (*detail, FOUND_BY_PROOFPATH)
+ return detail
+```
+
+In `_finding`, set `detail=_provenance(reference, detail),`.
+
+In `_verdict_finding`, replace everything from the opening `if verdict.label is Label.SUPPORTED:` down to just before `return Finding(` with:
+
+```python
+ found = status.reference.origin == "search"
+ source = "a page proofpath found" if found else "the cited source"
+ kind: Kind
+ detail: tuple[str, ...] = ()
+ if verdict.label is Label.SUPPORTED:
+ if not found:
+ return None
+ # The answer the user asked for: shown, with its passage (rule 1), where a
+ # supported *cited* claim stays silent.
+ kind = Kind.EVIDENCE_FOUND
+ title = EVIDENCE_FOUND_TITLE
+ elif verdict.label is Label.REFUTED and verdict.reason.startswith(NUMERIC_REASON):
+ # (keep the existing comment)
+ kind = Kind.NUMERIC_MISMATCH
+ title = f"claim contradicts {source}"
+ detail = (verdict.reason,)
+ elif verdict.label is Label.REFUTED:
+ kind = Kind.NOT_SUPPORTED
+ title = f"claim is not supported by {source}"
+ else:
+ kind = Kind.NEI
+ title = (
+ "a page proofpath found neither supports nor contradicts the claim"
+ if found
+ else "source neither supports nor contradicts the claim"
+ )
+```
+
+In the `return Finding(...)` that follows, change `detail=detail,` to `detail=_provenance(status.reference, detail),`. The author-side titles stay byte for byte as they are today ("claim contradicts the cited source", "claim is not supported by the cited source", "source neither supports nor contradicts the claim").
+
+- [ ] **Step 8: Report NO EVIDENCE FOUND per searched claim in `decide_all`**
+
+After the `for group, members in grouped.items(): add(...)` loop:
+
+```python
+ if prepared.search is not None and not cancelled:
+ # One line per searched claim that no found page decided. A cancelled run is
+ # left alone: an undecided claim there is unfinished, not unbacked.
+ backed = {
+ (result.claim.paragraph, result.claim.sentence)
+ for result in results
+ if result.verdict.label is not Label.NEI
+ }
+ for claim in prepared.claims.claims:
+ if (claim.paragraph, claim.sentence) not in backed:
+ add(_no_evidence(claim))
+```
+
+Also pass `search=prepared.search,` to `Report(...)`, and add:
+
+```python
+def _no_evidence(claim: Claim) -> Finding:
+ return Finding(
+ kind=Kind.NO_EVIDENCE,
+ level=LEVELS[Kind.NO_EVIDENCE],
+ locator=claim.locator,
+ title=NO_EVIDENCE_TITLE,
+ state=STATE_WORDS[Kind.NO_EVIDENCE],
+ reference=None,
+ claim=claim,
+ verdict=None,
+ source_id=None,
+ fetch_step=None,
+ tier=None,
+ detail=(NO_EVIDENCE_NOTE,),
+ )
+```
+
+- [ ] **Step 9: Run the search tests, then the whole verify suite**
+
+Run: `uv run pytest tests/test_verify_search.py -v`
+Expected: PASS.
+
+Run: `uv run pytest tests/test_verify.py tests/test_verify_golden.py tests/test_verify_routing.py -v`
+Expected: PASS. The old bare-claim tests still pass, because `engine()` builds no searcher. If one pins the old `NO_SOURCE_HINT` wording by value rather than by constant, update that expectation to the constant.
+
+- [ ] **Step 10: Stage**
+
+```bash
+git add src/proofpath/verify.py tests/test_verify_search.py tests/test_verify.py
+```
+
+#### Amendment A (2026-09-28): short text and claims that are not in English
+
+Task 4 now returns every sentence when none looks check-worthy, so "OpenAI battı" becomes one claim. Task 5 now gives `QueryPlan.hypotheses`: the English text to check for each claim, or `None`. Task 6 adds `report.LANGUAGE_UNSUPPORTED`. This amendment wires them in.
+
+**1. `document.Claim`** gets a field after `marker`:
+
+```python
+ # The English text the models check, when it is not ``text`` itself: a claim in
+ # another language that the judge translated (OPEN-ITEMS 17.1a). ``text`` stays
+ # the reader's own sentence, and that is what the report shows.
+ hypothesis: str | None = None
+```
+
+**2. In `_search`**, iterate over the hypotheses too, and skip an untranslated claim:
+
+```python
+for index, (claim, queries, hypothesis) in enumerate(
+ zip(chosen.claims, plan.queries, plan.hypotheses, strict=True), start=1
+):
+ if hypothesis is None:
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ claim.locator,
+ LANGUAGE_TITLE,
+ state=LANGUAGE_UNSUPPORTED,
+ detail=(LANGUAGE_HINT,),
+ )
+ )
+ emit(
+ Progress(
+ name=SEARCHING, done=index, total=len(chosen.claims), detail=claim.locator.label()
+ )
+ )
+ check()
+ continue
+ # … search as before …
+ searched.append(
+ replace(
+ claim,
+ cited_refs=tuple(cited),
+ hypothesis=None if hypothesis == claim.text else hypothesis,
+ )
+ )
+```
+
+Constants, next to the other titles:
+
+```python
+LANGUAGE_TITLE = "the claim is not in English and was not translated"
+LANGUAGE_HINT = "the models read English only; --judge translates a claim before it is searched"
+```
+
+Import `LANGUAGE_UNSUPPORTED` from `proofpath.report`. The untranslated claim is not appended to `searched`, so it gets no NO EVIDENCE FINDING, and the summary counts it as not searched.
+
+**3. The models check the hypothesis.**
+- In `_decide_claim`, use `checked = claim.hypothesis or claim.text` for both `cache_mod.claim_hash(...)` and `pipeline.decide_indexed(...)`.
+- In `_judging`, the `JudgeItem(claim=...)` becomes `result.claim.hypothesis or result.claim.text`.
+- Leave the other `claim_hash(result.claim.text)` / `claim_hash(item.claim.text)` pair in `_judging` / `_with_opinion` as it is: it only matches an opinion back to its result, and both sides use the same text.
+
+**4. A translated claim says what was checked.** In `_verdict_finding`, right before the `return Finding(...)`:
+
+```python
+ if claim.hypothesis:
+ detail = (*detail, f"{CHECKED_AS}{claim.hypothesis}")
+```
+
+with `CHECKED_AS = "checked as: "`. It is appended after the numeric reason, which must stay `detail[0]`, and before the provenance note, which `_provenance` appends last.
+
+**5. Tests**, appended to `tests/test_verify_search.py`:
+
+```python
+TURKISH = "OpenAI battı."
+ENGLISH = "OpenAI went bankrupt."
+BANKRUPT = "OpenAI went bankrupt in 2026, the court said."
+
+
+def test_a_one_line_post_is_searched_even_when_nothing_in_it_looks_checkworthy() -> None:
+ searcher = StubSearcher()
+ report = verify("OpenAI shut down", searching(searcher))
+ assert searcher.queries == ["OpenAI shut down"]
+ assert report.counts() == {Kind.NO_EVIDENCE: 1}
+
+
+def test_without_a_judge_a_turkish_claim_is_reported_not_searched() -> None:
+ searcher = StubSearcher({"OpenAI": [PAGE_A]})
+ report = verify(TURKISH, searching(searcher))
+ assert searcher.queries == []
+ [item] = report.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == LANGUAGE_UNSUPPORTED
+ assert report.search is not None and report.search.searched == 0
+
+
+def test_the_judge_translates_a_turkish_claim_and_the_report_shows_both() -> None:
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": ENGLISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ searcher = StubSearcher({"bankruptcy": [PAGE_A]})
+ built = searching(
+ searcher,
+ pages={PAGE_A: f"{BANKRUPT} {FILLER}"},
+ table={BANKRUPT: SUPPORTED_ROW},
+ )
+ built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type]
+ report = verify(TURKISH, built)
+ assert searcher.queries == ["OpenAI bankruptcy"]
+ found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND)
+ assert found.claim is not None and found.claim.text == TURKISH
+ assert f"checked as: {ENGLISH}" in found.detail
+ assert found.detail[-1] == FOUND_BY_PROOFPATH
+```
+
+(Import `json`, `LANGUAGE_UNSUPPORTED`. The judging stage runs after verifying with the same `FakeJudgeClient`. Its unscripted answers agree, so they change nothing, but check that `report.findings` holds no judge-unavailable line that breaks the `[item] =` destructuring in the no-judge test, where there is no judge at all.)
+
+---
+
+### Task 8: The surfaces (`--no-search`, footer lines, TUI footer and `/config`)
+
+**Files:**
+- Modify: `src/proofpath/cli.py` (`check`)
+- Modify: `src/proofpath/ui.py` (`footer`)
+- Modify: `src/proofpath/tui/widgets/footer.py` (`_hints`)
+- Modify: `src/proofpath/tui/widgets/config_panel.py` (`sections`, `_draw_summary`)
+- Test: `tests/test_check_cli.py`, `tests/test_ui.py`, and the TUI config panel test that pins `sections()`
+
+**Interfaces:**
+- Consumes: `Engine.default(search=…)` (Task 7), `Footer.search` (Task 6), `SearchConfig` and `SEARCH_PROVIDERS` (Task 1).
+
+- [ ] **Step 1: Write the failing tests**
+
+In `tests/test_check_cli.py` (it has `runner = CliRunner()` and imports `app`):
+
+```python
+def test_no_search_is_passed_to_the_engine(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ seen = install(monkeypatch)
+ path = write(tmp_path, "Transformers improved translation quality [1].")
+ runner.invoke(app, ["check", "--no-search", str(path)])
+ assert seen["search"] is False
+
+
+def test_search_is_on_unless_told_otherwise(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ seen = install(monkeypatch)
+ runner.invoke(
+ app, ["check", str(write(tmp_path, "Transformers improved translation quality [1]."))]
+ )
+ assert seen["search"] is True
+```
+
+
+In `tests/test_ui.py`, render a `Footer` with `search=("evidence search is experimental — …", "claims not searched 2")` through `ui.footer` into the module's captured console, and assert both lines appear, each after the key `search`. Follow the file's existing footer test for the console setup.
+
+- [ ] **Step 2: Run them to see them fail**
+
+Run: `uv run pytest tests/test_check_cli.py -k search tests/test_ui.py -k footer -v`
+Expected: FAIL with `KeyError: 'search'` and the missing lines.
+
+- [ ] **Step 3: Implement**
+
+`cli.py` `check`: add the option after `no_cache`:
+
+```python
+no_search: Annotated[
+ bool,
+ typer.Option(
+ "--no-search",
+ help="Do not search the web for evidence when the text cites no source.",
+ ),
+] = (False,)
+```
+
+and pass `search=not no_search,` to `verify_mod.Engine.default(...)`.
+
+`ui.py` `footer`: after `skipped(ui, item.browser_skipped)`:
+
+```python
+ # The evidence-search block (OPEN-ITEMS 17.1a): never suppressed, because a run
+ # that searched must not read like a run that checked cited sources (rule 6).
+ for line in item.search:
+ kv(ui, "search", line)
+```
+
+`tui/widgets/footer.py` `_hints`: before `if item.note:`, add:
+
+```python
+ # The docked footer has one hints row: the experimental banner and the searched
+ # line go there, and the full block is in the run's report.
+ hints.extend(item.search[:3])
+```
+
+`tui/widgets/config_panel.py`:
+
+- In `sections()`, build `search = SearchConfig()` beside the others.
+- Add a `permissions.web_search` choice row after `permissions.network`, with the meaning `"search the web when a text cites nothing; ask = deny (never prompts)"`.
+- Append a new section after `judge`, with rows following the `judge` section's text-row shape:
+ - `Row("search.provider", "choice", SEARCH_PROVIDERS, "off until set; tavily needs TAVILY_API_KEY in .env", _default(search.provider))`
+ - `Row("search.base_url", "text", (), "your SearXNG instance", _default(search.base_url))`
+ - `Row("search.api_key_env", "text", (), "the variable holding the key", _default(search.api_key_env))`
+- In `_draw_summary`, add `f"search={config.search.provider}",` after `judge=`.
+
+Import `SearchConfig` and `SEARCH_PROVIDERS` from `proofpath.config`.
+
+- [ ] **Step 4: Run the surface tests**
+
+Run: `uv run pytest tests/test_check_cli.py tests/test_ui.py tests/test_tui_app.py tests/test_tui_commands.py -v`
+Expected: PASS. A config-panel test that pins the row list or the summary line gets the new rows and `search=off` added to its expectation.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add src/proofpath/cli.py src/proofpath/ui.py src/proofpath/tui/widgets/footer.py src/proofpath/tui/widgets/config_panel.py tests/test_check_cli.py tests/test_ui.py tests/test_tui_*.py
+```
+
+---
+
+### Task 9: The AVeriTeC search-mode eval (the §17.1 gate)
+
+**Files:**
+- Modify: `src/proofpath/eval/averitec.py` (`Claim.fact_check_url`)
+- Modify: `scripts/eval_averitec.py` (`--search`)
+- Test: `tests/test_averitec.py`, `tests/test_eval_averitec.py`
+
+**Interfaces:**
+- Consumes: `build_searcher`, `keep_hits`, `sentence_query` (Tasks 3 and 5).
+- Produces:
+ - `averitec.Claim.fact_check_url: str = ""`.
+ - `eval_averitec.search_urls(searcher: Searcher, claim: averitec.Claim, limit: int) -> tuple[str, ...]`.
+
+- [ ] **Step 1: Confirm the dataset key**
+
+Run: `uv run python -c "import json; from proofpath.paths import cache_dir; p = cache_dir() / 'datasets'; f = next(p.glob('*dev*.json')); print(sorted(json.loads(f.read_text(encoding='utf-8'))[0]))"`
+Expected: a key list that includes `fact_checking_article`. If the key is named differently, use that name in Step 3. If the dataset is not downloaded, read the key list from the AVeriTeC README instead. Do not download it in a test.
+
+- [ ] **Step 2: Write the failing tests**
+
+`tests/test_averitec.py`: extend the existing inline dev-row fixture with `"fact_checking_article": "https://factcheck.test/claim-1"`, and assert `load(...)[0].fact_check_url == "https://factcheck.test/claim-1"`. Also assert that a row without the key loads as `""`.
+
+`tests/test_eval_averitec.py`:
+
+```python
+def test_search_urls_never_hand_back_the_fact_check_itself() -> None:
+ class Stub:
+ name = "stub"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ return [
+ SearchHit("https://factcheck.test/claim-1", "", 1),
+ SearchHit("https://factcheck.test/other", "", 2),
+ SearchHit("https://news.test/a", "", 3),
+ ]
+
+ claim = averitec.Claim(
+ id=0, text="A claim.", label="Refuted", source_urls=(),
+ fact_check_url="https://factcheck.test/claim-1",
+ ) # fmt: skip
+ assert eval_averitec.search_urls(Stub(), claim, 3) == ("https://news.test/a",)
+```
+
+(Import the script the way `test_eval_averitec.py` already does.)
+
+- [ ] **Step 3: Implement**
+
+`averitec.Claim`: add `fact_check_url: str = ""` after `non_urls`, with the comment `# The article that settled the claim: the answer key, never a search result.` In `load`, pass `fact_check_url=str(row.get("fact_checking_article") or ""),`.
+
+`scripts/eval_averitec.py`:
+
+```python
+from urllib.parse import urlsplit # already imported
+
+from proofpath.search import Searcher, keep_hits
+from proofpath.search.providers import build_searcher
+from proofpath.search.queries import sentence_query
+
+SEARCH_RESULTS_NAME = "averitec_search_results.json"
+
+
+def search_urls(searcher: Searcher, claim: averitec.Claim, limit: int) -> tuple[str, ...]:
+ """What the evidence search would read for this claim, minus the answer key.
+
+ The whole fact-checking site is excluded, not just the one article: a site that
+ rated the claim quotes its own rating on every related page (the leakage the
+ §17.1 gate must not measure).
+ """
+ checker = (urlsplit(claim.fact_check_url).hostname or "").removeprefix("www.")
+ hits = [
+ hit
+ for hit in searcher.search(sentence_query(claim.text), limit + 3)
+ if not checker or (urlsplit(hit.url).hostname or "").removeprefix("www.") != checker
+ ]
+ return tuple(hit.url for hit in keep_hits(hits, own_host=None, limit=limit))
+```
+
+In `main`:
+- Add `parser.add_argument("--search", action="store_true", help="read what the evidence search finds instead of the gold URLs")`.
+- Make `_results_path()` take `search: bool` and return `SEARCH_RESULTS_NAME` when it is set.
+- After the engine is built:
+
+```python
+searcher = None
+if args.search:
+ setup = build_searcher(config.search, engine.client)
+ if setup.searcher is None:
+ print(
+ f"error evidence search is not set up: {setup.problem or 'search.provider is off'}",
+ file=sys.stderr,
+ )
+ engine.close()
+ return 2
+ searcher = setup.searcher
+```
+
+- In the loop, when `searcher is not None`, decide `replace(claim, source_urls=search_urls(searcher, claim, config.search.results_per_claim), non_urls=0)`. Import `replace` from `dataclasses`.
+- The default `--out` becomes `…-averitec-search.md` under `--search`.
+- Update the module docstring's usage line.
+
+- [ ] **Step 4: Run the eval tests**
+
+Run: `uv run pytest tests/test_averitec.py tests/test_eval_averitec.py -v`
+Expected: PASS.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add src/proofpath/eval/averitec.py scripts/eval_averitec.py tests/test_averitec.py tests/test_eval_averitec.py
+```
+
+- [ ] **Step 6 (manual, needs the user's key; not a subagent step):** run `uv run python scripts/eval_averitec.py --search --limit 100 --no-browser --resume`, and copy the 3-way score into `report.AVERITEC_SEARCH_SCORE` (for example `"0.312"`).
+
+---
+
+### Task 10: Docs and the whole-suite check
+
+**Files:**
+- Modify: `README.md` (a new section, and the X row)
+- Modify: `src/proofpath/providers/social.py` (`X_HINT`)
+- Modify: `docs/superpowers/OPEN-ITEMS.md` (§17.1a status, §17.2 closed)
+
+- [ ] **Step 1: README.** After `## Posts and the links inside them (v0.4)`, add `## Claims with no source (experimental)`. Content:
+ - When it runs: a text with no link, marker or bibliography, and a provider set.
+ - The setup lines:
+ ```bash
+ proofpath config set search.provider tavily
+ echo 'TAVILY_API_KEY=tvly-…' >> .env
+ ```
+ or `search.provider searxng` plus `search.base_url`.
+ - The two states, `SUPPORTED (found by proofpath)` and `NO EVIDENCE FOUND (searched)`, and the `FOUND BY PROOFPATH (not cited by the author)` note.
+ - The cap (5 × 3), `--no-search`, the judge-written queries and their fallback notice.
+ - The experimental banner, with the AVeriTeC number.
+
+ Change the X row to: "cannot be read at all. Paste the text instead: a link inside it is verified normally, and with evidence search set up a text with no link is searched (experimental)."
+
+- [ ] **Step 1b: "What it reads" (added 2026-09-28).** Main gained a plain "What it reads" overview after this branch was cut: README `## What it reads` and the landing page's `site/src/data/reads.ts`. First rebase this branch on `main` (`git rebase main` in the worktree; the controller does this, not a subagent). Then add one row to each.
+ - README table, after the Sites-that-block-robots row: `| 🔑 | Claims with no source, searched on the web (experimental) | a free Tavily key, or your own SearXNG |`
+ - `reads.ts`, in the "Needs one thing from you" column, after the LLM item: `{ icon: 'search', name: 'Claims with no source', note: 'searched on the web: a free Tavily key, or your own SearXNG' }`. The `search` icon already exists in `SketchIcon.astro`.
+
+- [ ] **Step 2: `social.X_HINT`** → `"X cannot be read; paste the post text — its links are verified, and with search.provider set a text with no link is searched"`. Update the test that pins the old string by value, if one does.
+
+- [ ] **Step 3: OPEN-ITEMS.** Mark 17.1a with `**Implemented **` and the plan path. Close 17.2 with a strike-through and "fixed by the README X row and `X_HINT`, 2026-09-28".
+
+- [ ] **Step 4: The whole suite and the linters**
+
+Run: `uv run ruff check . && uv run ruff format --check . && uv run pytest -q`
+Expected: ruff clean, all tests pass. Paste the final summary line in the task report.
+
+- [ ] **Step 5: Stage**
+
+```bash
+git add README.md src/proofpath/providers/social.py docs/superpowers/OPEN-ITEMS.md tests/
+```
+
+- [ ] **Step 6 (manual):** try it by hand:
+ - `proofpath check -` with a real Tavily key, fed a real bare claim (for example "ChatGPT was released by OpenAI in November 2022.").
+ - The same text in the TUI.
+ - `--no-search`, which should give the old parse error.
+ - The key removed, which should give `UNVERIFIED (credentials missing)` naming `TAVILY_API_KEY`.
+
+---
+
+### Task 11: Local fallback judge (spec §11)
+
+**Files:**
+- Modify: `src/proofpath/config.py` (`JudgeConfig.fallback: str = "auto"`, `/config` coercion if needed)
+- Modify: `src/proofpath/judge.py` (model picker, fallback client, notice)
+- Modify: `src/proofpath/verify.py` (emit the switch as a `Note`, carry it into the report footer)
+- Modify: `src/proofpath/report.py` (footer line for the notice)
+- Modify: `src/proofpath/cli.py`, `src/proofpath/tui/app.py` (build the fallback-capable judge; pass through only)
+- Create: `src/proofpath/tui/widgets/judge_notice.py` (right-aligned one-line notice above the prompt)
+- Modify: `src/proofpath/settings_hints.py` (the `judge.fallback` hint), the TUI `/config` panel entry
+- Test: `tests/test_judge_fallback.py`, plus the TUI test module that covers `#bottom`
+
+**Interfaces:**
+- Produces:
+ - `judge.pick_local_model(names: Sequence[str]) -> str | None`. This is the pure selection rule of spec §11.
+ - `judge.installed_models(base_url: str, *, timeout: float) -> list[str]`: `GET {root}/api/tags`, where `root` is the Ollama base URL without `/v1`. Returns `[]` on any error; the error never raises.
+ - `judge.FallbackClient`. It has the same public surface `Judge` uses from `JudgeClient`: `complete`, `provider`, `model`, `cost`, `close`, and the context manager.
+ - It wraps a primary `JudgeClient` plus a factory for the local client, which is built lazily on the first failure.
+ - After a switch, `provider` and `model` report the local model.
+ - `switched: str | None` holds the notice text once a switch has happened.
+ - `on_switch: Callable[[str], None] | None` is called once, at the switch.
+ - `JUDGE_FALLBACK_LIMIT = "{provider} limit reached — judging with local ollama {model}"` and `JUDGE_FALLBACK_DOWN = "{provider} did not answer — judging with local ollama {model}"`. The wording is verbatim; the provider name is capitalised as displayed (`Groq`).
+
+- [ ] **Step 1: Tests first** (respx, no network):
+ - Model choice:
+ - `pick_local_model` over the user's real list returns `"qwen3.6:35b-a3b"`. The list is `qwen3-embedding:8b`, `nomic-embed-text:latest`, `qwen3-embedding:0.6b`, `gemma4:12b`, `qwen3.5:9b`, `qwen3.6:35b-a3b`, `qwen3:8b`, `qwen2.5-coder:7b-instruct-q4_K_M`, `llama3.1:8b`.
+ - Only `qwen2.5-coder:7b` → it is chosen (a coder beats nothing).
+ - Only embeddings → `None`.
+ - Switching:
+ - A primary 429 (after the client's own retries; set a zero backoff) followed by a local 200 → the opinion's model is `ollama qwen3.6:35b-a3b`, `switched` equals the LIMIT wording, and `on_switch` was called once.
+ - A second `complete` after the switch never calls Groq again. Assert this with respx call counts.
+ - A primary 503 gives the DOWN wording.
+ - When there is no fallback, the original `JudgeUnavailable` is raised, with its detail plus `; no local fallback: `. This covers three cases: `/api/tags` connection refused, no qwen model installed, and `fallback = "off"`.
+ - `fallback = ""` uses that name without calling `/api/tags`.
+ - A primary that is already `ollama` gets no fallback wrapper.
+ - `verify`: a run whose judge switches emits exactly one `Note` with the notice, and the report footer carries it.
+ - CLI: the notice is on stderr.
+ - TUI (Textual pilot): after a run whose events include the switch `Note`, the notice widget is visible, right-aligned, directly above `PromptFrame` inside `#bottom`, and hidden again when the next run starts. It must not break the fixed-height footer rule. Hide it with `display: none`, never with an empty line.
+- [ ] **Step 2: Implement.**
+ - The local client is a normal `JudgeClient` built from `provider_defaults("ollama")` with `model=` and a 180 s timeout. The first call loads the model into memory.
+ - Keys are never logged or echoed. The local client has no key.
+ - `Judge.queries` and `Judge.review` need no change beyond receiving the wrapper.
+ - `search/queries.fallback_notice` stays for the case where the fallback fails as well.
+- [ ] **Step 3:** `uv run pytest -q`, then `uv run ruff check . && uv run ruff format --check .`. Do not commit; leave the work unstaged for review.
+
+#### Task 11 — Amendment B: switch at once, and prove it live (user, 2026-09-28)
+
+A similar fallback in earlier projects of the user's "did not quite work". This one
+must switch **immediately and without a visible stall**. These are requirements, not
+polish:
+
+1. **Fail fast on the primary.** Today `JudgeClient.complete` makes up to 3 attempts
+ and honours `Retry-After` on a 429 (up to `max_wait`). When a fallback exists, that
+ waiting is the bug.
+ - Add `JudgeClient(..., fail_fast: bool = False)`. With it on, the first 429, 413,
+ 5xx, 4xx other than the two 400 downgrades, or transport error raises
+ `JudgeUnavailable` at once: no retry, no sleep.
+ - `FallbackClient` builds its primary with `fail_fast=True`.
+ - Test: the respx route for Groq is called exactly once, and the fake sleep is never
+ called.
+ - Without a fallback (`fallback = off`, or no local model found), keep today's retry
+ behaviour. Either decide the fallback before the first call (`installed_models` is
+ cheap), or hand the error back to a normal retrying client. Pick one and test it.
+2. **The failed batch is not lost.** The request that failed on Groq is sent again to
+ the local model inside the same `complete` call. The caller sees one successful
+ `Completion`.
+3. **Thread-safe switch.** The TUI runs judges on worker threads. The switch goes
+ through a lock, `on_switch` fires exactly once, and every later call goes straight
+ to the local client. Test: two threads calling `complete` against a 429 primary
+ give one switch, one notice, and one Groq call per thread at most.
+4. **The notice comes before the wait.** `on_switch` fires *before* the first local
+ request, so the user sees "judging with local ollama …" while the model loads.
+5. **Local model output must parse.**
+ - qwen3.x are thinking models. The request must not let thinking eat `max_tokens`
+ or leak `…` into the JSON. Check the current Ollama
+ OpenAI-compatible API with context7 (`reasoning_effort`, the `think` option,
+ `response_format` json_schema support) and send whatever Ollama really honours.
+ - Strip a leading `…` block before JSON parsing, defensively.
+ - Test with a fixture reply containing a think block.
+6. **Warm load, generous timeout.** The local client uses a 180 s timeout (the first
+ call loads the model into RAM) and `keep_alive` so later batches do not reload it.
+7. **Cost stays honest.** Local calls are counted apart from API calls: `api_calls`
+ counts the provider requests only. The report's cost line says how many
+ completions went to the local model.
+8. **Live verification (controller runs this; the implementer runs it too if Ollama
+ is up).** A script under `scripts/` (not a test):
+ - Build a `Judge` whose primary points at a local respx-free stub server, or at a
+ `base_url` that returns 429. A tiny `http.server` thread that always answers 429
+ is enough.
+ - Run a real `Judge.queries(["OpenAI battı."])` and a real `review` on one item
+ against the real local Ollama.
+ - Print the notice, the chosen model, the wall time from the first call to the
+ switch (it must be under 1 s), the time to the first local answer, and the parsed
+ result.
+ - It must exit 0 on this machine, and its output goes in the report.
+9. **Every run starts on the primary again (user, 2026-09-28).** The switch is sticky
+ within one run only. The next run tries the configured API first, whether it is a
+ new link in the TUI or a new `proofpath check` call. Today the CLI builds its judge
+ per invocation and the TUI builds engines and judges per run or per summary, so this
+ holds by construction. Pin it:
+ - `FallbackClient.reset()` returns to the primary and clears `switched`.
+ - Whatever builds a judge for a run either builds a fresh `FallbackClient` or calls
+ `reset()` at run start.
+ - Test: two runs in one TUI session. Run 1 sees a 429 and switches. Run 2's primary
+ route answers 200, and run 2 uses Groq with no notice. The notice from run 1 is
+ cleared when run 2 starts.
diff --git a/docs/superpowers/specs/2026-09-28-evidence-search-design.md b/docs/superpowers/specs/2026-09-28-evidence-search-design.md
new file mode 100644
index 0000000..74759f7
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-28-evidence-search-design.md
@@ -0,0 +1,225 @@
+# Evidence search for source-less text
+
+**Status:** approved 2026-09-28; implementation follows this document. Implements
+OPEN-ITEMS §17.1 and revises its "never on by default". Amends spec §9 (data flow),
+§11 (LLM budget), §13 (CLI surface) and §15 (honesty states) of
+`2026-09-10-proofpath-design.md`. The search is **experimental** until the §7 gate passes.
+
+## 1. Why
+
+A pasted text that has no link, no citation marker and no bibliography currently ends
+in `PARSE ERROR: the text links or cites no source; nothing to verify against`
+(`verify.NO_SOURCE_IN_TEXT`, v0.4.6, `04341ea`). The common case is a linkless tweet:
+X cannot be read (§3 non-goal), and pasted tweet text that links nothing gives the tool
+nothing to check. The user wants plain text recognised as plain text and searched, with
+proofpath saying whether it found evidence for what the text claims.
+
+## 2. Decisions (2026-09-28, with the user)
+
+| Question | Decision |
+|---|---|
+| Search provider | **Tavily** (API key) and **SearXNG** (the user's own instance, `base_url`, no key), both behind one `Searcher` protocol. Brave and the Google Fact Check API are not in v1 |
+| Query | **Hybrid.** By default the query is the sentence itself, cleaned, with zero LLM calls, so §11 holds. When the judge is configured, the LLM writes 1–2 decontextualised queries per claim. If the LLM fails because of a rate limit, quota or outage, the run **falls back to the sentence and tells the user**: "LLM limit reached — searched with the sentence text" |
+| Which sentences | **A heuristic check-worthiness filter plus a cap**: at most 5 claims, each with up to 3 results. Sentences left unsearched are counted in coverage (rule 6) |
+| Short text (added 2026-09-28) | When **no** sentence passes the filter, every sentence is searched, up to the cap. A one-line post ("OpenAI battı") is exactly the claim the user wants checked, so it is never filtered away |
+| Language (added 2026-09-28) | The models are English-only (`nli-deberta-v3-base`, `bge-small-en-v1.5`). A claim that does not look English is **translated by the judge** when `--judge` is on: the English rendering is used for the queries and is the text the models check, while the report shows the user's sentence with a `checked as: …` note. With no judge, or a judge that did not answer, the claim is `UNVERIFIED (language not supported)` and is not searched. That is never a silent NEI |
+| Trigger | **Automatic.** When the text has no source and a search provider is configured, it is searched without asking. Configuring the key or URL is the consent. `--no-search` turns it off for one run, and `permissions.web_search = deny` turns it off for good |
+
+Rejected, and kept rejected: letting `--judge` answer when there is no source. That makes
+it a truth oracle (§3 non-goal). The LLM only writes *queries*; verdicts still come from
+passages that were fetched.
+
+## 3. Flow
+
+A new stage, `Searching`, runs in `verify.prepare()` between Claims and Resolving. It runs
+only when all of these hold:
+
+- the document has no `references`, and `find_markers` finds nothing;
+- for a post or page kind, it carries no links (the conditions at `verify.py:755` and
+ `:763`);
+- network permission is `allow`, and `permissions.web_search` is not `deny`;
+- `--no-search` is not set;
+- a search provider is configured.
+
+1. **Select claims.** `claims.checkworthy(doc) -> Claims` is modelled on `pair_links`: each
+ claim gets a synthetic `CitationMarker` with an empty span. A sentence qualifies when it
+ has at least 5 words, is not a question, and contains at least one of: a digit (a
+ number, year or percentage), a capitalised token that is not the first word, or a
+ quotation. The first `search.max_claims` (5) qualifying sentences are kept, in document
+ order.
+2. **Build queries** (`search/queries.py`).
+ - *Default*: strip URLs, `#hashtags`, `@handles` and emoji from the sentence, collapse
+ the whitespace, and truncate at a word boundary to 400 characters (Tavily's query
+ limit).
+ - *With the judge*: one batched `JudgeClient.complete` call using
+ `prompts/queries.md` and a strict JSON schema
+ `{items: [{claim: int, queries: [str, 1..2]}]}`. It shares the run's LLM budget (§11).
+ - *Fallback*: on `JudgeUnavailable` or `JudgeError`, use the default queries and add a
+ run-level notice (§5). A new `Judge.queries()` follows `review()`'s rule and
+ never raises. A 429 ("HTTP 429 from …" in `Judge.detail`) is reported as
+ "LLM limit reached"; any other failure is reported as "LLM did not answer".
+3. **Search** (`search/providers.py`).
+ - The protocol is `Searcher.search(query: str, max_results: int) -> list[SearchHit]`,
+ with `SearchHit(url, title, rank)`.
+ - `TavilySearcher` sends `POST https://api.tavily.com/search`. The key comes from
+ `secrets.resolve_api_key(search.api_key_env)`, which defaults to `TAVILY_API_KEY`.
+ - `SearxngSearcher` sends `GET {base_url}/search?q=…&format=json`.
+ - Both go through `polite.PoliteClient`, with their hosts added to `MIN_INTERVAL`.
+ - **The provider's own snippet or `content` is never used as evidence.** Each page is
+ fetched by proofpath through the fetch ladder, so the quoted passage comes from the
+ source itself (rule 1).
+4. **Filter hits.**
+ - Canonicalise URLs (drop the fragment and tracking parameters) and dedupe them.
+ - Drop the input's own host (for a post or page), and drop hosts that cannot be read
+ (`social.is_social` hosts that proofpath cannot read, e.g. x.com).
+ - Keep `search.results_per_claim` (3) per claim. Claims may share a page.
+5. **Synthetic references.**
+ - Each distinct hit becomes `Reference(number, raw=url, locator, origin="search")`.
+ `origin` is a new field on `document.Reference` and defaults to `"author"`.
+ - Each claim's `cited_refs` point at its hits.
+ - From here Resolving (a bare URL goes to the `WebProvider`), Fetching and
+ `decide_all` run **unchanged**. `_jobs` makes one job per page, so a claim gets one
+ `ClaimResult` per page.
+
+## 4. Honesty states (amends §15)
+
+| State | Cause |
+|---|---|
+| `FOUND BY PROOFPATH (not cited by the author)` | provenance tag carried by **every** finding and source whose reference has `origin="search"`, together with its usual verdict or state. It is never dropped in any renderer (terminal, markdown, JSON, SARIF, TUI) |
+| `SUPPORTED (found by proofpath)` | a page the search found supports the claim. A note, with its passage (rule 1): this is the answer the user asked for, so it is shown and not left silent the way a supported *cited* claim is |
+| `NO EVIDENCE FOUND (searched)` | the search ran for a claim, but no hit could be read, or every read hit was NEI. This is **not** REFUTED, and the report says so (rule 2) |
+| `UNVERIFIED (language not supported)` | the claim is not in English and no judge translated it (§2). The finding says `--judge` would translate it |
+| `UNVERIFIED (search unavailable)` | the search provider was down, rate limited, or answered with a non-JSON body after backoff (`ProviderError`) |
+| `UNVERIFIED (credentials missing)` | existing state; reused when a configured provider cannot run: `tavily` with no key, or `searxng` with no `base_url`. The finding names what to set |
+
+A page found by the search that turns out unreadable keeps its usual §15 state
+(blocked, unreachable, robots.txt …). Rules 1–3 are unchanged: no passage, no verdict,
+and unreachable stays unreachable.
+
+## 5. Report and coverage
+
+- **Experimental banner**, printed at the top of any run that searched:
+ `evidence search is experimental — AVeriTeC search-mode vs 0.708 majority baseline`.
+ The number is a constant, updated whenever the eval is re-run (§7).
+- **Run notices.** The LLM-fallback notice from §3 step 2 is shown in the terminal,
+ markdown, JSON (`notices: [...]`) and the TUI run header.
+- **Coverage.** Search only runs on a document that cites nothing (§3), so a run never
+ mixes the author's sources with found ones. The usual coverage block therefore counts
+ the pages that were found, and a separate block beneath it says where they came from
+ and how many claims were left unsearched:
+ ```
+ evidence search (experimental)
+ claims searched 3 of 7 check-worthy
+ claims not searched 4 (cap 5 / filtered)
+ pages found / read 9 / 6
+ ```
+- **No provider configured.** The PARSE ERROR stays as it is. `NO_SOURCE_HINT` now names
+ the one-line setup: `proofpath config set search.provider tavily` plus
+ `TAVILY_API_KEY`, or `search.provider searxng` plus `search.base_url`.
+
+## 6. Configuration and CLI (amends §13)
+
+```toml
+[search]
+provider = "off" # "off", "tavily", "searxng"
+api_key_env = "TAVILY_API_KEY"
+base_url = "" # SearXNG only
+max_claims = 5
+results_per_claim = 3
+
+[permissions]
+web_search = "allow" # only effective once a provider is set; "ask"/"deny" turn it off (search never prompts)
+```
+
+- `config._coerce` gains an `int` branch.
+- `--no-search` goes on `check` and on the TUI (`/config` panel entries plus
+ `settings_hints.py` strings).
+- `cli` and `tui` only pass the flag through; all logic stays in `verify()`.
+- The key is only ever read through `secrets.resolve_api_key`. It is never written to
+ config or logged.
+
+## 7. Evaluation (the §17.1 gate)
+
+`scripts/eval_averitec.py --search` replaces each claim's gold `source_urls` with the
+search hits and then scores the same way (`score`, `render_report`). The claim's own
+fact-check article is excluded by URL and host, so the tool does not find the answer
+key. Results go to `docs/eval/-averitec-search.md`. The feature ships flagged
+experimental, with the number printed, and the flag comes off only once the score is
+above the 0.708 majority baseline.
+
+## 8. Tests (no network)
+
+- `respx` plus `tests/fixtures/search/{tavily,searxng}.json`, and the fakes in
+ `tests/fakes.py`.
+- A plain tweet text leads to searched claims, each finding tagged `FOUND BY PROOFPATH`.
+- Empty hits or all-NEI results give `NO EVIDENCE FOUND (searched)` and never REFUTED.
+- A judge 429 falls back to sentence queries, and the run carries the notice.
+- With no provider, the old PARSE ERROR appears with the new hint. `--no-search` and
+ `web_search = deny` mean no search call is made.
+- Coverage counts the sentences past the cap as not searched. The provider snippet
+ never appears as a passage.
+- The golden report `tests/data/verify-report-golden.json` is updated.
+
+## 9. Product rules, checked
+
+1. No passage, no verdict: pages are fetched and the passage comes from the page, never
+ from the search snippet.
+2. Absence of evidence is not evidence of absence: `NO EVIDENCE FOUND (searched)` is its
+ own state.
+3. No false ghosts: not affected, because search results are web pages and never
+ bibliographic references.
+4. No prompt without a TTY: the search never prompts, and the consent is the configured
+ provider.
+5. No large install: none.
+6. Coverage: a separate block, plus the experimental banner.
+
+## 10. Out of scope (v1)
+
+- Searching for the uncited sentences of a partly cited document (§17.3).
+- Brave, Google Fact Check (ClaimReview), and LLM-based check-worthiness.
+- Using the search snippet as a fallback when the page cannot be read.
+
+## 11. Local fallback judge (decided 2026-09-28)
+
+The judge runs on Groq's free tier, which hits its rate limit. When the configured
+judge stops answering, the run switches to a model already installed in the user's
+local Ollama instead of losing the second opinion (and, for the search, the
+judge-written queries and translations).
+
+- **Trigger.** Any `JudgeUnavailable` from the primary provider: a 429, a quota
+ error, or the provider not answering. The switch lasts for the rest of the run, so
+ a rate-limited Groq is not hit again on every batch.
+ The switch is immediate: with a fallback available, the primary gets no retry and
+ no `Retry-After` wait. The failed request is sent again to the local model, so no
+ batch is lost, and the notice appears before the model starts loading.
+ The next run (a new link, a new `check`) tries the configured API first again.
+- **Which model.** The fallback is `qwen3.5:9b` (the user's choice, backed by a
+ 2026-09-28 translation benchmark: 15/15 correct TR→EN, where qwen3.6 and
+ gpt-oss:20b changed the meaning). It must already be installed; proofpath never
+ pulls it (rule 5). `GET {ollama}/api/tags` only confirms that it is there. Not
+ installed, or no Ollama, means no fallback: the run reports the original
+ `JudgeUnavailable` with `; no local fallback: qwen3.5:9b is not installed` (or
+ why Ollama could not be asked). The order is fixed: the configured API, then this
+ model, and nothing else.
+- **Reasoning.** Every Ollama request (the fallback, and a judge configured as
+ `ollama` directly) sends `reasoning_effort: none` (`think: false`): a thinking qwen
+ spent its whole token budget and answered nothing. `judge.provider ollama` now
+ defaults to the same `qwen3.5:9b`, so there is one local model. A 400 that names
+ `reasoning_effort` still drops the field, as for any provider.
+- **Config.** `judge.fallback`: `qwen3.5:9b` by default; `off` (in any case) turns
+ the fallback off; another model name is the user's own override, used as written
+ and checked against the tags the same way. It does nothing when the judge is
+ already Ollama.
+- **Telling the user.** The switch is never silent.
+ - The run emits a `Note` that names the cause: `Groq limit reached — judging with
+ local ollama ` (429, 413), `Groq rejected the key (HTTP 401) — …` (401,
+ 403), or `Groq did not answer () — …` for anything else (`HTTP 500`,
+ `timeout`, `connection failed`, `empty answer`). Every failure switches; the
+ cause never carries a body or a key.
+ - The CLI prints that line on stderr.
+ - The TUI shows it in a one-line notice, right-aligned, directly above the prompt
+ bar. It stays until the next run starts.
+ - The report records it as well. Each opinion's `model` already names who gave it
+ (`ollama qwen3.5:9b`), and the footer carries the notice.
+- **Honesty.** A local model's opinion is still only a second opinion (spec §11). It
+ never replaces a verdict's passage.
diff --git a/scripts/eval_averitec.py b/scripts/eval_averitec.py
index 8ee6903..59570f8 100644
--- a/scripts/eval_averitec.py
+++ b/scripts/eval_averitec.py
@@ -2,7 +2,7 @@
Usage:
uv run python scripts/eval_averitec.py [--limit 100] [--no-browser] [--sleep 1.0]
- [--resume] [--out docs/eval/-averitec.md]
+ [--resume] [--search] [--out docs/eval/-averitec.md]
SciFact measures retrieval and entailment over abstracts we are handed. AVeriTeC
measures the whole product: a real-world claim, its real source pages, fetched over
@@ -29,7 +29,7 @@
import time
from collections import Counter
from collections.abc import Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from datetime import date
from pathlib import Path
from urllib.parse import urlsplit
@@ -40,6 +40,8 @@
from proofpath.fetch import Fetched
from proofpath.models import Label, Passage
from proofpath.paths import cache_dir
+from proofpath.search import Searcher, keep_hits
+from proofpath.search.queries import sentence_query
from proofpath.verify import NO_TEXT, Engine
# The state a claim gets when the dataset itself names no page to read. It is not a
@@ -53,6 +55,7 @@
# split's evidence is already a snapshot, and that changes what the numbers measure.
ARCHIVE_HOST = "web.archive.org"
RESULTS_NAME = "averitec_results.json"
+SEARCH_RESULTS_NAME = "averitec_search_results.json"
@dataclass(frozen=True)
@@ -90,6 +93,35 @@ def state_for(fetched: Fetched) -> str:
return fetched.outcome.value
+def search_urls(searcher: Searcher, claim: averitec.Claim, limit: int) -> tuple[str, ...]:
+ """What the evidence search would read for this claim, minus the answer key.
+
+ The whole fact-checking site is excluded, not just the one article: a site that
+ rated the claim quotes its own rating on every related page (the leakage the
+ §17.1 gate must not measure). "Site" reaches across subdomains both ways, so a
+ rating on ``factcheck.afp.com`` also rules out ``www.afp.com``. That errs towards
+ a lower score, never a leaked one.
+ """
+ checker = _site(claim.fact_check_url)
+ # Three times the limit, not a fixed few: a checker's own pages can crowd the top
+ # of the results, and the claim should still get its full share of other pages.
+ hits = [
+ hit
+ for hit in searcher.search(sentence_query(claim.text), limit * 3)
+ if not checker or not _same_site(_site(hit.url), checker)
+ ]
+ return tuple(hit.url for hit in keep_hits(hits, own_host=None, limit=limit))
+
+
+def _site(url: str) -> str:
+ return (urlsplit(url).hostname or "").removeprefix("www.")
+
+
+def _same_site(host: str, checker: str) -> bool:
+ """One host is the other, or a subdomain of it, in either direction."""
+ return host == checker or host.endswith(f".{checker}") or checker.endswith(f".{host}")
+
+
@dataclass(frozen=True)
class AveritecResult:
n: int
@@ -211,8 +243,8 @@ def render_report(result: AveritecResult, *, date: str, limit: int) -> str:
# --- live half: run by hand, not covered by tests ------------------------
-def _results_path() -> Path:
- return cache_dir() / "datasets" / RESULTS_NAME
+def _results_path(*, search: bool) -> Path:
+ return cache_dir() / "datasets" / (SEARCH_RESULTS_NAME if search else RESULTS_NAME)
def _load_rows(path: Path) -> list[Row]:
@@ -314,10 +346,15 @@ def main(argv: list[str] | None = None) -> int:
)
parser.add_argument("--sleep", type=float, default=1.0, help="seconds between fetches")
parser.add_argument("--resume", action="store_true", help="skip claim ids already scored")
+ parser.add_argument(
+ "--search",
+ action="store_true",
+ help="read what the evidence search finds instead of the gold URLs",
+ )
parser.add_argument("--out", default="")
args = parser.parse_args(argv)
- results = _results_path()
+ results = _results_path(search=args.search)
if results.exists() and not args.resume:
# The file is the only record of a run that costs hours of network; a fresh
# run must not quietly replace it. Checked before anything is downloaded, so
@@ -346,11 +383,35 @@ def main(argv: list[str] | None = None) -> int:
# prompted for (product rule 4); ``--no-browser`` forces the browser step off
# outright, otherwise the config's own permission decides.
engine = Engine.default(config, interactive=False, browser=False if args.no_browser else None)
+ searcher: Searcher | None = None
+ if args.search:
+ # ``Engine.default`` already resolved the configured provider onto its own
+ # polite client (verify.py); reusing that instead of building a second one
+ # keeps the run's searcher assembled exactly once, like everything else here.
+ if engine.searcher is None:
+ print(
+ f"error evidence search is not set up: "
+ f"{engine.search_problem or 'search.provider is off'}",
+ file=sys.stderr,
+ )
+ engine.close()
+ return 2
+ searcher = engine.searcher
+
scored = 0
try:
for position, claim in enumerate(claims, start=1):
if claim.id in done:
continue
+ if searcher is not None:
+ # The gold URLs are the answer key; search mode reads what evidence
+ # search would actually find, which is the §17.1 gate this run exists
+ # to measure. ``non_urls=0`` because a search hit is always a URL.
+ claim = replace(
+ claim,
+ source_urls=search_urls(searcher, claim, config.search.results_per_claim),
+ non_urls=0,
+ )
stored.append(_decide_claim(engine, claim, sleep=args.sleep))
scored += 1
# After every claim, not at the end: a run this long is interrupted more
@@ -370,10 +431,11 @@ def main(argv: list[str] | None = None) -> int:
report = render_report(score(rows), date=date.today().isoformat(), limit=args.limit)
print()
print(report)
+ suffix = "-search" if args.search else ""
out = (
Path(args.out)
if args.out
- else Path("docs/eval") / f"{date.today().isoformat()}-averitec.md"
+ else Path("docs/eval") / f"{date.today().isoformat()}-averitec{suffix}.md"
)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(report, encoding="utf-8")
diff --git a/scripts/live_judge_fallback.py b/scripts/live_judge_fallback.py
new file mode 100644
index 0000000..24dd771
--- /dev/null
+++ b/scripts/live_judge_fallback.py
@@ -0,0 +1,199 @@
+"""Live check of the local fallback judge (spec section 11, Task 11 Amendment B 8).
+
+Usage:
+ uv run python scripts/live_judge_fallback.py [--keep-loaded]
+
+A stand-in for Groq runs on a local port and answers every request with a 429 and a
+``Retry-After: 60``. The judge is built exactly as ``proofpath check --judge`` builds
+it (``judge.build_client``), pointed at that stand-in, and asked for real:
+``Judge.queries`` on the Turkish claim for "OpenAI went bust" and one ``Judge.review``
+item. Both are answered by the configured fallback, qwen3.5:9b, which must already
+be installed in the local Ollama; nothing is pulled.
+
+What it proves, and exits 1 if it does not hold:
+ * the switch happens on the first 429, with no retry and no ``Retry-After`` wait
+ (the stand-in sees exactly one request, and the switch comes in under a second);
+ * the notice is the spec's wording and fires before the local model is loaded;
+ * the failed request is answered by the local model, and both answers parse.
+
+By default the chosen model is unloaded first, so the time to the first local answer
+includes a cold load, which is the case a user meets. ``--keep-loaded`` skips that.
+
+This is not a test: it needs a running Ollama with qwen3.5:9b installed. No
+API key is read, sent or printed; the stand-in needs none.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import ClassVar
+
+import httpx
+
+from proofpath import judge
+from proofpath.config import JudgeConfig
+from proofpath.models import Label
+from proofpath.search.queries import plan_queries
+
+SWITCH_BUDGET_S = 1.0
+CLAIM = "OpenAI battı." # noqa: RUF001 - Turkish, verbatim: the claim the user tried
+REVIEW_ITEM = judge.JudgeItem(
+ id="c1",
+ claim="OpenAI went bankrupt in 2025.",
+ passage="OpenAI reported record revenue in 2025 and closed a new funding round.",
+ verdict=Label.REFUTED,
+ tier="low",
+)
+
+
+class RateLimited(BaseHTTPRequestHandler):
+ """Groq on a bad day: every request is a 429 asking for a minute's patience."""
+
+ hits: ClassVar[list[float]] = []
+
+ def do_POST(self) -> None:
+ RateLimited.hits.append(time.perf_counter())
+ length = int(self.headers.get("Content-Length") or 0)
+ self.rfile.read(length)
+ body = b'{"error": {"message": "Rate limit reached", "type": "tokens"}}'
+ self.send_response(429)
+ self.send_header("Retry-After", "60")
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, format: str, *args: object) -> None: # noqa: A002 - stdlib name
+ return # quiet: the script prints its own lines
+
+
+def _unload(model: str) -> None:
+ """Take ``model`` out of memory, so the first local answer includes the load."""
+ root = judge.provider_defaults("ollama").base_url.removesuffix("/v1")
+ try:
+ httpx.post(f"{root}/api/generate", json={"model": model, "keep_alive": 0}, timeout=60)
+ except httpx.HTTPError as exc:
+ print(f"warning: could not unload {model}: {type(exc).__name__}")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--keep-loaded", action="store_true", help="do not unload first")
+ args = parser.parse_args()
+
+ ollama = judge.provider_defaults("ollama").base_url
+ installed = judge.installed_models(ollama, timeout=judge.TAGS_TIMEOUT)
+ expected = JudgeConfig().fallback
+ print(f"ollama models {', '.join(installed) or '(none)'}")
+ print(f"judge.fallback {expected} (the default)")
+ if expected not in installed:
+ print(f"FAIL: {expected} is not installed in the local Ollama; it is never pulled")
+ return 1
+ print(f"reasoning sent {judge.LOCAL_REASONING}")
+ if not args.keep_loaded:
+ _unload(expected)
+ print(f"unloaded {expected} (the first local answer includes a cold load)")
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), RateLimited)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ port = server.server_address[1]
+ # Groq as configured by default, only pointed at the stand-in. No key variable:
+ # the stand-in needs none, and this script never reads a real one.
+ primary = JudgeConfig(base_url=f"http://127.0.0.1:{port}/openai/v1", api_key_env="")
+
+ notices: list[tuple[float, str]] = []
+ client = judge.build_client(
+ primary, None, on_switch=lambda text: notices.append((time.perf_counter(), text))
+ )
+ assert isinstance(client, judge.FallbackClient)
+ j = judge.Judge(client)
+
+ started = time.perf_counter()
+ written = j.queries([CLAIM])
+ first_answer = time.perf_counter()
+ review_started = time.perf_counter()
+ opinions = j.review([REVIEW_ITEM])
+ review_done = time.perf_counter()
+ server.shutdown()
+
+ failures: list[str] = []
+ print()
+ if notices:
+ switched_at, notice = notices[0]
+ switch_s = switched_at - started
+ print(f"notice {notice}")
+ print(f"chosen model {client.model} (provider {client.provider})")
+ print(f"time to switch {switch_s:.3f}s (first call to the notice; budget 1s)")
+ if switch_s >= SWITCH_BUDGET_S:
+ failures.append(f"the switch took {switch_s:.3f}s")
+ wording = judge.JUDGE_FALLBACK_LIMIT.format(provider="Groq", model=expected)
+ if notice != wording:
+ failures.append(f"the notice was {notice!r}, not {wording!r}")
+ if len(notices) != 1:
+ failures.append(f"the notice fired {len(notices)} times")
+ else:
+ failures.append("no switch notice")
+ print(f"first local answer {first_answer - started:.2f}s after the first call")
+ print(f"review answer {review_done - review_started:.2f}s (model already loaded)")
+ print(f"groq stand-in hit {len(RateLimited.hits)} time(s) (no retry, no Retry-After wait)")
+ if len(RateLimited.hits) != 1:
+ failures.append(f"the rate-limited primary was asked {len(RateLimited.hits)} times")
+ if client.model != expected:
+ failures.append(f"switched to {client.model}, expected {expected}")
+
+ print()
+ print(f"queries({CLAIM!r})")
+ if j.unavailable:
+ failures.append(f"queries: judge unavailable ({j.detail})")
+ for index, claim in sorted(written.items()):
+ print(f" [{index}] english {claim.english}")
+ for query in claim.queries:
+ print(f" query {query}")
+ if 0 not in written or not written[0].queries:
+ failures.append("queries: no parsed query for claim 0")
+ # Reported, not failed on: ``plan_queries`` refuses an echo either way, so an
+ # untranslated claim ends up "language not supported" instead of being checked.
+ plan = plan_queries([CLAIM], j)
+ translated = plan.hypotheses[0] is not None
+ print(f" translated {'yes' if translated else 'no'} (hypothesis {plan.hypotheses[0]!r})")
+
+ print()
+ print(f"review({REVIEW_ITEM.claim!r} / verdict {REVIEW_ITEM.verdict.value})")
+ for ident, opinion in opinions.items():
+ print(f" {ident} {opinion.label.value} by {opinion.model}")
+ print(f" rationale {opinion.rationale}")
+ for line in j.skipped:
+ print(f" skipped: {line}")
+ if j.unavailable:
+ failures.append(f"review: judge unavailable ({j.detail})")
+ if "c1" not in opinions:
+ failures.append("review: no parsed opinion for c1")
+ elif opinions["c1"].model != f"ollama {expected}":
+ failures.append(f"review: the opinion names {opinions['c1'].model}")
+
+ cost = client.cost
+ print()
+ print(
+ f"cost {cost.calls} answers, {cost.local_calls} local, "
+ f"{cost.calls - cost.local_calls} API; {cost.prompt_tokens:,} prompt · "
+ f"{cost.completion_tokens:,} completion tokens; waited {cost.waited_s:.1f}s"
+ )
+ if cost.waited_s:
+ failures.append(f"the client waited {cost.waited_s:.1f}s")
+ client.close()
+
+ print()
+ if failures:
+ for failure in failures:
+ print(f"FAIL: {failure}")
+ return 1
+ print("OK: switched at once, answered locally, both answers parsed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/proofpath/claims.py b/src/proofpath/claims.py
index eb04c4e..b52265c 100644
--- a/src/proofpath/claims.py
+++ b/src/proofpath/claims.py
@@ -395,6 +395,89 @@ def strip_links(text: str) -> str:
return _BEFORE_PUNCTUATION.sub("", " ".join(text.split()))
+# What makes a sentence worth searching for (OPEN-ITEMS 17.1a): long enough to state
+# something, not a question, and carrying something a page could confirm -- a figure,
+# a name, or a quotation. Opinion ("I think this is bad") has nothing to find. Three
+# words, not more: Turkish says "Ankara Türkiye'nin başkentidir" in three.
+CHECKWORTHY_MIN_WORDS = 3
+# A word keeps its apostrophe suffix: "Türkiye'nin" is one word, not two.
+_WORDS = re.compile(r"\w+(?:['’]\w+)*") # noqa: RUF001 - a real apostrophe, not a typo
+# Hashtags and handles label a post; they state nothing and name no one.
+_TAGS = re.compile(r"(? bool:
+ stripped = _TAGS.sub(" ", text).strip()
+ if stripped.endswith("?"):
+ return False
+ words = _WORDS.findall(stripped)
+ if len(words) < CHECKWORTHY_MIN_WORDS:
+ return False
+ if _FIGURE.search(stripped) or _QUOTED.search(stripped):
+ return True
+ first = words[0]
+ if first[:1].isupper() and first.casefold() not in _OPENERS:
+ return True
+ return any(word[:1].isupper() for word in words[1:]) or any(
+ any(char.isupper() for char in word[1:]) for word in words
+ )
+
+
+def checkworthy(doc: Document, *, limit: int) -> Checkworthy:
+ """Claims for a document that cites nothing, to be searched for.
+
+ They cite no reference yet. The Searching stage points each at the pages it finds,
+ and a claim it finds nothing for keeps an empty ``cited_refs`` and is reported
+ as NO EVIDENCE FOUND, never dropped (product rule 6). When no sentence looks
+ check-worthy, every sentence is taken instead: a one-line post is exactly the
+ claim the reader wants checked.
+ """
+ every: list[Claim] = []
+ worthy: list[Claim] = []
+ for paragraph in doc.paragraphs:
+ for position, sentence in enumerate(paragraph.sentences):
+ text = strip_links(sentence.text)
+ if not _WORD.search(text):
+ continue
+ claim = Claim(
+ text=text,
+ locator=sentence.locator,
+ cited_refs=(),
+ paragraph=paragraph.index,
+ sentence=position,
+ )
+ every.append(claim)
+ if is_checkworthy(text):
+ worthy.append(claim)
+ chosen = worthy or every
+ return Checkworthy(claims=tuple(chosen[:limit]), eligible=len(chosen), sentences=len(every))
+
+
def pair_numeric(
doc: Document, markers: Sequence[CitationMarker]
) -> tuple[list[Claim], list[CitationMarker]]:
diff --git a/src/proofpath/cli.py b/src/proofpath/cli.py
index 6fe377d..a74149e 100644
--- a/src/proofpath/cli.py
+++ b/src/proofpath/cli.py
@@ -527,6 +527,13 @@ def check(
no_cache: Annotated[
bool, typer.Option("--no-cache", help="Neither read nor fill the local cache.")
] = False,
+ no_search: Annotated[
+ bool,
+ typer.Option(
+ "--no-search",
+ help="Do not search the web for evidence when the text cites no source.",
+ ),
+ ] = False,
out_path: Annotated[
Path | None,
typer.Option(
@@ -559,6 +566,10 @@ def on_event(event: events.Event) -> None:
# order. ``Progress`` has no bar in v0.1 — the TUI (Phase 8) draws one.
if isinstance(event, events.StageEnd):
ui.stage_row(human, event.name, event.by, event.summary, event.elapsed)
+ elif isinstance(event, events.Note) and event.notice:
+ # The judge switched to its local fallback: said at once, on stderr, and
+ # never dropped, not even under ``-q`` (spec section 11).
+ ui.kv(out, "judge", event.text, err=True)
elif isinstance(event, events.Note):
ui.note(human, event.text)
@@ -574,6 +585,7 @@ def on_event(event: events.Event) -> None:
# ``--summarize`` on its own is one call over the finished report and
# nothing else; the escalation stage is what ``--judge`` buys.
escalate=judge,
+ search=not no_search,
) as engine,
_interruptible(cancel),
):
@@ -667,7 +679,9 @@ def _build_judge(out: ui.Ui, config: cfg.Config, *, asked_by: str = "") -> judge
# Said once, before anything is sent, and on stderr so --format json keeps
# one document on stdout (spec section 11).
ui.hint(out, judge_mod.GEMINI_DATA_USE, err=True)
- return judge_mod.Judge(judge_mod.JudgeClient(config.judge, key))
+ # Built per invocation, so every ``check`` starts on the configured provider and
+ # only this run can switch to the local fallback (Amendment B 9).
+ return judge_mod.Judge(judge_mod.build_client(config.judge, key))
def _flush_streams() -> None:
diff --git a/src/proofpath/config.py b/src/proofpath/config.py
index 0206954..945d03e 100644
--- a/src/proofpath/config.py
+++ b/src/proofpath/config.py
@@ -10,7 +10,7 @@
import sys
from dataclasses import dataclass, field, fields
from pathlib import Path
-from typing import Any, Literal, get_args
+from typing import Any, Literal, get_args, get_origin
from proofpath.paths import config_path
@@ -34,6 +34,10 @@ class Permissions:
# Step 3 of the fetch ladder: ~280 MB browser engine. Never installed silently.
install_browser: Permission = "ask"
network: Permission = "allow"
+ # Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). Only matters
+ # once ``search.provider`` is set: configuring a provider is the consent. The
+ # search never prompts, so ``ask`` is read as ``deny`` (rule 4).
+ web_search: Permission = "allow"
@dataclass(frozen=True)
@@ -47,6 +51,10 @@ class Contact:
email: str = ""
+#: The local fallback judge (spec section 11): Groq, then this, and nothing else.
+LOCAL_JUDGE_MODEL = "qwen3.5:9b"
+
+
@dataclass(frozen=True)
class JudgeConfig:
"""Optional LLM second opinion (spec section 11). Off unless ``--judge`` is passed.
@@ -59,6 +67,28 @@ class JudgeConfig:
model: str = "openai/gpt-oss-120b"
base_url: str = "https://api.groq.com/openai/v1"
api_key_env: str = "GROQ_API_KEY"
+ # When the provider stops answering (spec section 11) the run switches to this
+ # Ollama model: qwen3.5:9b by default (the user's choice, backed by a TR->EN
+ # translation benchmark on 2026-09-28), ``off`` never switches, and another name
+ # is the user's own override. It must already be installed; nothing is pulled.
+ fallback: str = LOCAL_JUDGE_MODEL
+
+
+SearchProvider = Literal["off", "tavily", "searxng"]
+SEARCH_PROVIDERS: tuple[str, ...] = get_args(SearchProvider)
+
+
+@dataclass(frozen=True)
+class SearchConfig:
+ """Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). Off until a
+ provider is named. The key is never stored here: ``api_key_env`` names the
+ variable that holds it, exactly as ``JudgeConfig`` does."""
+
+ provider: SearchProvider = "off"
+ api_key_env: str = "TAVILY_API_KEY"
+ base_url: str = "" # SearXNG only: the user's own instance
+ max_claims: int = 5
+ results_per_claim: int = 3
@dataclass(frozen=True)
@@ -67,6 +97,7 @@ class Config:
fetch: FetchConfig = field(default_factory=FetchConfig)
contact: Contact = field(default_factory=Contact)
judge: JudgeConfig = field(default_factory=JudgeConfig)
+ search: SearchConfig = field(default_factory=SearchConfig)
_SECTIONS: dict[str, type] = {
@@ -74,6 +105,7 @@ class Config:
"fetch": FetchConfig,
"contact": Contact,
"judge": JudgeConfig,
+ "search": SearchConfig,
}
@@ -119,6 +151,16 @@ def _coerce(section: str, key: str, value: Any, expected: Any, path: Path) -> An
if isinstance(value, str):
return value
raise ConfigError(f"{path}: {where} must be a string, got {value!r}")
+ if get_origin(expected) is Literal:
+ choices = get_args(expected)
+ if isinstance(value, str) and value in choices:
+ return value
+ raise ConfigError(f"{path}: {where} must be one of {', '.join(choices)}, got {value!r}")
+ if expected is int:
+ # ``bool`` is an ``int`` to Python and a different answer to a reader.
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 1:
+ return value
+ raise ConfigError(f"{path}: {where} must be a positive integer, got {value!r}")
raise ConfigError(f"{path}: {where} has an unsupported type") # pragma: no cover
@@ -138,7 +180,13 @@ def _build_section(section: str, raw: Any, path: Path) -> Any:
def _resolve_type(annotation: Any) -> Any:
# ``from __future__ import annotations`` leaves field types as strings.
if isinstance(annotation, str):
- return {"Permission": Permission, "bool": bool, "str": str}[annotation]
+ return {
+ "Permission": Permission,
+ "SearchProvider": SearchProvider,
+ "bool": bool,
+ "int": int,
+ "str": str,
+ }[annotation]
return annotation
@@ -165,6 +213,8 @@ def load_config(path: Path | None = None) -> Config:
def _toml_value(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
+ if isinstance(value, int):
+ return str(value)
if isinstance(value, str):
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
@@ -203,6 +253,13 @@ def set_value(dotted_key: str, raw_value: str, path: Path | None = None) -> Conf
if lowered not in {"true", "false"}:
raise ConfigError(f"{dotted_key} must be true or false, got {raw_value!r}")
value = lowered == "true"
+ if expected is int:
+ try:
+ value = int(raw_value.strip())
+ except ValueError:
+ raise ConfigError(
+ f"{dotted_key} must be a positive integer, got {raw_value!r}"
+ ) from None
coerced = _coerce(section, key, value, expected, path)
current = load_config(path)
body = getattr(current, section)
diff --git a/src/proofpath/document.py b/src/proofpath/document.py
index 1b915d7..7413d36 100644
--- a/src/proofpath/document.py
+++ b/src/proofpath/document.py
@@ -60,6 +60,12 @@ class Paragraph:
lines: tuple[tuple[int, int], ...]
+# Who put a reference in the document: its author, or proofpath's evidence search
+# (OPEN-ITEMS 17.1a). Carried on the reference so every finding about a found page
+# can say so, whichever stage wrote it.
+Origin = Literal["author", "search"]
+
+
@dataclass(frozen=True)
class Reference:
"""One bibliography entry, kept verbatim; resolve.py does the parsing."""
@@ -67,6 +73,7 @@ class Reference:
number: int # 1-based bibliography position, what "[12]" refers to
raw: str # the entry verbatim, whitespace collapsed
locator: Locator
+ origin: Origin = "author"
def __post_init__(self) -> None:
if self.number < 1:
@@ -153,6 +160,10 @@ class Claim:
# Shared by every Claim of one paragraph-scoped citation, e.g. "p3:118-122".
group: str | None = None
marker: CitationMarker | None = None
+ # The English text the models check, when it is not ``text`` itself: a claim in
+ # another language that the judge translated (OPEN-ITEMS 17.1a). ``text`` stays
+ # the reader's own sentence, and that is what the report shows.
+ hypothesis: str | None = None
def __post_init__(self) -> None:
if self.sentence < 0:
diff --git a/src/proofpath/eval/averitec.py b/src/proofpath/eval/averitec.py
index ac16547..9c07ca7 100644
--- a/src/proofpath/eval/averitec.py
+++ b/src/proofpath/eval/averitec.py
@@ -65,6 +65,8 @@ class Claim:
# Counted rather than dropped: a run has to be able to say it never tried them,
# instead of reporting them as web sources it failed to reach (product rule 2).
non_urls: int = 0
+ # The article that settled the claim: the answer key, never a search result.
+ fact_check_url: str = ""
def to_label(label: str) -> Label | None:
@@ -127,6 +129,7 @@ def load(path: Path, *, limit: int | None = None, expected_sha256: str = SHA256)
label=str(row["label"]),
source_urls=urls,
non_urls=non_urls,
+ fact_check_url=str(row.get("fact_checking_article") or ""),
)
)
return claims
diff --git a/src/proofpath/events.py b/src/proofpath/events.py
index 7dbce65..05d4ab3 100644
--- a/src/proofpath/events.py
+++ b/src/proofpath/events.py
@@ -57,6 +57,10 @@ class Note:
#: A line that describes something in progress and is taken back when it is over
#: (the TUI removes it; the CLI logs it).
transient: bool = False
+ #: A line the user must see whatever the output mode: the judge switching to its
+ #: local fallback (spec section 11). The CLI prints it on stderr and never drops
+ #: it, not even under ``-q``; the TUI shows it above the prompt until the next run.
+ notice: bool = False
@dataclass(frozen=True)
diff --git a/src/proofpath/judge.py b/src/proofpath/judge.py
index 9015099..66e0582 100644
--- a/src/proofpath/judge.py
+++ b/src/proofpath/judge.py
@@ -17,10 +17,13 @@
import importlib.resources
import json
+import re
import string
+import threading
import time
from collections.abc import Callable, Sequence
-from dataclasses import dataclass
+from contextlib import suppress
+from dataclasses import dataclass, replace
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from types import TracebackType
@@ -29,7 +32,7 @@
import httpx
from proofpath import __version__
-from proofpath.config import JudgeConfig
+from proofpath.config import LOCAL_JUDGE_MODEL, JudgeConfig
from proofpath.models import Label, Tier
from proofpath.secrets import ApiKey, default_dotenv_paths, read_dotenv, resolve_api_key
@@ -49,9 +52,27 @@ class JudgeError(ValueError):
class JudgeUnavailable(JudgeError): # noqa: N818 - reads as a state, not as a failed run
- """The provider did not answer after retries. A run reports this, never fails on it."""
+ """The provider did not answer after retries. A run reports this, never fails on it.
+
+ ``status`` is the HTTP status of the last answer, or ``None`` when there was none
+ (a transport error, an unreadable 200). ``cause`` is the same fact in a few words
+ a notice can print (``HTTP 500``, ``timeout``, ``empty answer``): never a body,
+ never a URL, never a key. The fallback reads both to say why it switched without
+ parsing its own message back.
+ """
+
+ def __init__(self, detail: str, *, status: int | None = None, cause: str = "") -> None:
+ super().__init__(detail)
+ self.status = status
+ self.cause = cause or (f"HTTP {status}" if status is not None else "no answer")
+#: ``think: false`` in Ollama's OpenAI-compatible terms, sent to every Ollama model
+#: whatever the caller asked for: qwen3.5:9b on ``low`` spent a 1500-token budget
+#: thinking and answered nothing, where ``none`` answered in 1.6 s (Ollama 0.34.4,
+#: measured 2026-09-28). A 400 naming the field still drops it.
+LOCAL_REASONING = "none"
+
# Free, no card, OpenAI-compatible (verified 2026-09-11). Ollama is the offline default.
_PROVIDERS: dict[str, JudgeConfig] = {
"groq": JudgeConfig(),
@@ -61,9 +82,10 @@ class JudgeUnavailable(JudgeError): # noqa: N818 - reads as a state, not as a f
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
api_key_env="GEMINI_API_KEY",
),
+ # One local model for everything: the same one the fallback uses (spec section 11).
"ollama": JudgeConfig(
provider="ollama",
- model="llama3.1",
+ model=LOCAL_JUDGE_MODEL,
base_url="http://localhost:11434/v1",
api_key_env="",
),
@@ -164,6 +186,10 @@ class JudgeCost:
completion_tokens: int = 0
waited_s: float = 0.0
model: str = ""
+ #: Of ``calls``, the answers the local fallback gave (spec section 11). Counted
+ #: apart because ``api_calls`` promises provider requests, and a report that
+ #: folded local answers into it would overstate what the free tier was spent on.
+ local_calls: int = 0
# Three attempts per request: Groq's free tier answers roughly once a minute, so a
@@ -194,6 +220,31 @@ def _retry_after_seconds(value: str) -> float | None:
return max(0.0, (when - datetime.now(timezone.utc)).total_seconds())
+# A thinking model that ignores the "no thinking" request can still open its answer
+# with its reasoning. Only a *leading*, *closed* block is taken off: anything else is
+# the answer itself, and guessing at it would be worse than reporting it unreadable.
+_THINK = re.compile(r"\A\s*.*?\s*", re.DOTALL)
+_CLOSE = ""
+
+
+def _strip_think(text: str) -> str:
+ """The answer without a leading reasoning block.
+
+ Some chat templates open the block in the prompt, so the answer carries only its
+ end (``...reasoning...{json}``). That orphan end is taken off only when
+ the answer does not already start as JSON and JSON follows it: a valid answer
+ that merely mentions ```` inside a string must reach the reader intact.
+ """
+ if _THINK.match(text):
+ return _THINK.sub("", text, count=1)
+ head = text.lstrip()[:1]
+ end = text.find(_CLOSE)
+ if end < 0 or head in ("{", "["):
+ return text
+ rest = text[end + len(_CLOSE) :].lstrip()
+ return rest if rest[:1] in ("{", "[") else text
+
+
def _blames_response_format(response: httpx.Response) -> bool:
"""True when a 400 is about ``response_format``. The body is read, never printed."""
body = response.text.lower()
@@ -229,7 +280,10 @@ def __init__(
client: httpx.Client | None = None,
sleep: Callable[[float], None] = time.sleep,
max_wait: float = 120.0,
- timeout: float = 60.0,
+ timeout: float | httpx.Timeout = 60.0,
+ fail_fast: bool = False,
+ force_reasoning: str | None = None,
+ retry_timeouts: bool = True,
) -> None:
if config.api_key_env and key is None:
raise JudgeError(_no_key_detail(config.api_key_env))
@@ -249,6 +303,22 @@ def __init__(
# 400 again on each of them, against a per-minute tier with no room for it.
self._schema_refused = False
self._effort_refused = False
+ #: With a fallback behind this client, waiting is the bug (Amendment B 1): the
+ #: first failure raises at once, no retry and no ``Retry-After``. Public and
+ #: mutable because ``FallbackClient`` only knows whether a fallback exists once
+ #: it has asked Ollama, which it does on the first call, not at construction.
+ self.fail_fast = fail_fast
+ # Replaces whatever reasoning effort a caller asks for. The local fallback
+ # sends ``none``: Ollama maps it to ``think: false``, the one value every
+ # model accepts, where ``low`` is a 400 on a model that cannot think and a
+ # budget spent thinking on one that can (measured on Ollama 0.34.4). Every
+ # Ollama client gets it, the direct ``judge.provider ollama`` one included.
+ if force_reasoning is None and config.provider == "ollama":
+ force_reasoning = LOCAL_REASONING
+ self._force_reasoning = force_reasoning
+ # A local model that timed out after 180 s is loading or stuck; asking twice
+ # more would hold the run for nine minutes (fix round 1, D4).
+ self._retry_timeouts = retry_timeouts
self.cost = JudgeCost(model=config.model)
def __repr__(self) -> str:
@@ -270,6 +340,19 @@ def model(self) -> str:
"""
return self._config.model
+ @property
+ def timeout(self) -> float | httpx.Timeout:
+ """How long one request may take before it counts as unanswered.
+
+ Settable per client, because ``FallbackClient`` only learns that a fallback
+ exists on the first call, and only then may it shorten the primary's wait.
+ """
+ return self._timeout
+
+ @timeout.setter
+ def timeout(self, value: float | httpx.Timeout) -> None:
+ self._timeout = value
+
def close(self) -> None:
"""Release the HTTP client, but only one this client built for itself.
@@ -320,10 +403,14 @@ def complete(
"json_schema": {"name": _SCHEMA_NAME, "schema": json_schema, "strict": True},
}
)
+ if self._force_reasoning is not None:
+ reasoning_effort = self._force_reasoning
if self._effort_refused:
reasoning_effort = None
attempt = 0
last = "no request was made"
+ status: int | None = None
+ cause = ""
while attempt < _MAX_ATTEMPTS:
payload: dict[str, Any] = {
"model": self._config.model,
@@ -341,13 +428,19 @@ def complete(
)
except httpx.HTTPError as exc:
last = f"request failed: {type(exc).__name__}"
+ status = None
+ timed_out = isinstance(exc, httpx.TimeoutException)
+ cause = "timeout" if timed_out else "connection failed"
delay = _backoff(attempt)
+ if timed_out and not self._retry_timeouts:
+ break
else:
status = response.status_code
if status == 200:
return self._record(response, messages)
# Never echo the body: providers put the key or account details in it.
last = f"HTTP {status} from {self._url}"
+ cause = f"HTTP {status}"
if (
status == 400
and response_format is not None
@@ -381,10 +474,14 @@ def complete(
delay = _backoff(attempt)
else:
break
+ if self.fail_fast:
+ # Checked after the two downgrades above, which ``continue`` before
+ # reaching here: a model refusing a field is not a model that is down.
+ break
attempt += 1
if attempt < _MAX_ATTEMPTS:
self._wait(delay)
- raise JudgeUnavailable(last)
+ raise JudgeUnavailable(last, status=status, cause=cause)
def _wait(self, delay: float) -> None:
delay = min(delay, self._max_wait)
@@ -399,7 +496,7 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
choice = choices[0] if choices else {}
content = choice.get("message", {}).get("content")
finish_reason = choice.get("finish_reason")
- text = content if isinstance(content, str) else ""
+ text = _strip_think(content) if isinstance(content, str) else ""
usage = body.get("usage") or {}
# A missing count is estimated; an honest zero is kept exactly as sent.
reported_prompt = usage.get("prompt_tokens")
@@ -416,7 +513,9 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
except (AttributeError, IndexError, KeyError, TypeError, ValueError):
# A 200 that is not a completion is a bot wall or a proxy page. Its shape
# is not ours to guess at, and its body is never repeated back.
- raise JudgeUnavailable(f"unreadable 200 response from {self._url}") from None
+ raise JudgeUnavailable(
+ f"unreadable 200 response from {self._url}", status=200, cause="unreadable answer"
+ ) from None
if not text:
# No choices, or a choice with no text: a refusal, a cut-off answer or a
# wall, never an empty opinion. It costs the caller a `JudgeUnavailable`,
@@ -424,7 +523,11 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
# `finish_reason` is a fixed provider token, so it may be named; the body
# may not.
why = f" (finish_reason={finish_reason})" if isinstance(finish_reason, str) else ""
- raise JudgeUnavailable(f"no completion in the 200 response from {self._url}{why}")
+ raise JudgeUnavailable(
+ f"no completion in the 200 response from {self._url}{why}",
+ status=200,
+ cause="empty answer",
+ )
self.cost.calls += 1
self.cost.prompt_tokens += prompt_tokens
self.cost.completion_tokens += completion_tokens
@@ -437,6 +540,394 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
)
+# --- the local fallback (spec section 11) -----------------------------------------
+
+#: The switch notices (spec section 11; the cause added in fix round 1). ``provider``
+#: is the name as a reader knows it (``Groq``), ``model`` the Ollama tag the run
+#: switched to. Every failure switches; the notice says which one it was, so a wrong
+#: key is never mistaken for a busy free tier.
+JUDGE_FALLBACK_LIMIT = "{provider} limit reached — judging with local ollama {model}"
+JUDGE_FALLBACK_KEY = (
+ "{provider} rejected the key (HTTP {status}) — judging with local ollama {model}"
+)
+JUDGE_FALLBACK_DOWN = "{provider} did not answer ({cause}) — judging with local ollama {model}"
+_LIMIT_STATUSES = frozenset({429, 413}) # Groq answers 413 when a request busts TPM
+_KEY_STATUSES = frozenset({401, 403})
+#: ``judge.fallback``: the Ollama model to switch to (``LOCAL_JUDGE_MODEL`` unless the
+#: user named another), or ``off``, in any case, for no fallback.
+FALLBACK_OFF = "off"
+#: The first local call loads the model into memory, which can take tens of seconds
+#: on first use, so the local client waits far longer than an API client does.
+LOCAL_TIMEOUT = 180.0
+#: How long Ollama keeps the model loaded after the warm-up. Sent on the native
+#: ``/api/generate`` route because the OpenAI-compatible one ignores ``keep_alive``
+#: (Ollama 0.34.4, measured 2026-09-28); later requests keep the session's value.
+LOCAL_KEEP_ALIVE = "30m"
+#: ``/api/tags`` is a local listing; a server that takes longer than this is not
+#: one to hand a run to.
+TAGS_TIMEOUT = 2.0
+#: The primary's wait once a fallback exists: a host that does not even accept the
+#: connection within five seconds is not going to answer, and the local model is.
+PRIMARY_CONNECT_TIMEOUT = 5.0
+PRIMARY_READ_TIMEOUT = 60.0
+#: How long a thread that did not switch waits for the switching thread's notice and
+#: warm-up: the warm-up's own timeout plus a margin, so its request never starts its
+#: read timer while the model is still loading, and a hung warm-up cannot hold it
+#: for ever.
+WARM_WAIT_S = LOCAL_TIMEOUT + 5.0
+# The local failures that mean the model is gone for this run: no answer at all
+# (``status`` None: refused, reset, timed out), a server error, or no such model.
+# A cut-off, unreadable or refused *answer* is about that one request, not the model.
+_DEAD_STATUSES = frozenset({404})
+
+_DISPLAY_NAMES = {"groq": "Groq", "gemini": "Gemini", "ollama": "Ollama"}
+# Why there is no fallback, in the words the error carries after "no local fallback:".
+_WHY_OFF = "judge.fallback is off"
+
+
+def _ollama_url() -> str:
+ return provider_defaults("ollama").base_url
+
+
+def _ollama_root(base_url: str) -> str:
+ """The server root: the native API lives beside ``/v1``, not under it."""
+ root = base_url.rstrip("/")
+ return root[: -len("/v1")] if root.endswith("/v1") else root
+
+
+def _display(provider: str) -> str:
+ return _DISPLAY_NAMES.get(provider, provider.capitalize())
+
+
+def _tags(base_url: str, *, timeout: float) -> list[str] | None:
+ """The installed model names, or ``None`` when Ollama could not be asked."""
+ try:
+ response = httpx.get(f"{_ollama_root(base_url)}/api/tags", timeout=timeout)
+ if response.status_code != 200:
+ return None
+ models = response.json().get("models") or []
+ return [str(item["name"]) for item in models if isinstance(item, dict) and "name" in item]
+ except (httpx.HTTPError, ValueError, AttributeError, TypeError):
+ return None
+
+
+def _installed(model: str, names: Sequence[str]) -> bool:
+ """Whether ``model`` is among ``names``; an untagged name means its ``:latest``."""
+ wanted = model.strip().lower()
+ if ":" not in wanted:
+ wanted += ":latest"
+ return any(name.lower() == wanted for name in names)
+
+
+def installed_models(base_url: str, *, timeout: float) -> list[str]:
+ """``GET {root}/api/tags``: what Ollama has installed. ``[]`` on any error."""
+ return _tags(base_url, timeout=timeout) or []
+
+
+def local_client(model: str, base_url: str | None = None) -> JudgeClient:
+ """The fallback's client: Ollama's OpenAI route, no key, thinking off, a timeout
+ long enough for the first call to load the model, and no second try after it."""
+ settings = replace(provider_defaults("ollama"), model=model)
+ if base_url is not None:
+ settings = replace(settings, base_url=base_url)
+ return JudgeClient(
+ settings,
+ None,
+ timeout=LOCAL_TIMEOUT,
+ force_reasoning=LOCAL_REASONING,
+ retry_timeouts=False,
+ )
+
+
+def warm_local(base_url: str, model: str) -> None:
+ """Load ``model`` and keep it loaded for ``LOCAL_KEEP_ALIVE``. Never raises.
+
+ An empty ``/api/generate`` is Ollama's documented "load this model" request. A
+ warm-up that fails costs nothing: the chat request after it loads the model
+ itself, and says so if it cannot.
+ """
+ url = f"{_ollama_root(base_url)}/api/generate"
+ try:
+ httpx.post(
+ url, json={"model": model, "keep_alive": LOCAL_KEEP_ALIVE}, timeout=LOCAL_TIMEOUT
+ )
+ except httpx.HTTPError:
+ return
+
+
+@dataclass(frozen=True)
+class _Plan:
+ """Whether this run has a fallback: the model, or why there is none."""
+
+ model: str | None
+ why: str = ""
+
+
+class FallbackClient:
+ """A primary ``JudgeClient`` that switches, once and at once, to a local model.
+
+ The same surface ``Judge`` uses from ``JudgeClient``. The first call decides
+ whether a fallback exists (``judge.fallback``, and one cheap ``/api/tags`` to
+ confirm that model is installed). With one, the primary fails fast and connects within
+ ``PRIMARY_CONNECT_TIMEOUT``: its first failure of any kind raises, the notice
+ goes out naming the cause, the model is warmed up and the very request that
+ failed is sent to it, so the caller sees one completion. Every later call goes
+ straight to the local model. Without a fallback, the primary keeps its retries
+ and its error says why no fallback was used. A local model that fails too is
+ latched dead for the run: every later call raises at once with the same detail.
+
+ The switch is sticky for the life of the client, which is one run: whatever
+ builds a judge for a run builds a new client or calls :meth:`reset`.
+ """
+
+ def __init__(
+ self,
+ primary: JudgeClient,
+ *,
+ fallback: str = LOCAL_JUDGE_MODEL,
+ on_switch: Callable[[str], None] | None = None,
+ ollama_url: str | None = None,
+ local_factory: Callable[[str, str], JudgeClient] = local_client,
+ warm: Callable[[str, str], None] | None = warm_local,
+ ) -> None:
+ self._primary = primary
+ self._primary_timeout = primary.timeout
+ self._fallback = fallback.strip()
+ self._ollama_url = ollama_url or _ollama_url()
+ self._local_factory = local_factory
+ self._warm = warm
+ self.on_switch = on_switch
+ #: The notice, once the switch has happened; ``None`` before.
+ self.switched: str | None = None
+ # Worker threads share one client (the TUI's summary, a run's stages): the
+ # plan and the switch go through this lock so that exactly one thread
+ # switches and every other one sees the result.
+ self._lock = threading.Lock()
+ # Set once the switching thread has handed the notice over and the warm-up
+ # has finished (or failed). A thread that finds the switch already made waits
+ # for it, so no local request starts before the notice is out, and none starts
+ # its read timer while the model is still loading.
+ self._ready = threading.Event()
+ self._plan: _Plan | None = None
+ self._local: JudgeClient | None = None
+ self._primary_detail = ""
+ #: The combined detail once the local model has failed too; ``None`` before.
+ self._dead: str | None = None
+ # Local answers from before a ``reset``: the client is gone, the spend is not.
+ self._spent = JudgeCost()
+
+ def __repr__(self) -> str:
+ return (
+ f"FallbackClient(primary={self._primary!r}, fallback={self._fallback!r}, "
+ f"switched={self.switched is not None})"
+ )
+
+ __str__ = __repr__
+
+ @property
+ def provider(self) -> str:
+ return "ollama" if self._local is not None else self._primary.provider
+
+ @property
+ def model(self) -> str:
+ return self._local.model if self._local is not None else self._primary.model
+
+ @property
+ def cost(self) -> JudgeCost:
+ """Primary and local spend together; ``local_calls`` says which were local."""
+ primary = self._primary.cost
+ parts = [primary, self._spent]
+ if self._local is not None:
+ parts.append(self._local.cost)
+ local = self._spent.calls + (self._local.cost.calls if self._local is not None else 0)
+ return JudgeCost(
+ calls=sum(part.calls for part in parts),
+ prompt_tokens=sum(part.prompt_tokens for part in parts),
+ completion_tokens=sum(part.completion_tokens for part in parts),
+ waited_s=sum(part.waited_s for part in parts),
+ model=self._local.cost.model if self._local is not None else primary.model,
+ local_calls=local,
+ )
+
+ def reset(self) -> None:
+ """Back to the primary, for a new run: the notice, the plan and a dead local
+ model are all forgotten."""
+ with self._lock:
+ if self._local is not None:
+ spent = self._local.cost
+ self._spent.calls += spent.calls
+ self._spent.prompt_tokens += spent.prompt_tokens
+ self._spent.completion_tokens += spent.completion_tokens
+ self._spent.waited_s += spent.waited_s
+ self._local.close()
+ self._local = None
+ self._plan = None
+ self._dead = None
+ self.switched = None
+ self._primary_detail = ""
+ self._ready = threading.Event()
+ self._primary.fail_fast = False
+ self._primary.timeout = self._primary_timeout
+
+ def close(self) -> None:
+ self._primary.close()
+ if self._local is not None:
+ self._local.close()
+
+ def __enter__(self) -> FallbackClient:
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
+ self.close()
+
+ def complete(
+ self,
+ messages: list[dict[str, str]],
+ *,
+ json_schema: dict[str, Any] | None = None,
+ max_tokens: int = 1024,
+ temperature: float = 0.0,
+ reasoning_effort: str | None = None,
+ ) -> Completion:
+ """One completion, from the primary or, once it has failed, from Ollama."""
+
+ def ask(client: JudgeClient) -> Completion:
+ return client.complete(
+ messages,
+ json_schema=json_schema,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ reasoning_effort=reasoning_effort,
+ )
+
+ local = self._route()
+ if local is None:
+ try:
+ return ask(self._primary)
+ except JudgeUnavailable as exc:
+ local, notice = self._switch(exc)
+ if notice is not None:
+ try:
+ self._announce(notice)
+ if self._warm is not None:
+ # After the notice, before the request: the user reads the
+ # line while the model loads (Amendment B 4). Outside the lock.
+ self._warm(self._ollama_url, local.model)
+ finally:
+ self._ready.set()
+ # A thread that did not switch waits for the notice and the warm-up, bounded
+ # by the warm-up's own timeout (fix round 2).
+ self._ready.wait(timeout=WARM_WAIT_S)
+ if self._dead is not None:
+ raise JudgeUnavailable(self._dead)
+ try:
+ return ask(local)
+ except JudgeUnavailable as exc:
+ # The primary's reason first: it is what the run switched away from, and
+ # the search's own notice reads "HTTP 429" out of it (spec section 11).
+ detail = f"{self._primary_detail}; local ollama {local.model}: {exc}"
+ if exc.status is None or exc.status >= 500 or exc.status in _DEAD_STATUSES:
+ # Latched: a dead or stuck Ollama costs this run one wait, not one per
+ # batch. Anything else failed this request only (fix round 2).
+ self._dead = detail
+ raise JudgeUnavailable(detail, status=exc.status, cause=exc.cause) from None
+
+ def _announce(self, notice: str) -> None:
+ """Hand the notice to the listener, once, and never at the answer's expense."""
+ if self.on_switch is not None:
+ # A listener that raises costs the line, not the batch (fix round 1).
+ with suppress(Exception):
+ self.on_switch(notice)
+
+ def _route(self) -> JudgeClient | None:
+ """The local client once switched; otherwise ``None``, with the plan made."""
+ with self._lock:
+ if self._local is not None:
+ return self._local
+ if self._plan is None:
+ self._plan = self._choose()
+ if self._plan.model is not None:
+ # Decided before the first request (Amendment B 1): with a
+ # fallback the primary must not wait, neither on a retry nor on a
+ # host that does not answer the connect. Without one it keeps
+ # today's retries and timeout.
+ self._primary.fail_fast = True
+ self._primary.timeout = httpx.Timeout(
+ PRIMARY_READ_TIMEOUT, connect=PRIMARY_CONNECT_TIMEOUT
+ )
+ return None
+
+ def _choose(self) -> _Plan:
+ """The configured model, once ``/api/tags`` confirms it is installed.
+
+ The tags are asked only to confirm, never to choose: the order is fixed
+ (the configured API, then this model) and nothing is ever pulled (rule 5).
+ """
+ setting = self._fallback
+ if not setting or setting.lower() == FALLBACK_OFF:
+ return _Plan(None, _WHY_OFF)
+ names = _tags(self._ollama_url, timeout=TAGS_TIMEOUT)
+ if names is None:
+ return _Plan(None, f"ollama is not reachable at {_ollama_root(self._ollama_url)}")
+ if not _installed(setting, names):
+ return _Plan(None, f"{setting} is not installed")
+ return _Plan(setting) # as written: the user's own spelling of the name
+
+ def _switch(self, failure: JudgeUnavailable) -> tuple[JudgeClient, str | None]:
+ """Switch once: the local client, and the notice when *this* call switched.
+
+ Raises the primary's error, with the reason, when there is no fallback. The
+ notice is returned rather than sent from here: the listener runs after the
+ lock is released, so a slow or blocking listener can never hold another
+ thread's switch (fix round 1, C).
+ """
+ with self._lock:
+ if self._local is not None:
+ return self._local, None # another thread switched while we waited
+ plan = self._plan
+ if plan is None or plan.model is None:
+ why = _WHY_OFF if plan is None else plan.why
+ raise JudgeUnavailable(
+ f"{failure}; no local fallback: {why}",
+ status=failure.status,
+ cause=failure.cause,
+ ) from None
+ notice = _notice(self._primary.provider, plan.model, failure)
+ self._primary_detail = str(failure)
+ self._local = self._local_factory(plan.model, self._ollama_url)
+ self.switched = notice
+ return self._local, notice
+
+
+def _notice(provider: str, model: str, failure: JudgeUnavailable) -> str:
+ """The switch notice, naming why the primary was left (fix round 1, B)."""
+ name = _display(provider)
+ if failure.status in _LIMIT_STATUSES:
+ return JUDGE_FALLBACK_LIMIT.format(provider=name, model=model)
+ if failure.status in _KEY_STATUSES:
+ return JUDGE_FALLBACK_KEY.format(provider=name, status=failure.status, model=model)
+ return JUDGE_FALLBACK_DOWN.format(provider=name, cause=failure.cause, model=model)
+
+
+def build_client(
+ config: JudgeConfig,
+ key: ApiKey | None,
+ *,
+ on_switch: Callable[[str], None] | None = None,
+) -> JudgeClient | FallbackClient:
+ """The judge's client for one run: the configured provider, with the local
+ fallback behind it unless the provider already is Ollama (spec section 11)."""
+ primary = JudgeClient(config, key)
+ if config.provider == "ollama":
+ return primary
+ return FallbackClient(primary, fallback=config.fallback, on_switch=on_switch)
+
+
# --- the batching judge -----------------------------------------------------------
@@ -468,6 +959,14 @@ class JudgeItem:
tier: Tier
+@dataclass(frozen=True)
+class JudgedClaim:
+ """What the judge wrote for one claim: search queries, and the claim in English."""
+
+ queries: tuple[str, ...]
+ english: str # "" when the model gave none
+
+
# 20 claims per prompt, and a prompt under 3.5k tokens: Groq's free tier caps at 8k
# tokens per minute and the cap below counts the *prompt* only, so it has to leave
# room for ``_REVIEW_TOKENS`` on top of it -- 3500 + 4096 still fits under 8k, where
@@ -515,6 +1014,32 @@ class JudgeItem:
"additionalProperties": False,
}
+# Evidence search (OPEN-ITEMS 17.1a): the judge writes *queries*, never an answer.
+# Deciding a claim without a passage is the truth oracle spec section 3 forbids.
+_QUERIES_TOKENS = 1500
+_QUERIES_PER_CLAIM = 2
+_QUERY_CHARS = 400
+_QUERIES_SCHEMA: dict[str, Any] = {
+ "type": "object",
+ "properties": {
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {"type": "integer"},
+ "english": {"type": "string"},
+ "queries": {"type": "array", "items": {"type": "string"}},
+ },
+ "required": ["id", "english", "queries"],
+ "additionalProperties": False,
+ },
+ }
+ },
+ "required": ["items"],
+ "additionalProperties": False,
+}
+
_SYSTEM_HEADING = "## System"
_USER_HEADING = "## User"
@@ -557,7 +1082,7 @@ class Judge:
def __init__(
self,
- client: JudgeClient,
+ client: JudgeClient | FallbackClient,
*,
batch_size: int = BATCH_SIZE,
token_cap: int = TOKEN_CAP,
@@ -567,6 +1092,7 @@ def __init__(
self._token_cap = token_cap
self._template = load_prompt("review")
self._summary_template = load_prompt("summarize")
+ self._queries_template = load_prompt("queries")
#: One line per opinion that could not be used, in the order they arrived.
self.skipped: list[str] = []
#: True once the provider stopped answering; ``detail`` says why.
@@ -583,6 +1109,26 @@ def cost(self) -> JudgeCost:
"""What the judge has spent so far. The client keeps the running total."""
return self._client.cost
+ @property
+ def switched(self) -> str | None:
+ """The local-fallback notice once this run has switched, else ``None``."""
+ return self._client.switched if isinstance(self._client, FallbackClient) else None
+
+ def listen(self, on_switch: Callable[[str], None] | None) -> None:
+ """Point the switch notice at whoever is watching this run.
+
+ Set per run rather than when the judge is built: the judge is built before
+ the run's listener exists, and a stage that runs later (the summary) may
+ report to another one.
+ """
+ if isinstance(self._client, FallbackClient):
+ self._client.on_switch = on_switch
+
+ def reset(self) -> None:
+ """A new run starts on the configured provider again (Amendment B 9)."""
+ if isinstance(self._client, FallbackClient):
+ self._client.reset()
+
def close(self) -> None:
self._client.close()
@@ -678,6 +1224,41 @@ def summarize(self, report_markdown: str, *, max_tokens: int = _SUMMARY_TOKENS)
return ""
return text
+ def queries(self, claims: Sequence[str]) -> dict[int, JudgedClaim]:
+ """One or two search queries per claim, and the claim in English, keyed by
+ its position; ``{}`` on trouble.
+
+ It never raises, like ``review``: a provider that is down or out of quota
+ costs the written queries and the translation, and ``unavailable``/``detail``
+ say why, so the run can fall back to the sentence itself and tell the reader
+ (OPEN-ITEMS 17.1a). The English text is what Amendment A hands the check
+ models, which were trained on English alone.
+ """
+ self.unavailable = False
+ self.detail = ""
+ if not claims:
+ return {}
+ items = "\n".join(
+ f"- id: {index}\n claim: {_one_line(text)}" for index, text in enumerate(claims)
+ )
+ system, user = _split_prompt(self._queries_template.substitute(items=items))
+ messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+ try:
+ completion = self._client.complete(
+ messages,
+ json_schema=_QUERIES_SCHEMA,
+ max_tokens=_QUERIES_TOKENS,
+ reasoning_effort=_REASONING_EFFORT,
+ )
+ except JudgeUnavailable as exc:
+ self._give_up(str(exc))
+ return {}
+ except Exception as exc:
+ # The type is named, the message is not: it can carry the key.
+ self._give_up(f"{type(exc).__name__} from the judge client")
+ return {}
+ return _queries_from(completion.text, len(claims))
+
def _give_up(self, detail: str) -> None:
self.unavailable = True
self.detail = detail
@@ -778,6 +1359,42 @@ def _loads(text: str) -> list[Any] | None:
return None
+def _queries_from(text: str, count: int) -> dict[int, JudgedClaim]:
+ """The ``items`` of a queries answer, read leniently; an id out of range is dropped."""
+ for candidate in _json_candidates(text):
+ try:
+ payload = json.loads(candidate)
+ except ValueError:
+ continue
+ if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
+ continue
+ found: dict[int, JudgedClaim] = {}
+ for entry in payload["items"]:
+ if not isinstance(entry, dict):
+ continue
+ try:
+ ident = int(entry.get("id"))
+ except (TypeError, ValueError):
+ continue
+ if not 0 <= ident < count:
+ continue
+ raw = entry.get("queries")
+ written = (
+ tuple(
+ _one_line(query)[:_QUERY_CHARS]
+ for query in raw
+ if isinstance(query, str) and query.strip()
+ )[:_QUERIES_PER_CLAIM]
+ if isinstance(raw, list)
+ else ()
+ )
+ english = _one_line(str(entry.get("english", "")))[:_QUERY_CHARS]
+ if written or english:
+ found[ident] = JudgedClaim(queries=written, english=english)
+ return found
+ return {}
+
+
def _json_candidates(text: str) -> list[str]:
stripped = text.strip()
start, end = stripped.find("{"), stripped.rfind("}")
diff --git a/src/proofpath/polite.py b/src/proofpath/polite.py
index acfcb72..7cace9d 100644
--- a/src/proofpath/polite.py
+++ b/src/proofpath/polite.py
@@ -50,6 +50,10 @@
"lobste.rs": 0.5,
"www.reddit.com": 1.0,
"oauth.reddit.com": 1.0,
+ # Evidence search (OPEN-ITEMS 17.1a): a run asks at most a handful of queries, and
+ # the user's own key is what they spend. SearXNG is the user's own instance, so it
+ # is not listed and gets the default.
+ "api.tavily.com": 0.5,
}
MAX_RETRY_AFTER = 60.0
RETRYABLE = (429, 500, 502, 503, 504)
@@ -217,17 +221,18 @@ def post(
url: str,
*,
data: dict[str, str] | None = None,
+ json_body: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
auth: tuple[str, str] | None = None,
) -> httpx.Response:
- """A form POST, throttled and retried like :meth:`get`.
+ """A POST, throttled and retried like :meth:`get`.
- Only one caller needs it: the OAuth token exchange a user's own Reddit app
- makes (spec section 6.2). ``auth`` is HTTP basic and ``data`` is the form
- body, so neither ever reaches the URL — a credential does not belong in a
- query string, which is the part of a request that gets logged.
+ Two callers: the OAuth token exchange a user's own Reddit app makes (spec
+ section 6.2), which sends a form in ``data``, and the evidence search, which
+ sends JSON in ``json_body``. A credential rides in ``auth`` or ``headers`` and
+ never in the URL: a query string is the part of a request that gets logged.
"""
- return self._send("POST", url, data=data, headers=headers, auth=auth)
+ return self._send("POST", url, data=data, json_body=json_body, headers=headers, auth=auth)
def _send(
self,
@@ -236,6 +241,7 @@ def _send(
*,
params: dict[str, Any] | None = None,
data: dict[str, str] | None = None,
+ json_body: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
auth: tuple[str, str] | None = None,
) -> httpx.Response:
@@ -257,6 +263,7 @@ def _send(
url,
params=params,
data=data,
+ json=json_body,
headers=headers,
auth=auth or httpx.USE_CLIENT_DEFAULT,
)
diff --git a/src/proofpath/prompts/queries.md b/src/proofpath/prompts/queries.md
new file mode 100644
index 0000000..ed41dbc
--- /dev/null
+++ b/src/proofpath/prompts/queries.md
@@ -0,0 +1,40 @@
+## System
+
+You write web search queries for a fact-checking tool. You never say whether a claim
+is true: another stage reads the pages your queries find and decides that from their
+own words. Your only job is queries that would find a page stating the facts the
+claim is about.
+
+## User
+
+For each claim below, write one or two search queries in English, and the claim
+itself in English.
+
+Rules:
+
+- Make each query stand alone: replace pronouns and vague references ("he", "the
+ company", "yesterday") with what the claim itself names, when it names it.
+- Keep the claim's names, numbers and dates; drop opinion words and emotion.
+- Plain keywords or a short sentence, at most 25 words, in English.
+- Add no fact, answer or source that is not in the claim.
+- `english`: the claim translated into English, as close to word for word as
+ English allows. Never copy a claim that is not in English unchanged: translate
+ it. If it is already English, repeat it unchanged. Add nothing, soften nothing.
+
+Answer with JSON only: {"items": [{"id": , "english": "...", "queries": ["...", "..."]}]}
+
+Examples (they show the shape only; they are not claims to answer, never copy them
+into your answer):
+
+Example claim E1, not in English: Şirket battı.
+Example answer: {"items": [{"id": "E1", "english": "The company went bankrupt.", "queries": ["company went bankrupt", "company bankruptcy filing"]}]}
+
+Example claim E2, already in English: The Eiffel Tower is in Paris.
+Example answer: {"items": [{"id": "E2", "english": "The Eiffel Tower is in Paris.", "queries": ["Eiffel Tower location Paris"]}]}
+
+In E1 the claim was translated, verb and all ("battı" is "went bankrupt"), not
+copied; in E2 the English was kept word for word.
+
+Claims:
+
+$items
diff --git a/src/proofpath/report.py b/src/proofpath/report.py
index 14a8273..8224bcc 100644
--- a/src/proofpath/report.py
+++ b/src/proofpath/report.py
@@ -50,6 +50,25 @@
# Prefix shared by every state in the UNVERIFIED family (spec section 15).
UNVERIFIED_PREFIX = "UNVERIFIED"
+# Evidence search (OPEN-ITEMS 17.1a). The provenance note every finding about a found
+# page carries, so no renderer can show one as a source the author cited.
+FOUND_BY_PROOFPATH = "FOUND BY PROOFPATH (not cited by the author)"
+SEARCH_UNAVAILABLE = f"{UNVERIFIED_PREFIX} (search unavailable)"
+# A claim in another language that no judge translated: the models read English only,
+# so it is not searched, and it is not called NEI either (OPEN-ITEMS 17.1a).
+LANGUAGE_UNSUPPORTED = f"{UNVERIFIED_PREFIX} (language not supported)"
+# The §17.1 gate: printed on every run that searched until the eval beats the
+# majority baseline. Updated by hand from docs/eval/*-averitec-search.md.
+AVERITEC_SEARCH_SCORE = "not measured yet"
+AVERITEC_BASELINE = "0.708"
+
+
+def search_experimental() -> str:
+ return (
+ f"evidence search is experimental — AVeriTeC search-mode {AVERITEC_SEARCH_SCORE} "
+ f"vs {AVERITEC_BASELINE} majority baseline"
+ )
+
def no_bibliography(markers: int) -> str:
"""What a run says when it found citations but no reference list to check them
@@ -75,6 +94,8 @@ class Kind(str, Enum):
UNRESOLVED_MARKER = "unresolved-marker"
PARSE_ERROR = "parse-error"
PROVIDER_UNAVAILABLE = "provider-unavailable"
+ EVIDENCE_FOUND = "evidence-found" # a page the search found supports the claim
+ NO_EVIDENCE = "no-evidence-found" # the search found nothing that decides the claim
Level = Literal["error", "warning", "note"]
@@ -94,10 +115,12 @@ class Kind(str, Enum):
Kind.UNRESOLVED_MARKER: "warning",
Kind.PARSE_ERROR: "warning",
Kind.PROVIDER_UNAVAILABLE: "warning",
+ Kind.NO_EVIDENCE: "warning",
Kind.NEI: "note",
Kind.ABSTRACT_ONLY: "note",
Kind.PARAGRAPH_SCOPED: "note",
Kind.UNSUPPORTED_STYLE: "note",
+ Kind.EVIDENCE_FOUND: "note",
}
# The state word a renderer colours (spec section 13.3), quoted from section 15 and
@@ -122,11 +145,14 @@ class Kind(str, Enum):
Kind.UNRESOLVED_MARKER: "UNRESOLVED MARKER",
Kind.PARSE_ERROR: "PARSE ERROR",
Kind.PROVIDER_UNAVAILABLE: Outcome.UNAVAILABLE.value,
+ Kind.EVIDENCE_FOUND: "SUPPORTED (found by proofpath)",
+ Kind.NO_EVIDENCE: "NO EVIDENCE FOUND (searched)",
}
# The kinds that claim the document says something the source does not. They are the
# only ones that assert, so they are the only ones that owe a passage (rule 1).
-_ASSERTING = frozenset({Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH})
+# EVIDENCE_FOUND asserts support, and support needs its passage just as much (rule 1).
+_ASSERTING = frozenset({Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND})
TextKind = Literal["fulltext", "abstract", "none"]
@@ -267,6 +293,20 @@ def weak(self) -> bool:
return unread / self.references >= WEAK_UNVERIFIED
+@dataclass(frozen=True)
+class SearchSummary:
+ """What the evidence search did: the block beneath the coverage block (rule 6)."""
+
+ by: str # the searcher: "tavily" | "searxng"
+ queries_by: str # "sentence", or the judge that wrote them
+ sentences: int # sentences with a word in them
+ eligible: int # of those, check-worthy
+ searched: int # of those, actually searched
+ pages_found: int
+ pages_read: int # full text or abstract
+ notices: tuple[str, ...] = () # the LLM fallback, said once
+
+
@dataclass(frozen=True)
class Stage:
"""One pipeline stage as the report shows it, with the provider that produced it."""
@@ -308,6 +348,12 @@ class Report:
# What the optional judge spent, or ``None`` on a run that never asked one. A
# run states its own cost for the same reason it states its own coverage.
judge_cost: JudgeCost | None = None
+ # Set only on a run that searched (OPEN-ITEMS 17.1a).
+ search: SearchSummary | None = None
+ # The notice of a judge that switched to its local fallback (spec section 11), or
+ # ``None``. A run that changed who gives its second opinion says so in the report,
+ # not only in a line that scrolled past while it ran.
+ judge_notice: str | None = None
def exit_code(self) -> int:
"""0 when the run is clean, 1 when it has findings. 2 is never returned here.
@@ -581,6 +627,13 @@ class Footer:
# whole and are shortened where they are printed, so a caller can still tell one
# from another -- the terminal needs to know which one is a missing credential.
reasons: tuple[tuple[str, int], ...] = ()
+ # ``search_lines``: empty unless the run searched.
+ search: tuple[str, ...] = ()
+ # Of the judge's answers, those the local fallback gave; beside ``api_calls``,
+ # which counts provider requests only (spec section 11).
+ local_calls: int = 0
+ # ``Report.judge_notice``: the run switched to a local judge.
+ judge_notice: str | None = None
def render_diagnostics(report: Report) -> list[Diagnostic]:
@@ -630,9 +683,22 @@ def render_footer(report: Report, *, written: str | None = None) -> Footer:
browser_skipped=report.coverage.browser_skipped,
judge_tokens=_judge_tokens(report),
reasons=coverage_reasons(report.coverage),
+ search=search_lines(report),
+ local_calls=0 if report.judge_cost is None else report.judge_cost.local_calls,
+ judge_notice=report.judge_notice,
)
+def calls_text(api_calls: int, local_calls: int = 0) -> str:
+ """``"3 API calls"``, or ``"3 API calls + 5 local"`` once a local judge answered.
+
+ One spelling for the terminal and the TUI footer, so a run that spent nothing on
+ the provider but still asked a local model never reads as a run that asked nobody.
+ """
+ text = f"{api_calls} API calls"
+ return f"{text} + {local_calls} local" if local_calls else text
+
+
def _judge_tokens(report: Report) -> tuple[int, int] | None:
"""The judge's token spend, or ``None`` when there is nothing to report.
@@ -655,6 +721,11 @@ def render_markdown(report: Report, *, written_at: datetime | None = None) -> st
lines.append(f"- tiers: {report.tier_note}")
lines.append(f"- elapsed: {report.elapsed:.1f}s")
lines.append(f"- api calls: {report.api_calls}")
+ local = 0 if report.judge_cost is None else report.judge_cost.local_calls
+ if local:
+ lines.append(f"- local calls: {local}")
+ if report.judge_notice:
+ lines.append(f"- judge fallback: {report.judge_notice}")
for what in (JUDGE, SUMMARY):
unavailable = _unavailable_header(report, what)
if unavailable is not None:
@@ -666,6 +737,7 @@ def render_markdown(report: Report, *, written_at: datetime | None = None) -> st
lines.extend(_markdown_checked(report))
lines.extend(_markdown_sources(report))
lines.extend(_markdown_coverage(report))
+ lines.extend(_markdown_search(report))
if report.summary:
# Labelled, and attributed, every time it is printed (spec section 11.1): a
# paragraph a model wrote must never be mistaken for the report's own findings.
@@ -1001,6 +1073,29 @@ def _markdown_coverage(report: Report) -> list[str]:
return lines
+def search_lines(report: Report) -> tuple[str, ...]:
+ """The evidence-search block, one line each, for every surface to print as is."""
+ item = report.search
+ if item is None:
+ return ()
+ return (
+ search_experimental(),
+ f"evidence search by {item.by}, queries by {item.queries_by}",
+ f"claims searched {item.searched} of {item.eligible} check-worthy "
+ f"({item.sentences} sentences)",
+ f"claims not searched {item.sentences - item.searched}",
+ f"pages found / read {item.pages_found} / {item.pages_read}",
+ *item.notices,
+ )
+
+
+def _markdown_search(report: Report) -> list[str]:
+ lines = search_lines(report)
+ if not lines:
+ return []
+ return ["## Evidence search", "", *(f"- {line}" for line in lines), ""]
+
+
def _one_line(text: str) -> str:
return " ".join(text.split())
diff --git a/src/proofpath/sarif.py b/src/proofpath/sarif.py
index bba3583..efacc42 100644
--- a/src/proofpath/sarif.py
+++ b/src/proofpath/sarif.py
@@ -79,6 +79,8 @@
Kind.UNRESOLVED_MARKER: "the citation marker points at no bibliography entry",
Kind.PARSE_ERROR: "part of the document could not be parsed",
Kind.PROVIDER_UNAVAILABLE: "a bibliographic provider could not be consulted",
+ Kind.EVIDENCE_FOUND: "a page proofpath found, not one the author cited, supports the claim",
+ Kind.NO_EVIDENCE: "the evidence search found no page that supports or contradicts the claim",
}
@@ -235,6 +237,23 @@ def _run_properties(report: Report) -> dict[str, Any]:
},
"models": dict(report.models),
"apiCalls": report.api_calls,
+ # Answers the local fallback judge gave, and the notice of the switch (spec
+ # section 11): a log read on its own still says who gave the second opinion.
+ "localCalls": 0 if report.judge_cost is None else report.judge_cost.local_calls,
+ "judgeNotice": report.judge_notice,
"elapsed": report.elapsed,
"cancelled": report.cancelled,
+ "evidenceSearch": None
+ if report.search is None
+ else {
+ "experimental": True,
+ "by": report.search.by,
+ "queriesBy": report.search.queries_by,
+ "sentences": report.search.sentences,
+ "eligible": report.search.eligible,
+ "searched": report.search.searched,
+ "pagesFound": report.search.pages_found,
+ "pagesRead": report.search.pages_read,
+ "notices": list(report.search.notices),
+ },
}
diff --git a/src/proofpath/search/__init__.py b/src/proofpath/search/__init__.py
new file mode 100644
index 0000000..bfb6bef
--- /dev/null
+++ b/src/proofpath/search/__init__.py
@@ -0,0 +1,79 @@
+"""Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a).
+
+A searcher turns a query into addresses and nothing else. What a page says is read by
+the fetch ladder, exactly like a cited source, so a provider's snippet can never
+become the passage a verdict rests on (product rule 1).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from proofpath.providers import is_social
+
+
+@dataclass(frozen=True)
+class SearchHit:
+ """One address a provider answered with, in its own order."""
+
+ url: str
+ title: str
+ rank: int # 1-based position in the provider's answer
+
+
+class Searcher(Protocol):
+ """A web search provider. ``search`` raises ``polite.ProviderError`` when the
+ provider does not answer; an empty list means it answered with nothing."""
+
+ name: str # "tavily" | "searxng": what the Searching stage is attributed to
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]: ...
+
+
+# Parameters that name the campaign a link came from, not the page it points at.
+_TRACKING = re.compile(r"^(utm_\w+|fbclid|gclid|mc_cid|mc_eid)$")
+
+
+def canonical(url: str) -> str:
+ """One spelling per page: lower-cased scheme and host, no fragment, no tracking."""
+ parts = urlsplit(url)
+ host = (parts.hostname or "").lower()
+ if parts.port:
+ host = f"{host}:{parts.port}"
+ query = urlencode(
+ [
+ (k, v)
+ for k, v in parse_qsl(parts.query, keep_blank_values=True)
+ if not _TRACKING.match(k)
+ ]
+ )
+ return urlunsplit((parts.scheme.lower(), host, parts.path or "/", query, ""))
+
+
+def _bare_host(url: str) -> str:
+ return (urlsplit(url).hostname or "").lower().removeprefix("www.")
+
+
+def keep_hits(hits: Sequence[SearchHit], *, own_host: str | None, limit: int) -> list[SearchHit]:
+ """The hits worth reading, in order: one per page, no posts, not the document itself.
+
+ A post is dropped because a platform post is a claim, not a source. X, the case
+ that brought this feature, cannot be read at all (spec section 3). The document's
+ own host is dropped so a page is never found to support itself.
+ """
+ own = (own_host or "").lower().removeprefix("www.")
+ kept: list[SearchHit] = []
+ seen: set[str] = set()
+ for hit in hits:
+ key = canonical(hit.url)
+ if key in seen or is_social(hit.url) or (own and _bare_host(hit.url) == own):
+ continue
+ seen.add(key)
+ kept.append(hit)
+ if len(kept) >= limit:
+ break
+ return kept
diff --git a/src/proofpath/search/language.py b/src/proofpath/search/language.py
new file mode 100644
index 0000000..1c02b87
--- /dev/null
+++ b/src/proofpath/search/language.py
@@ -0,0 +1,49 @@
+"""Whether a claim reads as English, the only language the models were trained on.
+
+Deliberately small and one-sided: English is the default, and a claim is called
+something else only on positive evidence -- a letter from another script (real
+Turkish, Cyrillic, CJK...), a letter only Turkish writes, or a common function word
+of another language. A wrong "English" costs a weaker check; a wrong "not English"
+would cost the claim its search without a judge, so the doubt goes to English.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+# Function words of languages a reader is likely to paste, none of them English words.
+_FOREIGN_WORDS = frozenset(
+ {
+ # Turkish
+ "ve", "bir", "bu", "için", "ile", "değil", "çok", "daha", "gibi", "olan",
+ "olarak", "ama", "şu", "mi", "mı", "mu", "mü", "da", "de", "ki", "yok", "var", # noqa: RUF001 - real Turkish letters
+ # Spanish / Portuguese / Italian
+ "el", "los", "las", "que", "y", "es", "por", "con", "una", "para", "del",
+ "não", "uma", "il", "della", "che", "sono",
+ # German / Dutch
+ "der", "das", "und", "ist", "nicht", "ein", "eine", "mit", "den", "het", "een",
+ # French
+ "le", "les", "et", "est", "une", "des", "du", "pas", "pour", "dans",
+ }
+) # fmt: skip
+_WORDS = re.compile(r"[^\W\d_]+")
+
+# Letters only Turkish writes. ç, ö and ü are not here: they turn up in names
+# ("Müller", "François"), and a name is not a language.
+_TURKISH_ONLY = frozenset("ğĞıİşŞ")
+
+
+def _foreign_letter(char: str) -> bool:
+ """A letter from another script, or one only Turkish uses. A Latin letter with
+ an accent ("André", "Beyoncé") is not evidence: English borrows names."""
+ if char in _TURKISH_ONLY:
+ return True
+ return not unicodedata.name(char, "LATIN").startswith("LATIN")
+
+
+def looks_english(text: str) -> bool:
+ letters = [char for char in text if char.isalpha()]
+ if any(_foreign_letter(char) for char in letters):
+ return False
+ return not any(word.casefold() in _FOREIGN_WORDS for word in _WORDS.findall(text))
diff --git a/src/proofpath/search/providers.py b/src/proofpath/search/providers.py
new file mode 100644
index 0000000..c03e91a
--- /dev/null
+++ b/src/proofpath/search/providers.py
@@ -0,0 +1,126 @@
+"""The two search providers of OPEN-ITEMS 17.1a: Tavily (a key) and SearXNG (an address)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from proofpath.config import SearchConfig
+from proofpath.polite import PoliteClient, ProviderError
+from proofpath.search import Searcher, SearchHit
+from proofpath.secrets import ApiKey, missing_hint, resolve_api_key
+
+TAVILY_URL = "https://api.tavily.com/search"
+# Tavily refuses a longer query (API reference, checked 2026-09-28).
+TAVILY_QUERY_LIMIT = 400
+_WEB = ("http://", "https://")
+SEARXNG_NEEDS_URL = "set search.base_url to your SearXNG instance"
+
+
+class TavilySearcher:
+ name = "tavily"
+
+ def __init__(self, client: PoliteClient, key: ApiKey) -> None:
+ self._client = client
+ self._key = key
+
+ def __repr__(self) -> str:
+ return f"TavilySearcher(key={self._key!r})" # ApiKey's repr never shows the value
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ response = self._client.post(
+ TAVILY_URL,
+ json_body={
+ "query": query[:TAVILY_QUERY_LIMIT],
+ "max_results": max_results,
+ "search_depth": "basic",
+ "include_answer": False,
+ "include_raw_content": False,
+ "include_images": False,
+ },
+ headers={"Authorization": f"Bearer {self._key.value}"},
+ )
+ return _hits(_payload(response, "api.tavily.com").get("results"), max_results)
+
+
+class SearxngSearcher:
+ name = "searxng"
+
+ def __init__(self, client: PoliteClient, base_url: str) -> None:
+ self._client = client
+ self._url = base_url.rstrip("/") + "/search"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ # ``format=json`` must be enabled in the instance's settings.yml; an instance
+ # that has not enabled it answers 403, which is reported, not guessed around.
+ response = self._client.get(self._url, {"q": query, "format": "json"}, mailto=False)
+ host = httpx.URL(self._url).host
+ return _hits(_payload(response, host).get("results"), max_results)
+
+
+def _payload(response: httpx.Response, host: str) -> dict[str, Any]:
+ """The answer's JSON object. The body is never quoted: a provider echoes keys."""
+ if response.status_code != 200:
+ raise ProviderError(f"HTTP {response.status_code}", response.status_code, None)
+ try:
+ body = response.json()
+ except ValueError:
+ raise ProviderError(
+ f"{host} answered with a non-JSON body", response.status_code, None
+ ) from None
+ if not isinstance(body, dict):
+ raise ProviderError(f"{host} answered with a non-JSON body", response.status_code, None)
+ return body
+
+
+def _hits(results: Any, limit: int) -> list[SearchHit]:
+ """Addresses and titles, in the provider's order. ``content`` is never read."""
+ hits: list[SearchHit] = []
+ if not isinstance(results, list):
+ return hits
+ for entry in results:
+ if not isinstance(entry, dict):
+ continue
+ url = entry.get("url")
+ if not isinstance(url, str) or not url.lower().startswith(_WEB):
+ continue
+ title = entry.get("title")
+ hits.append(
+ SearchHit(url=url, title=title if isinstance(title, str) else "", rank=len(hits) + 1)
+ )
+ if len(hits) >= limit:
+ break
+ return hits
+
+
+@dataclass(frozen=True)
+class SearchSetup:
+ """The run's searcher, or why a configured one could not be built.
+
+ ``problem`` is empty when search is simply off. A named provider that cannot run
+ (no key, no address) is reported rather than skipped (rule 2).
+ """
+
+ searcher: Searcher | None
+ problem: str
+
+
+def build_searcher(
+ config: SearchConfig,
+ client: PoliteClient,
+ *,
+ resolve_key: Callable[[str], ApiKey | None] = resolve_api_key,
+) -> SearchSetup:
+ if config.provider == "tavily":
+ key = resolve_key(config.api_key_env)
+ if key is None:
+ return SearchSetup(None, missing_hint(config.api_key_env))
+ return SearchSetup(TavilySearcher(client, key), "")
+ if config.provider == "searxng":
+ if not config.base_url:
+ return SearchSetup(None, SEARXNG_NEEDS_URL)
+ return SearchSetup(SearxngSearcher(client, config.base_url), "")
+ return SearchSetup(None, "")
diff --git a/src/proofpath/search/queries.py b/src/proofpath/search/queries.py
new file mode 100644
index 0000000..7d31f18
--- /dev/null
+++ b/src/proofpath/search/queries.py
@@ -0,0 +1,95 @@
+"""What the evidence search asks: the sentence itself, or what the judge wrote.
+
+The default makes zero LLM calls (spec section 11). With ``--judge``, the judge
+rewrites each claim as a query that stands on its own, and translates it to English
+when it is not (Amendment A): the check models are English-only, so a claim that is
+not English has no query, no English text, and nothing for Task 7 to check unless a
+judge translated it. When the judge cannot answer -- a rate limit, an exhausted
+quota, an outage -- the sentence is used instead and the run says so, because a
+weaker query is a weaker search and the reader has to know which one they got
+(product rule 6).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from proofpath.search.language import looks_english
+
+if TYPE_CHECKING:
+ from proofpath.judge import Judge
+
+QUERY_LIMIT = 400
+SENTENCE_BY = "sentence"
+LLM_LIMIT = "LLM limit reached — searched with the sentence text"
+LLM_UNAVAILABLE = "LLM did not answer — searched with the sentence text"
+
+_URL = re.compile(r"https?://\S+")
+_HANDLE = re.compile(r"(? str:
+ """The sentence as a query: no addresses, handles or emoji, at most 400 characters."""
+ cleaned = _EMOJI.sub(" ", _HANDLE.sub(" ", _URL.sub(" ", text)))
+ cleaned = " ".join(_HASH.sub(r"\1", cleaned).split())
+ if len(cleaned) <= QUERY_LIMIT:
+ return cleaned
+ cut = cleaned[:QUERY_LIMIT]
+ return cut.rsplit(" ", 1)[0] if " " in cut else cut
+
+
+@dataclass(frozen=True)
+class QueryPlan:
+ queries: tuple[tuple[str, ...], ...] # per claim, in claim order; never empty
+ hypotheses: tuple[str | None, ...] # per claim: the English text to check, or None
+ by: str # SENTENCE_BY, or the judge's name
+ notice: str | None # set when the judge was asked and the sentence was used instead
+
+
+def fallback_notice(detail: str) -> str:
+ head = LLM_LIMIT if "HTTP 429" in detail else LLM_UNAVAILABLE
+ return f"{head} ({detail})"
+
+
+def plan_queries(texts: Sequence[str], judge: Judge | None) -> QueryPlan:
+ english = tuple(looks_english(text) for text in texts)
+ fallback = tuple((sentence_query(text),) for text in texts)
+ # An English claim is checked as written; another language only once translated.
+ as_written = tuple(text if eng else None for text, eng in zip(texts, english, strict=True))
+ if judge is None or not texts:
+ return QueryPlan(fallback, as_written, SENTENCE_BY, None)
+ written = judge.queries(texts)
+ if judge.unavailable:
+ return QueryPlan(fallback, as_written, SENTENCE_BY, fallback_notice(judge.detail))
+ queries = tuple(
+ (written[i].queries if i in written and written[i].queries else fallback[i])
+ for i in range(len(texts))
+ )
+ hypotheses = tuple(
+ texts[i]
+ if english[i]
+ else (_translation(texts[i], written[i].english) if i in written else None)
+ for i in range(len(texts))
+ )
+ return QueryPlan(queries, hypotheses, judge.name, None)
+
+
+def _translation(original: str, english: str) -> str | None:
+ """The judge's English text, or ``None`` when it is not a translation at all.
+
+ A model can copy the claim back unchanged (a local qwen with thinking off does,
+ measured 2026-09-28) or answer in the claim's own language. Either one handed to
+ the English-only models would be checked as if it were English; ``None`` sends
+ the claim down the no-translation path instead (``language not supported``).
+ """
+ if not english:
+ return None
+ if english.strip() == original.strip() or not looks_english(english):
+ return None
+ return english
diff --git a/src/proofpath/settings_hints.py b/src/proofpath/settings_hints.py
index b16c76e..7a186c7 100644
--- a/src/proofpath/settings_hints.py
+++ b/src/proofpath/settings_hints.py
@@ -15,3 +15,9 @@
BROWSER_SETTING = "permissions.install_browser ask"
#: ``permissions.network`` set to ``allow``: every network-bound stage runs.
NETWORK_SETTING = "permissions.network allow"
+#: ``search.provider`` set to ``tavily``: a text that cites nothing is searched with
+#: the user's own Tavily key (OPEN-ITEMS 17.1a).
+SEARCH_SETTING = "search.provider tavily"
+#: ``search.provider`` set to ``searxng``: the other v1 provider, the user's own
+#: SearXNG instance addressed by ``search.base_url`` instead of a key.
+SEARXNG_SETTING = "search.provider searxng"
diff --git a/src/proofpath/tui/app.py b/src/proofpath/tui/app.py
index 0d00495..3565136 100644
--- a/src/proofpath/tui/app.py
+++ b/src/proofpath/tui/app.py
@@ -38,6 +38,7 @@
import os
from collections.abc import Callable
from concurrent.futures import Future
+from contextlib import suppress
from typing import ClassVar, Protocol
from rich.text import Text
@@ -45,7 +46,9 @@
from textual.app import App, ComposeResult
from textual.binding import Binding, BindingType
from textual.containers import Vertical, VerticalScroll
+from textual.css.query import NoMatches
from textual.geometry import Size
+from textual.message import Message
from textual.widget import AwaitMount, Widget
from textual.widgets import Button, Input, Static
@@ -53,8 +56,8 @@
from proofpath.browser import Answer
from proofpath.commands import config_view
from proofpath.config import Config, ConfigError, load_config
-from proofpath.events import Event
-from proofpath.judge import Judge, JudgeClient, JudgeError, resolve_api_key
+from proofpath.events import Event, Note
+from proofpath.judge import Judge, JudgeError, build_client, resolve_api_key
from proofpath.report import Report, render_markdown, summary_silence
from proofpath.tui import commands
from proofpath.tui.history import History
@@ -70,6 +73,7 @@
ConfigPanel,
CoverageFooter,
FindingLine,
+ JudgeNotice,
KvLine,
NoteLine,
PermissionPrompt,
@@ -228,6 +232,10 @@ class ProofpathApp(App[None]):
#bottom { dock: bottom; height: auto; }
#footer { height: 3; padding: 0 1; }
#suggestions { height: auto; padding: 0 1; display: none; }
+ /* The judge's switch to a local model (spec section 11): one row, right-aligned,
+ directly above the bar. Hidden by ``display: none`` in the widget, so an
+ unused notice costs the dock no row. */
+ #judge-notice { height: 1; padding: 0 1; text-align: right; }
/* The bar is one flat row: its frame's rows and edges exist but are hidden, so
the same composition serves every theme and width (wordmark design section 9). */
#prompt-frame { height: 1; margin: 0; }
@@ -379,7 +387,29 @@ def _default_judge(self) -> Judge:
key = resolve_api_key(settings.api_key_env)
if settings.api_key_env and key is None:
raise JudgeError(NO_JUDGE_NOTE)
- return Judge(JudgeClient(settings, key))
+ # A fresh client per summary: each one tries the configured provider first.
+ return Judge(build_client(settings, key, on_switch=self._judge_switched))
+
+ class JudgeSwitched(Message):
+ """The summary's judge switched to its local model (spec section 11)."""
+
+ def __init__(self, text: str) -> None:
+ super().__init__()
+ self.text = text
+
+ def _judge_switched(self, text: str) -> None:
+ """The summary's judge switched to its local model; say so above the bar.
+
+ Called on the summary's worker thread, before the model is loaded. It only
+ posts a message -- ``post_message`` is thread-safe and never waits -- so the
+ worker is not held, and the widget is touched on the loop alone.
+ """
+ self.post_message(self.JudgeSwitched(text))
+
+ def on_proofpath_app_judge_switched(self, message: JudgeSwitched) -> None:
+ # A message that lands while the app is shutting down finds no widget.
+ with suppress(NoMatches):
+ self.query_one(JudgeNotice).show(message.text)
# --- the section 7.1 prompt, inline ---------------------------------------------
@@ -508,6 +538,7 @@ def compose(self) -> ComposeResult:
with Vertical(id="bottom", classes=self._theme.name):
yield CoverageFooter(self._out, self._theme)
yield Suggestions(theme=self._theme, coloured=self._out.color)
+ yield JudgeNotice(self._theme, coloured=self._out.color)
yield PromptFrame(
theme=self._theme,
coloured=self._out.color,
@@ -639,12 +670,17 @@ def _suggestion_lines(self, size: Size) -> int:
"""
rows = self.query_one(Banner).get_content_height(size, size, size.width)
rows += self.query_one(CoverageFooter).rows
+ rows += self.query_one(JudgeNotice).rows
rows += self.query_one(PromptFrame).rows
return max(size.height - rows - MIN_LOG_ROWS, 1)
# --- the scheduler's two callbacks ----------------------------------------------
def _on_event(self, run: Run, event: Event) -> None:
+ if isinstance(event, Note) and event.notice:
+ # Before the block lookup: a run whose block ``ctrl+l`` cleared still
+ # switched, and the row above the bar is the one place that says so.
+ self.query_one(JudgeNotice).show(event.text)
block = self._blocks.get(run.id)
if block is None:
# A run whose block never got built still said something; dropping it
@@ -666,6 +702,9 @@ def _on_state(self, run: Run) -> None:
self._watch_eyes(run)
return
if block is None:
+ # A new run starts on the configured judge (Amendment B 9), so the last
+ # run's switch notice is no longer true of what happens next.
+ self.query_one(JudgeNotice).hide()
block = RunBlock(run, self._out, self._theme)
self._blocks[run.id] = block
self.query_one(RunLog).mount(block)
@@ -1161,6 +1200,7 @@ def run(config: Config, out: ui.Ui) -> None:
"ConfigPanel",
"CoverageFooter",
"FindingLine",
+ "JudgeNotice",
"KvLine",
"NoteLine",
"PermissionPrompt",
diff --git a/src/proofpath/tui/widgets/__init__.py b/src/proofpath/tui/widgets/__init__.py
index e9e39a6..399a42e 100644
--- a/src/proofpath/tui/widgets/__init__.py
+++ b/src/proofpath/tui/widgets/__init__.py
@@ -13,6 +13,7 @@
from proofpath.tui.widgets.config_panel import ConfigPanel
from proofpath.tui.widgets.finding import FindingLine, FindingsRule, Line
from proofpath.tui.widgets.footer import CoverageFooter
+from proofpath.tui.widgets.judge_notice import JudgeNotice
from proofpath.tui.widgets.prompt import (
ANSWER_ID,
ANSWER_LABELS,
@@ -44,6 +45,7 @@
"FindingLine",
"FindingsRule",
"FrameEdge",
+ "JudgeNotice",
"KvLine",
"Line",
"NoteLine",
diff --git a/src/proofpath/tui/widgets/config_panel.py b/src/proofpath/tui/widgets/config_panel.py
index 8cdfbb1..1add608 100644
--- a/src/proofpath/tui/widgets/config_panel.py
+++ b/src/proofpath/tui/widgets/config_panel.py
@@ -33,11 +33,13 @@
from proofpath.commands import ConfigView, config_set, config_view
from proofpath.config import (
PERMISSION_VALUES,
+ SEARCH_PROVIDERS,
ConfigError,
Contact,
FetchConfig,
JudgeConfig,
Permissions,
+ SearchConfig,
)
from proofpath.judge import JudgeError, default_dotenv_paths, known_providers
from proofpath.secrets import resolve_api_key
@@ -98,6 +100,7 @@ def sections() -> tuple[Section, ...]:
"""The panel's rows in the config's own order. Built, not constant, because the
provider list is the judge module's, and the defaults are the dataclasses' own."""
permissions, fetch, judge, contact = Permissions(), FetchConfig(), JudgeConfig(), Contact()
+ search = SearchConfig()
return (
Section(
"permissions",
@@ -116,6 +119,13 @@ def sections() -> tuple[Section, ...]:
"every fetch and every provider call; deny = offline",
_default(permissions.network),
),
+ Row(
+ "permissions.web_search",
+ "choice",
+ PERMISSION_VALUES,
+ "search the web when a text cites nothing; ask = deny (never prompts)",
+ _default(permissions.web_search),
+ ),
),
),
Section(
@@ -143,6 +153,39 @@ def sections() -> tuple[Section, ...]:
Row("judge.model", "text", (), "", _default(judge.model)),
Row("judge.base_url", "text", (), "", _default(judge.base_url)),
Row("judge.api_key_env", "text", (), "", _default(judge.api_key_env)),
+ Row(
+ "judge.fallback",
+ "text",
+ (),
+ "local model when the API is limited or down; off = none",
+ _default(judge.fallback),
+ ),
+ ),
+ ),
+ Section(
+ "search",
+ (
+ Row(
+ "search.provider",
+ "choice",
+ SEARCH_PROVIDERS,
+ "off until set; tavily needs TAVILY_API_KEY in .env",
+ _default(search.provider),
+ ),
+ Row(
+ "search.base_url",
+ "text",
+ (),
+ "your SearXNG instance",
+ _default(search.base_url),
+ ),
+ Row(
+ "search.api_key_env",
+ "text",
+ (),
+ "the variable holding the key",
+ _default(search.api_key_env),
+ ),
),
),
Section(
@@ -594,12 +637,13 @@ def _draw_legend(self) -> Text:
)
def _draw_summary(self) -> Text:
- """The one-line record: the four settings a run decides by, as the file has them."""
+ """The one-line record: the settings a run decides by, as the file has them."""
config = self._view.config
parts = (
f"install_browser={config.permissions.install_browser}",
f"network={config.permissions.network}",
f"respect_robots={_default(config.fetch.respect_robots)}",
f"judge={config.judge.provider}",
+ f"search={config.search.provider}",
)
return Text(" config " + " ".join(parts))
diff --git a/src/proofpath/tui/widgets/footer.py b/src/proofpath/tui/widgets/footer.py
index 24fe652..c2fe8ef 100644
--- a/src/proofpath/tui/widgets/footer.py
+++ b/src/proofpath/tui/widgets/footer.py
@@ -6,7 +6,14 @@
from textual.widgets import Static
from proofpath import ui
-from proofpath.report import BROWSER_SKIPPED_REASON, Footer, Report, no_bibliography, render_footer
+from proofpath.report import (
+ BROWSER_SKIPPED_REASON,
+ Footer,
+ Report,
+ calls_text,
+ no_bibliography,
+ render_footer,
+)
from proofpath.tui.theme import Theme
from proofpath.tui.widgets._shared import DEFAULT_WIDTH, _elide
@@ -73,7 +80,8 @@ def render(self) -> Text:
INDENT, _elide(item.counts, width - len(coverage) - 8), " ", coverage
)
written = f"{item.written} written" if item.written else "no report written"
- cost = f"{written} · {item.api_calls} API calls · {item.elapsed:.1f}s"
+ calls = calls_text(item.api_calls, item.local_calls)
+ cost = f"{written} · {calls} · {item.elapsed:.1f}s"
line.append("\n" + INDENT + _elide(cost, width - 4))
# Every caveat the run owes the reader, on one line and never dropped: a
# low-coverage run must not look like a clean one, and the one place that
@@ -160,6 +168,13 @@ def _hints(item: Footer) -> list[str]:
f"{item.browser_skipped} source(s) {BROWSER_SKIPPED_REASON}"
f" — /config set {ui.BROWSER_SETTING}"
)
+ # The docked footer has one hints row: the experimental banner and the searched
+ # line go there, and the full block is in the run's report.
+ hints.extend(item.search[:3])
+ if item.judge_notice:
+ # The notice above the prompt goes when the next run starts; the footer is
+ # this run's record, so it keeps saying who gave the second opinion.
+ hints.append(item.judge_notice)
if item.note:
hints.append(item.note)
return hints
diff --git a/src/proofpath/tui/widgets/judge_notice.py b/src/proofpath/tui/widgets/judge_notice.py
new file mode 100644
index 0000000..1158a56
--- /dev/null
+++ b/src/proofpath/tui/widgets/judge_notice.py
@@ -0,0 +1,45 @@
+"""The one-line notice above the prompt: the judge switched to a local model.
+
+Spec section 11: the switch is never silent. The run's own block carries the line as
+a note too, but that scrolls away with the log; this row sits directly above the bar,
+right-aligned, from the moment the run switches until the next run starts.
+"""
+
+from __future__ import annotations
+
+from rich.text import Text
+from textual.widgets import Static
+
+from proofpath.tui.theme import Theme
+
+
+class JudgeNotice(Static):
+ """Hidden (``display: none``) until :meth:`show`; never an empty line.
+
+ An empty line would still take a row out of the bottom dock, and the dock's
+ height is what keeps the footer and the bar in place (rule 6).
+ """
+
+ def __init__(self, theme: Theme, *, coloured: bool) -> None:
+ super().__init__(id="judge-notice")
+ self._theme = theme
+ self._coloured = coloured
+ self.text = ""
+ self.display = False
+
+ @property
+ def rows(self) -> int:
+ """What the notice takes from the screen: one row when shown, none when not."""
+ return 1 if self.display else 0
+
+ def show(self, text: str) -> None:
+ self.text = text
+ # ``caution``: the run is still judged, by a different model than configured.
+ style = self._theme.tone("caution") if self._coloured else ""
+ self.update(Text(text, style=style, no_wrap=True, overflow="ellipsis"))
+ self.display = True
+
+ def hide(self) -> None:
+ self.text = ""
+ self.update("")
+ self.display = False
diff --git a/src/proofpath/ui.py b/src/proofpath/ui.py
index c6a550e..0848d4e 100644
--- a/src/proofpath/ui.py
+++ b/src/proofpath/ui.py
@@ -30,6 +30,7 @@
Diagnostic,
Footer,
Level,
+ calls_text,
no_bibliography,
reason_label,
)
@@ -59,7 +60,14 @@
# so this is what the TUI's inline prompt paints its buttons with -- named here
# because no other module names a colour.
PROMPT_COLOUR = "yellow"
-GREEN = ("ok", "RESOLVED", "SUPPORTED", "fulltext", "not retracted")
+GREEN = (
+ "ok",
+ "RESOLVED",
+ "SUPPORTED",
+ "fulltext",
+ "not retracted",
+ "SUPPORTED (found by proofpath)",
+)
YELLOW_PREFIXES = (
"UNVERIFIED",
"LOW CONFIDENCE",
@@ -71,6 +79,9 @@
"UNRESOLVED MARKER",
# A retraction check nobody answered: caution, not "not retracted" (rule 2).
"unavailable",
+ # The evidence search found nothing that decides the claim (OPEN-ITEMS 17.1a):
+ # caution, like every other UNVERIFIED-shaped state, never red (rule 2).
+ "NO EVIDENCE FOUND",
)
RED = ("GHOST REFERENCE", "REFUTED", "RETRACTED", "FAILED", "NOT SUPPORTED", "PARSE ERROR")
DIM = ("NEI", "none", "—", "cancelled")
@@ -342,15 +353,24 @@ def footer(ui: Ui, item: Footer) -> None:
reasons=item.reasons,
)
skipped(ui, item.browser_skipped)
+ # The evidence-search block (OPEN-ITEMS 17.1a): never suppressed, because a run
+ # that searched must not read like a run that checked cited sources (rule 6).
+ for line in item.search:
+ kv(ui, "search", line)
if item.note:
hint(ui, item.note)
written = f"{item.written} written" if item.written else "no report written"
- ui.out.print(f"{written} · {item.api_calls} API calls · {item.elapsed:.1f}s")
+ calls = calls_text(item.api_calls, item.local_calls)
+ ui.out.print(f"{written} · {calls} · {item.elapsed:.1f}s")
if item.judge_tokens is not None:
# Its own line, and never suppressed: what the optional judge spent is part
# of what the run cost, and a free tier is metered in tokens, not in calls.
prompt, completion = item.judge_tokens
kv(ui, "judge", f"{prompt:,} prompt · {completion:,} completion tokens")
+ if item.judge_notice:
+ # Who gave the second opinion changed mid-run (spec section 11); the line
+ # that said so live went to stderr, and this one stays with the report.
+ kv(ui, "judge", item.judge_notice)
def error(ui: Ui, text: str) -> None:
diff --git a/src/proofpath/verify.py b/src/proofpath/verify.py
index 353f331..46aa8dd 100644
--- a/src/proofpath/verify.py
+++ b/src/proofpath/verify.py
@@ -36,6 +36,7 @@
from dataclasses import dataclass, field, replace
from functools import partial
from pathlib import Path
+from urllib.parse import urlsplit
import httpx
@@ -91,7 +92,10 @@
from proofpath.providers.social import SocialProvider
from proofpath.providers.web import WebProvider
from proofpath.report import (
+ FOUND_BY_PROOFPATH,
+ LANGUAGE_UNSUPPORTED,
LEVELS,
+ SEARCH_UNAVAILABLE,
STATE_WORDS,
SUMMARY,
ClaimResult,
@@ -99,6 +103,7 @@
Finding,
Kind,
Report,
+ SearchSummary,
SourceStatus,
Stage,
TextKind,
@@ -108,11 +113,17 @@
)
from proofpath.resolve import Resolver, ResolveResult, Retraction, State
from proofpath.retrieval import Embedder, FastEmbedder, PassageIndex
-from proofpath.settings_hints import BROWSER_SETTING
+from proofpath.search import Searcher, SearchHit, canonical, keep_hits
+from proofpath.search.providers import build_searcher
+from proofpath.search.queries import plan_queries
+from proofpath.secrets import CREDENTIALS_MISSING
+from proofpath.settings_hints import BROWSER_SETTING, SEARCH_SETTING, SEARXNG_SETTING
# Stage names, as the report and the TUI print them (spec section 13.2).
PARSING = "Parsing"
CLAIMS = "Claims"
+# Between Claims and Resolving, and only for a text that cites nothing (OPEN-ITEMS 17.1a).
+SEARCHING = "Searching"
RESOLVING = "Resolving"
RETRACTIONS = "Retractions"
FETCHING = "Fetching"
@@ -176,16 +187,29 @@
# text to check any of it against, and the report has to tell the two apart (rule 6).
NOTHING_TO_VERIFY = "nothing to verify"
-# What the parsing stage says of pasted text that neither links nor cites anything:
-# a bare claim. The tool never goes looking for evidence of its own (README: it
-# checks the sources *behind* a claim), so a claim with no source behind it ends
-# with the stages all at zero -- which, unsaid, looks exactly like a clean run to
-# the reader who typed it (product rule 6). The hint says what to paste instead.
+# What the parsing stage says of a text that neither links nor cites anything: a bare
+# claim. Without a search provider there is nothing to check it against, and ending
+# with every stage at zero would read as a clean run (product rule 6). The hint names
+# the one setting that makes proofpath look for evidence itself (OPEN-ITEMS 17.1a).
NO_SOURCE_IN_TEXT = "the text links or cites no source; nothing to verify against"
NO_SOURCE_HINT = (
- "proofpath checks a claim against the sources it cites, and does not search for "
- "any: paste the claim with its source's address, or the address itself"
+ "proofpath checks a claim against the sources it cites; to have it search the web "
+ f"for evidence instead: proofpath config set {SEARCH_SETTING}, with TAVILY_API_KEY in "
+ f".env, or proofpath config set {SEARXNG_SETTING} plus search.base_url"
)
+# A provider is configured but cannot run (no key, no address). Reported as the
+# credential state, naming what to set -- never as a text that simply cites nothing.
+SEARCH_CANNOT_RUN = "the text cites no source, and the evidence search could not run"
+EVIDENCE_FOUND_TITLE = "a page proofpath found supports the claim"
+NO_EVIDENCE_TITLE = "no page proofpath found supports or contradicts the claim"
+NO_EVIDENCE_NOTE = "absence of evidence is not evidence against the claim"
+# A claim the models cannot read: not English, and no judge translated it. It is not
+# searched, and it is not called NEI either (OPEN-ITEMS 17.1a, Amendment A).
+LANGUAGE_TITLE = "the claim is not in English and was not translated"
+LANGUAGE_HINT = "the models read English only; --judge translates a claim before it is searched"
+# What a translated claim's finding says was actually checked. After the numeric
+# reason, which ``report`` reads back out of ``detail[0]``.
+CHECKED_AS = "checked as: "
# Said *before* the factories run rather than after: the first use of either model
# downloads it, which takes minutes, and a front end that says nothing here looks hung.
@@ -265,6 +289,13 @@ class Engine:
# for the final summary alone, and spec section 11.1 caps the summary at one
# call: escalating as well would quietly turn "one call" into several.
escalate: bool = True
+ # Evidence search for a text that cites nothing (OPEN-ITEMS 17.1a). ``None`` when
+ # no provider is configured; ``search_problem`` then says why a configured one
+ # could not be built (a missing key), and ``search`` is this run's own switch
+ # (``--no-search``, or ``permissions.web_search`` other than ``allow``).
+ searcher: Searcher | None = None
+ search_problem: str = ""
+ search: bool = True
# The polite HTTP client the social family reads posts through, and the one
# ``target_document`` uses when the target *is* a post (spec section 6.2). Built
# here by default so that every engine has one source family per spec section
@@ -304,6 +335,7 @@ def default(
thresholds: Thresholds = DEFAULT_THRESHOLDS,
judge: Judge | None = None,
escalate: bool = True,
+ search: bool = True,
) -> Engine:
"""The real wiring: the same one ``proofpath fetch`` builds, one layer up.
@@ -332,6 +364,8 @@ def default(
email = config.contact.email
client = PoliteClient(contact_email=email)
closers.append(client.client.close)
+ # On the run's polite client, so the provider is throttled like any host.
+ setup = build_searcher(config.search, client)
chain = OpenAccess(fetcher, client, contact_email=email, cache=cache)
# The resolver builds its own polite client, which would then have no
# owner to close it; handing it one keeps every socket on ``_closers``.
@@ -371,6 +405,10 @@ def scorer() -> Scorer:
device=device_name,
judge=judge,
escalate=escalate,
+ searcher=setup.searcher,
+ search_problem=setup.problem,
+ # The search never prompts, so ``ask`` is ``deny`` here (rule 4).
+ search=search and config.permissions.web_search == "allow",
_closers=closers,
)
@@ -437,6 +475,8 @@ class Prepared:
findings: list[Finding]
stages: list[Stage]
started: float # time.monotonic() when the run began
+ # Set only on a run that searched (OPEN-ITEMS 17.1a).
+ search: SearchSummary | None = None
def target_document(
@@ -691,6 +731,11 @@ def prepare(
"""
started = time.monotonic()
emit: Listener = on_event if on_event is not None else _ignore
+ if engine.judge is not None:
+ # A run starts on the configured provider, whatever the last one switched to
+ # (Amendment B 9), and a switch during it is told to this run's listener.
+ engine.judge.reset()
+ engine.judge.listen(_switch_notice(emit))
findings: list[Finding] = []
stages: list[Stage] = []
@@ -747,35 +792,74 @@ def closed(stage: str, by: str, summary: str, began: float) -> None:
# is what keeps the loss visible instead of it reading as a post that quoted
# nothing (product rule 2).
add(_finding(Kind.PARSE_ERROR, Locator(line=1), note))
- if document.kind in ("post", "page") and not document.references:
- # A post or a page that links to nothing has nothing behind it to check.
- # Reporting it as a document with no findings would make "nothing was
- # verified" look exactly like "everything checked out" (product rule 6).
- add(
- _finding(
- Kind.PARSE_ERROR,
- Locator(line=1),
- f"{document.kind} carries no links; nothing to verify against",
+ # Whether the document cites nothing at all, and whether this run will search for
+ # it (OPEN-ITEMS 17.1a). Markers rule a search out even on a post or a page with
+ # no links: "[1]" is the author citing something, and a search would put pages in
+ # its place the author never named (spec 2026-09-28 section 3).
+ unmarked = not document.references and not claims_mod.find_markers(document)
+ sourceless = not document.references and (
+ document.kind in ("post", "page") or (bool(document.paragraphs) and unmarked)
+ )
+ # A document a search could in principle run against: no references, no markers,
+ # and some text to search. The three branches below all start from this same
+ # question, so it is asked once instead of being spelled out three times.
+ searchable = not document.references and unmarked and bool(document.paragraphs)
+ chosen: claims_mod.Checkworthy | None = None
+ if searchable and allowed and engine.search and engine.searcher is not None:
+ chosen = claims_mod.checkworthy(document, limit=engine.config.search.max_claims)
+ # A text with no sentence to search ("???", a row of emoji) is not searched: a
+ # Searching stage over nothing would end the run with no finding at all, which
+ # reads as clean (product rule 6). It keeps the parse error below instead.
+ will_search = chosen is not None and bool(chosen.claims)
+ if sourceless and not will_search:
+ if searchable and engine.search and engine.search_problem and allowed:
+ # A provider was configured and cannot run. The fix is one setting, so
+ # the finding names it, in the credential state a missing key already has
+ # (spec 2026-09-28 section 4); a missing SearXNG address is fixed the same way.
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ Locator(line=1),
+ SEARCH_CANNOT_RUN,
+ state=CREDENTIALS_MISSING,
+ detail=(engine.search_problem,),
+ )
)
- )
- elif document.paragraphs and not document.references and not claims_mod.find_markers(document):
- # A file or pasted text that neither prints a bibliography nor links nor
- # marks a citation. The same silence as a post with no links, and reported
- # for the same reason; pasted text also gets told what to paste instead,
- # because the one who typed a bare claim is at the keyboard right now. Not
- # by ``reader``: a paste ending in ".pdf" is parsed as text but named by its
- # suffix, and the one thing every paste shares is not being a file.
- pasted = not _is_file(target)
- add(
- _finding(
- Kind.PARSE_ERROR,
- Locator(line=1),
- NO_SOURCE_IN_TEXT
- if pasted
- else f"{document.kind} cites nothing; nothing to verify against",
- detail=(NO_SOURCE_HINT,) if pasted else (),
+ elif document.kind in ("post", "page"):
+ # A post or a page that links to nothing has nothing behind it to check.
+ # Reporting it as a document with no findings would make "nothing was
+ # verified" look exactly like "everything checked out" (product rule 6).
+ add(
+ _finding(
+ Kind.PARSE_ERROR,
+ Locator(line=1),
+ f"{document.kind} carries no links; nothing to verify against",
+ )
+ )
+ else:
+ # A file or pasted text that neither prints a bibliography nor links nor
+ # marks a citation. The same silence as a post with no links, and reported
+ # for the same reason; pasted text also gets told how to have it searched,
+ # because the one who typed a bare claim is at the keyboard right now. Not
+ # by ``reader``: a paste ending in ".pdf" is parsed as text but named by its
+ # suffix, and the one thing every paste shares is not being a file.
+ pasted = not _is_file(target)
+ # The hint only makes sense when nothing is set up to search with. An
+ # engine that already has a searcher just did not search this run
+ # (``--no-search``, ``web_search`` ask/deny, a denied network, or no
+ # checkworthy sentence) -- telling the user to configure what is already
+ # configured would be wrong.
+ hint = pasted and engine.searcher is None
+ add(
+ _finding(
+ Kind.PARSE_ERROR,
+ Locator(line=1),
+ NO_SOURCE_IN_TEXT
+ if pasted
+ else f"{document.kind} cites nothing; nothing to verify against",
+ detail=(NO_SOURCE_HINT,) if hint else (),
+ )
)
- )
if not document.paragraphs and not document.references:
# Every page failed, or the file is a scan with no text layer. Without this
# the run would report nothing at all and read as clean (product rule 6).
@@ -833,6 +917,25 @@ def closed(stage: str, by: str, summary: str, began: float) -> None:
if not allowed and denied_note:
emit(Note(denied_note))
+ # --- 2b. searching (OPEN-ITEMS 17.1a) ------------------------------------
+ # Only for a document that cites nothing. Its claims are the check-worthy
+ # sentences, and its references are the pages found for them, each marked as
+ # found (``origin="search"``). From here on every stage runs exactly as it does
+ # for a cited source.
+ search: SearchSummary | None = None
+ if will_search and chosen is not None:
+ found = _search(
+ document,
+ chosen,
+ engine,
+ add=add,
+ emit=emit,
+ check=check,
+ opened=opened,
+ closed=closed,
+ )
+ document, claims, search = found.document, found.claims, found.summary
+
# --- 3. resolving -------------------------------------------------------
began = opened(RESOLVING, RESOLVERS_BY)
# By printed number only. ``claims`` already guarantees that every cited number
@@ -1081,6 +1184,12 @@ def record(number: int, read: EvidenceDoc) -> None:
began,
)
+ if search is not None:
+ # Read means full text or an abstract: what a verdict could be drawn from.
+ search = replace(
+ search,
+ pages_read=sum(1 for status in sources.values() if status.text_kind != "none"),
+ )
return Prepared(
document=document,
claims=claims,
@@ -1089,6 +1198,7 @@ def record(number: int, read: EvidenceDoc) -> None:
findings=findings,
stages=stages,
started=started,
+ search=search,
)
@@ -1118,6 +1228,9 @@ def decide_all(
carries the partial report, marked as partial (product rule 6).
"""
emit: Listener = on_event if on_event is not None else _ignore
+ if engine.judge is not None:
+ # Not reset here: a switch during the search is part of this same run.
+ engine.judge.listen(_switch_notice(emit))
findings = list(prepared.findings)
stages = list(prepared.stages)
results: list[ClaimResult] = []
@@ -1196,6 +1309,17 @@ def check() -> None:
for group, members in grouped.items():
add(_group_finding(group, members))
+ if prepared.search is not None and not cancelled:
+ # One line per searched claim that no found page decided. A cancelled run is
+ # left alone: an undecided claim there is unfinished, not unbacked.
+ backed = {
+ (result.claim.paragraph, result.claim.sentence)
+ for result in results
+ if result.verdict.label is not Label.NEI
+ }
+ for claim in prepared.claims.claims:
+ if (claim.paragraph, claim.sentence) not in backed:
+ add(_no_evidence(claim))
summary = _verify_summary(results, cached) if jobs else NOTHING_TO_VERIFY
elapsed = time.monotonic() - began
stages.append(Stage(name=VERIFYING, by=engine.device, summary=summary, elapsed=elapsed))
@@ -1226,11 +1350,14 @@ def check() -> None:
stages=tuple(stages),
models=_models(embedder, scorer, engine),
# Zero unless a judge ran: the default path calls nobody (spec section 11).
- api_calls=0 if judge_cost is None else judge_cost.calls,
+ # Provider requests only: a local fallback's answers are ``local_calls``.
+ api_calls=0 if judge_cost is None else judge_cost.calls - judge_cost.local_calls,
judge_cost=judge_cost,
elapsed=time.monotonic() - prepared.started,
tier_note=pipeline.tier_note(engine.thresholds),
cancelled=cancelled,
+ search=prepared.search,
+ judge_notice=None if engine.judge is None else engine.judge.switched,
)
if cancelled:
raise Cancelled("run cancelled", report=report)
@@ -1286,6 +1413,7 @@ def _summarise(
if judge is None:
raise ValueError("a summary needs a judge on the engine")
emit: Listener = on_event if on_event is not None else _ignore
+ judge.listen(_switch_notice(emit))
began = time.monotonic()
emit(StageStart(name=SUMMARISING, by=judge.name))
before = replace(judge.cost) # the running total, as it stood before this call
@@ -1316,7 +1444,7 @@ def _summarise(
*report.stages,
Stage(name=SUMMARISING, by=judge.name, summary=summary, elapsed=elapsed),
),
- api_calls=judge.cost.calls,
+ api_calls=judge.cost.calls - judge.cost.local_calls,
# A copy, not the client's own: ``JudgeCost`` is mutable and the client keeps
# adding to it for the next document, so an alias here would let a finished
# report's stated cost go on changing after the run that earned it was over.
@@ -1324,9 +1452,23 @@ def _summarise(
# The extra call is part of what the run took; a footer that left it out
# would price the summary at nothing.
elapsed=time.monotonic() - started,
+ judge_notice=judge.switched or report.judge_notice,
)
+def _switch_notice(emit: Listener) -> Callable[[str], None]:
+ """The judge's switch to its local fallback, as the run's one notice (spec 11).
+
+ Called on the thread that switched, before the local model is loaded, so a
+ front end can show the line while the wait it announces is still ahead.
+ """
+
+ def notice(text: str) -> None:
+ emit(Note(text, notice=True))
+
+ return notice
+
+
# --- the verifying stage ----------------------------------------------------------
# The line the group note carries. Quoted from section 15's PARAGRAPH-SCOPED row.
@@ -1406,14 +1548,17 @@ def _decide_claim(
stored or not stored at all -- which is what lets a cancelled run leave exactly
as many verdicts behind as it finished claims.
"""
- key = cache_mod.claim_hash(claim.text)
+ # The English text the models read: the claim itself, or the judge's translation
+ # of it (OPEN-ITEMS 17.1a). The cache is keyed by what was checked.
+ checked = claim.hypothesis or claim.text
+ key = cache_mod.claim_hash(checked)
stored = (
None if engine.cache is None else engine.cache.get_verdict(key, job.source_id, model_key)
)
if stored is not None:
return ClaimResult(claim, job.number, job.source_id, stored, from_cache=True)
verdict = pipeline.decide_indexed(
- claim.text, index, embedder, scorer, k=engine.k, thresholds=engine.thresholds
+ checked, index, embedder, scorer, k=engine.k, thresholds=engine.thresholds
)
if engine.cache is not None:
engine.cache.put_verdict(key, job.source_id, model_key, verdict)
@@ -1429,22 +1574,36 @@ def _verdict_finding(claim: Claim, status: SourceStatus, verdict: Verdict) -> Fi
note and may carry no passage at all: "the source does not say" is a different
statement from "the source says otherwise" (rule 2).
"""
- if verdict.label is Label.SUPPORTED:
- return None
+ found = status.reference.origin == "search"
+ source = "a page proofpath found" if found else "the cited source"
kind: Kind
detail: tuple[str, ...] = ()
- if verdict.label is Label.REFUTED and verdict.reason.startswith(NUMERIC_REASON):
+ if verdict.label is Label.SUPPORTED:
+ if not found:
+ return None
+ # The answer the user asked for: shown, with its passage (rule 1), where a
+ # supported *cited* claim stays silent.
+ kind = Kind.EVIDENCE_FOUND
+ title = EVIDENCE_FOUND_TITLE
+ elif verdict.label is Label.REFUTED and verdict.reason.startswith(NUMERIC_REASON):
# A rule decided this one and named both figures; the reason is the finding's
- # only note, and ``report`` reads the carets back out of it.
+ # first note, and ``report`` reads the carets back out of it.
kind = Kind.NUMERIC_MISMATCH
- title = "claim contradicts the cited source"
+ title = f"claim contradicts {source}"
detail = (verdict.reason,)
elif verdict.label is Label.REFUTED:
kind = Kind.NOT_SUPPORTED
- title = "claim is not supported by the cited source"
+ title = f"claim is not supported by {source}"
else:
kind = Kind.NEI
- title = "source neither supports nor contradicts the claim"
+ title = (
+ "a page proofpath found neither supports nor contradicts the claim"
+ if found
+ else "source neither supports nor contradicts the claim"
+ )
+ if claim.hypothesis:
+ # A translated claim says what the models actually read (Amendment A).
+ detail = (*detail, f"{CHECKED_AS}{claim.hypothesis}")
return Finding(
kind=kind,
level=LEVELS[kind],
@@ -1457,7 +1616,7 @@ def _verdict_finding(claim: Claim, status: SourceStatus, verdict: Verdict) -> Fi
source_id=status.source_id,
fetch_step=status.fetch_step,
tier=verdict.tier,
- detail=detail,
+ detail=_provenance(status.reference, detail),
group=claim.group,
)
@@ -1628,7 +1787,13 @@ def _judging(
ask: list[JudgeItem] = []
for index, result in escalated:
ident = f"c{index}"
- keys[ident] = (cache_mod.claim_hash(result.claim.text), result.source_id)
+ # Keyed by what the models actually read (the hypothesis when the claim was
+ # translated), matching ``_decide_claim``'s verdict cache key: a judgement
+ # about the English text belongs to that text, not to the reader's sentence.
+ keys[ident] = (
+ cache_mod.claim_hash(result.claim.hypothesis or result.claim.text),
+ result.source_id,
+ )
stored = (
None if engine.cache is None else engine.cache.get_judgement(*keys[ident], judge.name)
)
@@ -1641,7 +1806,7 @@ def _judging(
ask.append(
JudgeItem(
id=ident,
- claim=result.claim.text,
+ claim=result.claim.hypothesis or result.claim.text,
passage=passage.text,
verdict=result.verdict.label,
tier=result.verdict.tier,
@@ -1665,7 +1830,9 @@ def on_batch(done: int, total: int) -> None:
if engine.cache is not None:
try:
for ident, opinion in fresh.items():
- engine.cache.put_judgement(*keys[ident], judge.name, opinion)
+ # Under the model that gave it: a judge that switched mid-review has
+ # opinions from two models, and each belongs to its own cache row.
+ engine.cache.put_judgement(*keys[ident], opinion.model, opinion)
except sqlite3.Error as exc:
# The cache is a speed-up, never a gate. A judgement that cannot be
# stored -- a source row the cascade already took, a file gone read-only
@@ -1714,7 +1881,10 @@ def _with_opinion(item: Finding, by_claim: dict[tuple[str, str], JudgeOpinion])
"""
if item.claim is None or item.source_id is None:
return item
- opinion = by_claim.get((cache_mod.claim_hash(item.claim.text), item.source_id))
+ # Same key as ``_judging`` built ``by_claim`` with (hypothesis-or-text), so a
+ # translated claim's opinion is found under the English text it was asked about.
+ checked = item.claim.hypothesis or item.claim.text
+ opinion = by_claim.get((cache_mod.claim_hash(checked), item.source_id))
if opinion is None:
return item
detail = item.detail
@@ -1836,10 +2006,18 @@ def _finding(
source_id=source_id,
fetch_step=fetch_step,
tier=None,
- detail=detail,
+ detail=_provenance(reference, detail),
)
+def _provenance(reference: Reference | None, detail: tuple[str, ...]) -> tuple[str, ...]:
+ """A found page says so on every finding about it (OPEN-ITEMS 17.1a). Last, not
+ first: a numeric mismatch's reason is read back out of ``detail[0]``."""
+ if reference is not None and reference.origin == "search":
+ return (*detail, FOUND_BY_PROOFPATH)
+ return detail
+
+
def _parser_for(target: Path | str) -> str:
"""The parser a target will need, before it has been read.
@@ -2049,3 +2227,152 @@ def _nothing_read(status: SourceStatus) -> EvidenceDoc:
step=None,
title="the source was not read",
)
+
+
+# --- the searching stage (OPEN-ITEMS 17.1a) ----------------------------------------
+
+
+@dataclass(frozen=True)
+class _Searched:
+ document: Document
+ claims: Claims
+ summary: SearchSummary
+
+
+def _search(
+ document: Document,
+ chosen: claims_mod.Checkworthy,
+ engine: Engine,
+ *,
+ add: Callable[[Finding], None],
+ emit: Listener,
+ check: Callable[[], None],
+ opened: Callable[[str, str], float],
+ closed: Callable[[str, str, str, float], None],
+) -> _Searched:
+ """Search for each of ``chosen``'s sentences and turn the kept hits into references.
+
+ A provider that fails is reported once and the search stops there: the claims it
+ did not reach count as not searched rather than as searched and empty, which
+ would read as "nothing out there" (product rule 2). A claim that is not English
+ and was not translated is reported and skipped, never searched in a language the
+ models cannot read (Amendment A).
+ """
+ searcher = engine.searcher
+ if searcher is None: # pragma: no cover - ``will_search`` already checked
+ raise ValueError("no searcher on this engine")
+ settings = engine.config.search
+ began = opened(SEARCHING, searcher.name)
+ plan = plan_queries(
+ [claim.text for claim in chosen.claims], engine.judge if engine.escalate else None
+ )
+ if plan.notice:
+ emit(Note(plan.notice))
+ own = _own_host(document)
+ numbers: dict[str, int] = {}
+ references: list[Reference] = []
+ searched: list[Claim] = []
+ total = len(chosen.claims)
+ for index, (claim, queries, hypothesis) in enumerate(
+ zip(chosen.claims, plan.queries, plan.hypotheses, strict=True), start=1
+ ):
+ if hypothesis is None:
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ claim.locator,
+ LANGUAGE_TITLE,
+ state=LANGUAGE_UNSUPPORTED,
+ detail=(LANGUAGE_HINT,),
+ )
+ )
+ emit(Progress(name=SEARCHING, done=index, total=total, detail=claim.locator.label()))
+ check()
+ continue
+ hits: list[SearchHit] = []
+ try:
+ for query in queries:
+ hits.extend(searcher.search(query, settings.results_per_claim))
+ except ProviderError as exc:
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ claim.locator,
+ "the evidence search did not answer",
+ state=SEARCH_UNAVAILABLE,
+ detail=(f"{searcher.name}: {exc}",),
+ )
+ )
+ break
+ cited: list[int] = []
+ for hit in keep_hits(hits, own_host=own, limit=settings.results_per_claim):
+ key = canonical(hit.url)
+ if key not in numbers:
+ numbers[key] = len(references) + 1
+ references.append(Reference(numbers[key], hit.url, claim.locator, origin="search"))
+ cited.append(numbers[key])
+ searched.append(
+ replace(
+ claim,
+ cited_refs=tuple(cited),
+ hypothesis=None if hypothesis == claim.text else hypothesis,
+ )
+ )
+ emit(Progress(name=SEARCHING, done=index, total=total, detail=claim.locator.label()))
+ check()
+ closed(
+ SEARCHING,
+ searcher.name,
+ f"{len(searched)} of {chosen.eligible} claims, {len(references)} pages",
+ began,
+ )
+ summary = SearchSummary(
+ by=searcher.name,
+ queries_by=plan.by,
+ sentences=chosen.sentences,
+ eligible=chosen.eligible,
+ searched=len(searched),
+ pages_found=len(references),
+ pages_read=0, # filled in once the fetching stage has run
+ notices=(plan.notice,) if plan.notice else (),
+ )
+ # ``markers=()`` on purpose: the author made no citation, and the report's marker
+ # count must not say they did.
+ return _Searched(
+ document=replace(document, references=tuple(references)),
+ claims=Claims(claims=tuple(searched), markers=(), unsupported=(), unresolved=()),
+ summary=summary,
+ )
+
+
+def _own_host(document: Document) -> str | None:
+ """The host of a page or post read as the document: never found to back itself."""
+ match = _BARE_URL.fullmatch(document.name.strip())
+ return urlsplit(match.group(0)).hostname if match else None
+
+
+def _no_evidence(claim: Claim) -> Finding:
+ """A searched claim no found page decided: not REFUTED, and it says so (rule 2).
+
+ A translated claim (``claim.hypothesis`` set) also says what was checked, the same
+ way a verdict finding does (spec §2): the reader's own sentence stays the title and
+ locator, and the ``checked as:`` note names the English text the models actually
+ searched and judged.
+ """
+ return Finding(
+ kind=Kind.NO_EVIDENCE,
+ level=LEVELS[Kind.NO_EVIDENCE],
+ locator=claim.locator,
+ title=NO_EVIDENCE_TITLE,
+ state=STATE_WORDS[Kind.NO_EVIDENCE],
+ reference=None,
+ claim=claim,
+ verdict=None,
+ source_id=None,
+ fetch_step=None,
+ tier=None,
+ detail=(
+ NO_EVIDENCE_NOTE,
+ *((f"{CHECKED_AS}{claim.hypothesis}",) if claim.hypothesis else ()),
+ ),
+ )
diff --git a/tests/data/verify-report-golden.json b/tests/data/verify-report-golden.json
index 9c6fbb1..6e055e5 100644
--- a/tests/data/verify-report-golden.json
+++ b/tests/data/verify-report-golden.json
@@ -79,7 +79,8 @@
"line": 5,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
{
"number": 2,
@@ -88,7 +89,8 @@
"line": 6,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
{
"number": 3,
@@ -97,7 +99,8 @@
"line": 7,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
{
"number": 4,
@@ -106,7 +109,8 @@
"line": 8,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
}
],
"pages": 1,
@@ -123,7 +127,8 @@
"line": 5,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"resolve": {
"state": "RESOLVED",
@@ -168,7 +173,8 @@
"line": 6,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"resolve": {
"state": "GHOST REFERENCE",
@@ -197,7 +203,8 @@
"line": 7,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"resolve": {
"state": "UNVERIFIED (not in bibliographic indexes)",
@@ -226,7 +233,8 @@
"line": 8,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"resolve": {
"state": "RESOLVED",
@@ -291,7 +299,8 @@
"paragraph": 0,
"start": 42,
"end": 45
- }
+ },
+ "hypothesis": null
},
"reference": 1,
"source_id": "doi:10.1038/s41586-021-03819-2",
@@ -333,7 +342,8 @@
"paragraph": 0,
"start": 115,
"end": 118
- }
+ },
+ "hypothesis": null
},
"reference": 3,
"source_id": "url:https://example.test/air-quality.html",
@@ -371,7 +381,8 @@
"paragraph": 0,
"start": 156,
"end": 159
- }
+ },
+ "hypothesis": null
},
"reference": 4,
"source_id": "doi:10.5555/closed-access-2020",
@@ -404,7 +415,8 @@
"line": 7,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"claim": {
"text": "Air quality limits were tightened in 2021.",
@@ -429,7 +441,8 @@
"paragraph": 0,
"start": 115,
"end": 118
- }
+ },
+ "hypothesis": null
},
"verdict": {
"label": "NEI",
@@ -462,7 +475,8 @@
"line": 8,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"claim": {
"text": "The paper reports a measured effect.",
@@ -487,7 +501,8 @@
"paragraph": 0,
"start": 156,
"end": 159
- }
+ },
+ "hypothesis": null
},
"verdict": {
"label": "NEI",
@@ -520,7 +535,8 @@
"line": 6,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"claim": null,
"verdict": null,
@@ -550,7 +566,8 @@
"line": 8,
"column": 1,
"page": null
- }
+ },
+ "origin": "author"
},
"claim": null,
"verdict": null,
@@ -626,5 +643,7 @@
"cancelled": false,
"summary": null,
"summary_model": null,
- "judge_cost": null
+ "judge_cost": null,
+ "search": null,
+ "judge_notice": null
}
diff --git a/tests/fixtures/search/searxng.json b/tests/fixtures/search/searxng.json
new file mode 100644
index 0000000..a284713
--- /dev/null
+++ b/tests/fixtures/search/searxng.json
@@ -0,0 +1,8 @@
+{
+ "query": "ChatGPT shut down 2025",
+ "number_of_results": 2,
+ "results": [
+ {"url": "https://wiki.test/ChatGPT", "title": "ChatGPT", "content": "wiki", "engine": "wikipedia"},
+ {"url": "ftp://files.test/x", "title": "Not the web", "content": "", "engine": "x"}
+ ]
+}
diff --git a/tests/fixtures/search/tavily.json b/tests/fixtures/search/tavily.json
new file mode 100644
index 0000000..42ec53d
--- /dev/null
+++ b/tests/fixtures/search/tavily.json
@@ -0,0 +1,11 @@
+{
+ "query": "ChatGPT shut down 2025",
+ "results": [
+ {"title": "OpenAI statement", "url": "https://news.test/openai-statement?utm_source=x", "content": "SNIPPET-MUST-NOT-BE-EVIDENCE", "score": 0.91},
+ {"title": "Duplicate", "url": "https://news.test/openai-statement", "content": "dup", "score": 0.8},
+ {"title": "A post", "url": "https://x.com/someone/status/1", "content": "post", "score": 0.7},
+ {"title": "Encyclopedia", "url": "https://wiki.test/ChatGPT", "content": "wiki", "score": 0.6},
+ {"title": "No address", "content": "broken entry", "score": 0.5}
+ ],
+ "response_time": 0.8
+}
diff --git a/tests/test_averitec.py b/tests/test_averitec.py
index 69daeb6..b1cfaa8 100644
--- a/tests/test_averitec.py
+++ b/tests/test_averitec.py
@@ -25,6 +25,7 @@
"claim": "first claim",
"label": "Supported",
"justification": "because the sources say so",
+ "fact_checking_article": "https://factcheck.test/claim-1",
"questions": [
{
"question": "where was it published",
@@ -79,6 +80,18 @@ def test_evidence_urls_are_deduplicated_in_order_of_first_appearance(tmp_path: P
assert claims[1].source_urls == ("https://c.example/three",)
+def test_fact_check_url_is_read_from_the_fact_checking_article_key(tmp_path: Path) -> None:
+ path, digest = _write_dataset(tmp_path)
+ claims = averitec.load(path, expected_sha256=digest)
+ assert claims[0].fact_check_url == "https://factcheck.test/claim-1"
+
+
+def test_a_row_without_a_fact_checking_article_loads_as_empty(tmp_path: Path) -> None:
+ path, digest = _write_dataset(tmp_path)
+ claims = averitec.load(path, expected_sha256=digest)
+ assert claims[1].fact_check_url == ""
+
+
def test_limit_takes_the_first_n_claims(tmp_path: Path) -> None:
path, digest = _write_dataset(tmp_path)
claims = averitec.load(path, limit=1, expected_sha256=digest)
diff --git a/tests/test_cache.py b/tests/test_cache.py
index dc2533d..d65d624 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -670,7 +670,7 @@ def test_a_judgement_round_trips_and_is_keyed_by_the_judge_model(db: Cache) -> N
db.put_judgement("h", "s", "groq gpt-oss", OPINION, now=NOW)
assert db.get_judgement("h", "s", "groq gpt-oss") == OPINION
# Another judge has not been asked; absence of its opinion is not an opinion.
- assert db.get_judgement("h", "s", "ollama llama3.1") is None
+ assert db.get_judgement("h", "s", "ollama qwen3.5:9b") is None
assert db.get_judgement("other", "s", "groq gpt-oss") is None
diff --git a/tests/test_check_cli.py b/tests/test_check_cli.py
index 79e03cf..bf3c5a6 100644
--- a/tests/test_check_cli.py
+++ b/tests/test_check_cli.py
@@ -226,6 +226,26 @@ def write(tmp_path: Path, body: str, entries: Sequence[str] = (REAL,)) -> Path:
return path
+# --- --no-search ----------------------------------------------------------------------
+
+
+def test_no_search_is_passed_to_the_engine(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ seen = install(monkeypatch)
+ path = write(tmp_path, "Transformers improved translation quality [1].")
+ runner.invoke(app, ["check", "--no-search", str(path)])
+ assert seen["search"] is False
+
+
+def test_search_is_on_unless_told_otherwise(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ seen = install(monkeypatch)
+ runner.invoke(
+ app, ["check", str(write(tmp_path, "Transformers improved translation quality [1]."))]
+ )
+ assert seen["search"] is True
+
+
# --- the clean and the troubled run --------------------------------------------------
diff --git a/tests/test_claims.py b/tests/test_claims.py
index a175998..281bf46 100644
--- a/tests/test_claims.py
+++ b/tests/test_claims.py
@@ -13,9 +13,11 @@
from proofpath.claims import (
AUTHOR_YEAR,
NUMERIC,
+ checkworthy,
expand,
extract,
find_markers,
+ is_checkworthy,
pair,
pair_links,
strip_links,
@@ -848,3 +850,61 @@ def test_a_bracketed_number_in_a_link_citing_documents_prose_is_not_a_citation()
assert [claim.cited_refs for claim in result.claims] == [(1,), (1,)]
assert result.unresolved == ()
+
+
+# --- `is_checkworthy` and `checkworthy` -----------------------------------------------
+
+
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("ChatGPT was shut down yesterday.", True), # a name with an inner capital
+ ("The unemployment rate fell to 3.9% in 2023.", True), # a figure
+ ("Istanbul is the largest city in Turkey.", True), # a name past the first word
+ ('He said "we will never raise taxes" on stage.', True), # a quotation
+ ("I think this is really bad for everyone.", False), # opinion, nothing to find
+ ("Did Tesla really sell 2 million cars?", False), # a question
+ ("It rose.", False), # too short to state anything searchable
+ ("#Breaking #Tesla #News #Crash #Today", False), # hashtags label, they state nothing
+ ("İstanbul büyük bir şehirdir bugün.", True), # Turkish, name first
+ ("Ankara Türkiye'nin başkentidir.", True), # Turkish, apostrophe-joined suffix
+ ("Bu çok kötü bir haber.", False), # Turkish opinion, "Bu" is a common opener
+ ("OpenAI battı.", False), # a name, but only two words # noqa: RUF001
+ ],
+)
+def test_is_checkworthy(text: str, expected: bool) -> None:
+ assert is_checkworthy(text) is expected
+
+
+def test_checkworthy_keeps_document_order_up_to_the_cap_and_counts_the_rest() -> None:
+ doc = ingest.from_text(
+ "NASA landed on the Moon in 1969. I love this so much honestly. "
+ "Paris is the capital of France. The rate was 4% in 2020 overall.",
+ name="pasted text",
+ kind="text",
+ )
+ found = checkworthy(doc, limit=2)
+
+ assert [claim.text for claim in found.claims] == [
+ "NASA landed on the Moon in 1969.",
+ "Paris is the capital of France.",
+ ]
+ assert found.eligible == 3
+ assert found.sentences == 4
+ assert all(claim.cited_refs == () and claim.marker is None for claim in found.claims)
+
+
+def test_a_text_with_nothing_checkworthy_is_searched_sentence_by_sentence() -> None:
+ found = checkworthy(ingest.from_text("OpenAI battı", name="pasted text", kind="text"), limit=5) # noqa: RUF001
+ assert [claim.text for claim in found.claims] == ["OpenAI battı"] # noqa: RUF001
+ assert (found.eligible, found.sentences) == (1, 1)
+
+
+def test_the_fallback_still_respects_the_cap() -> None:
+ body = " ".join(
+ f"I really feel sad number {word}."
+ for word in ["one", "two", "three", "four", "five", "six"]
+ )
+ found = checkworthy(ingest.from_text(body, name="pasted text", kind="text"), limit=5)
+ assert len(found.claims) == 5
+ assert found.eligible == found.sentences == 6
diff --git a/tests/test_config.py b/tests/test_config.py
index 267c3a5..d000912 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -110,3 +110,55 @@ def test_set_value_rejects_unknown_key_and_bad_value(config_dir: Path) -> None:
cfg.set_value("nope.key", "x")
with pytest.raises(cfg.ConfigError, match="network"):
cfg.set_value("permissions.network", "sometimes")
+
+
+# --- search (OPEN-ITEMS 17.1a) -----------------------------------------------------
+
+
+def test_search_is_off_by_default_and_web_search_is_allowed() -> None:
+ config = cfg.Config()
+ assert config.search == cfg.SearchConfig()
+ assert config.search.provider == "off"
+ assert config.search.api_key_env == "TAVILY_API_KEY"
+ assert (config.search.max_claims, config.search.results_per_claim) == (5, 3)
+ assert config.permissions.web_search == "allow"
+
+
+def test_the_search_section_loads_from_toml(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ path.write_text(
+ '[search]\nprovider = "searxng"\nbase_url = "http://localhost:8888"\nmax_claims = 2\n',
+ encoding="utf-8",
+ )
+ config = cfg.load_config(path)
+ assert config.search.provider == "searxng"
+ assert config.search.base_url == "http://localhost:8888"
+ assert config.search.max_claims == 2
+
+
+@pytest.mark.parametrize(
+ "body",
+ ['provider = "bing"', "max_claims = 0", "max_claims = true", 'max_claims = "5"'],
+)
+def test_bad_search_values_are_refused_by_name(tmp_path: Path, body: str) -> None:
+ path = tmp_path / "config.toml"
+ path.write_text(f"[search]\n{body}\n", encoding="utf-8")
+ with pytest.raises(cfg.ConfigError, match=r"search\."):
+ cfg.load_config(path)
+
+
+def test_set_value_parses_an_integer_and_round_trips_it(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ updated = cfg.set_value("search.max_claims", "3", path)
+ assert updated.search.max_claims == 3
+ assert "max_claims = 3" in path.read_text(encoding="utf-8")
+ assert cfg.load_config(path).search.max_claims == 3
+ with pytest.raises(cfg.ConfigError, match="positive integer"):
+ cfg.set_value("search.max_claims", "many", path)
+
+
+def test_set_value_takes_a_search_provider(tmp_path: Path) -> None:
+ path = tmp_path / "config.toml"
+ assert cfg.set_value("search.provider", "tavily", path).search.provider == "tavily"
+ with pytest.raises(cfg.ConfigError, match=r"search\.provider"):
+ cfg.set_value("search.provider", "bing", path)
diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py
index 13f1462..73de263 100644
--- a/tests/test_config_cli.py
+++ b/tests/test_config_cli.py
@@ -28,7 +28,7 @@ def test_config_prints_path_and_every_section_as_toml(config_dir: Path) -> None:
assert result.exit_code == 0, result.output
assert str(config_dir / "config.toml") in result.stdout
assert "not written yet" in result.stdout
- for section in ("[permissions]", "[fetch]", "[contact]", "[judge]"):
+ for section in ("[permissions]", "[fetch]", "[contact]", "[judge]", "[search]"):
assert section in result.stdout
assert 'install_browser = "ask"' in result.stdout
assert "GROQ_API_KEY" in result.stdout
diff --git a/tests/test_eval_averitec.py b/tests/test_eval_averitec.py
index e185721..038d93e 100644
--- a/tests/test_eval_averitec.py
+++ b/tests/test_eval_averitec.py
@@ -15,6 +15,9 @@
import pytest
+from proofpath.eval import averitec
+from proofpath.search import SearchHit
+
SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "eval_averitec.py"
@@ -35,6 +38,7 @@ def _load_script() -> ModuleType:
score = eval_averitec.score
render_report = eval_averitec.render_report
state_for = eval_averitec.state_for
+search_urls = eval_averitec.search_urls
NOT_A_URL = eval_averitec.NOT_A_URL
NO_SOURCE = eval_averitec.NO_SOURCE
@@ -250,6 +254,64 @@ def test_the_report_ends_with_a_newline() -> None:
assert render_report(score(ROWS), date="2026-09-15", limit=100).endswith("\n")
+# --- search_urls -----------------------------------------------------------
+
+
+def test_search_urls_never_hand_back_the_fact_check_itself() -> None:
+ class Stub:
+ name = "stub"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ return [
+ SearchHit("https://factcheck.test/claim-1", "", 1),
+ SearchHit("https://factcheck.test/other", "", 2),
+ SearchHit("https://news.test/a", "", 3),
+ ]
+
+ claim = averitec.Claim(
+ id=0, text="A claim.", label="Refuted", source_urls=(),
+ fact_check_url="https://factcheck.test/claim-1",
+ ) # fmt: skip
+ assert search_urls(Stub(), claim, 3) == ("https://news.test/a",)
+
+
+def test_search_urls_exclude_the_checker_across_subdomains() -> None:
+ class Stub:
+ name = "stub"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ return [
+ SearchHit("https://www.afp.com/en/story", "", 1),
+ SearchHit("https://factcheck.afp.com/other", "", 2),
+ SearchHit("https://news.afp.com.evil.test/a", "", 3),
+ SearchHit("https://notafp.com/b", "", 4),
+ ]
+
+ claim = averitec.Claim(
+ id=0, text="A claim.", label="Refuted", source_urls=(),
+ fact_check_url="https://factcheck.afp.com/claim-1",
+ ) # fmt: skip
+ assert search_urls(Stub(), claim, 3) == (
+ "https://news.afp.com.evil.test/a",
+ "https://notafp.com/b",
+ )
+
+
+def test_search_urls_ask_for_enough_hits_to_fill_the_limit() -> None:
+ asked: list[int] = []
+
+ class Stub:
+ name = "stub"
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ asked.append(max_results)
+ return []
+
+ claim = averitec.Claim(id=0, text="A claim.", label="Refuted", source_urls=())
+ search_urls(Stub(), claim, 3)
+ assert asked == [9]
+
+
# --- main ----------------------------------------------------------------
diff --git a/tests/test_judge.py b/tests/test_judge.py
index 05d772c..e7c6e6d 100644
--- a/tests/test_judge.py
+++ b/tests/test_judge.py
@@ -5,6 +5,7 @@
import importlib.resources
import json
import re
+from dataclasses import replace
from pathlib import Path
from typing import Any
@@ -115,6 +116,10 @@ def test_check_without_a_key_on_a_keyed_provider_fails_cleanly() -> None:
# Ollama is the preset with no API key, so these tests need no secret at all.
OLLAMA = judge.provider_defaults("ollama")
+# The same keyless endpoint under another provider name: Ollama clients always send
+# ``reasoning_effort: none`` (fix round 1), so the tests about passing the caller's
+# effort through, and about the downgrade, need a provider that does not.
+PASSTHROUGH = replace(OLLAMA, provider="openai-compatible")
OLLAMA_URL = "http://localhost:11434/v1/chat/completions"
GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
MESSAGES = [
@@ -126,7 +131,7 @@ def test_check_without_a_key_on_a_keyed_provider_fails_cleanly() -> None:
def _ok(content: str = '{"opinions": []}', **extra: object) -> httpx.Response:
body: dict[str, object] = {
- "model": "llama3.1",
+ "model": "qwen3.5:9b",
"choices": [{"message": {"content": content}}],
}
body.update(extra)
@@ -146,7 +151,7 @@ def test_complete_sends_the_json_schema_and_fills_usage_from_the_response() -> N
client = judge.JudgeClient(OLLAMA, None)
done = client.complete(MESSAGES, json_schema=SCHEMA)
sent = json.loads(route.calls.last.request.content)
- assert sent["model"] == "llama3.1"
+ assert sent["model"] == "qwen3.5:9b"
assert sent["messages"] == MESSAGES
assert sent["max_tokens"] == 1024
assert sent["temperature"] == 0.0
@@ -156,13 +161,13 @@ def test_complete_sends_the_json_schema_and_fills_usage_from_the_response() -> N
# Ollama takes no key, so no Authorization header may be sent.
assert "authorization" not in route.calls.last.request.headers
assert done == judge.Completion(
- text='{"opinions": []}', prompt_tokens=120, completion_tokens=7, model="llama3.1"
+ text='{"opinions": []}', prompt_tokens=120, completion_tokens=7, model="qwen3.5:9b"
)
assert client.cost.calls == 1
assert client.cost.prompt_tokens == 120
assert client.cost.completion_tokens == 7
assert client.cost.waited_s == 0.0
- assert client.cost.model == "llama3.1"
+ assert client.cost.model == "qwen3.5:9b"
@respx.mock
@@ -196,11 +201,25 @@ def test_a_provider_that_rejects_the_schema_is_retried_once_with_json_object() -
assert second["response_format"] == {"type": "json_object"}
+@respx.mock
+def test_an_ollama_judge_is_always_asked_not_to_think() -> None:
+ """``judge.provider ollama`` defaults to qwen3.5:9b, a thinking model: on ``low``
+ it spent a 1500-token budget thinking and answered nothing (live, 2026-09-28)."""
+ assert OLLAMA.model == "qwen3.5:9b"
+ route = respx.post(OLLAMA_URL).mock(return_value=_ok("a paragraph"))
+ client = judge.JudgeClient(OLLAMA, None)
+ client.complete(MESSAGES, reasoning_effort="low")
+ assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "none"
+ client.complete(MESSAGES)
+ assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "none"
+ client.close()
+
+
@respx.mock
def test_complete_sends_the_reasoning_effort_it_was_given_and_omits_it_otherwise() -> None:
"""A reasoning model spends its budget thinking unless it is told how much to."""
route = respx.post(OLLAMA_URL).mock(return_value=_ok("a paragraph"))
- client = judge.JudgeClient(OLLAMA, None)
+ client = judge.JudgeClient(PASSTHROUGH, None)
client.complete(MESSAGES, reasoning_effort="low")
assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "low"
@@ -240,7 +259,7 @@ def test_each_downgrade_costs_one_extra_request_and_no_more() -> None:
_ok(),
]
)
- client = judge.JudgeClient(OLLAMA, None)
+ client = judge.JudgeClient(PASSTHROUGH, None)
done = client.complete(MESSAGES, json_schema=SCHEMA, reasoning_effort="low")
assert done.text == '{"opinions": []}'
@@ -270,7 +289,7 @@ def test_a_provider_that_refuses_reasoning_effort_is_only_taught_once() -> None:
_ok("second"),
]
)
- client = judge.JudgeClient(OLLAMA, None)
+ client = judge.JudgeClient(PASSTHROUGH, None)
client.complete(MESSAGES, reasoning_effort="low")
client.complete(MESSAGES, reasoning_effort="low")
@@ -310,7 +329,7 @@ def test_a_provider_that_refuses_the_schema_is_only_taught_once() -> None:
def test_a_client_that_never_sees_a_400_keeps_asking_for_both_fields() -> None:
"""The latch only ever closes on a refusal: a provider that takes both keeps both."""
route = respx.post(OLLAMA_URL).mock(return_value=_ok())
- client = judge.JudgeClient(OLLAMA, None)
+ client = judge.JudgeClient(PASSTHROUGH, None)
client.complete(MESSAGES, json_schema=SCHEMA, reasoning_effort="low")
client.complete(MESSAGES, json_schema=SCHEMA, reasoning_effort="low")
@@ -487,7 +506,7 @@ def test_a_filtered_answer_is_unavailable_and_names_the_finish_reason_only() ->
return_value=httpx.Response(
200,
json={
- "model": "llama3.1",
+ "model": "qwen3.5:9b",
"system_fingerprint": "gsk_secret_value",
"choices": [{"message": {"content": None}, "finish_reason": "content_filter"}],
},
@@ -505,7 +524,7 @@ def test_a_filtered_answer_is_unavailable_and_names_the_finish_reason_only() ->
def test_an_empty_string_answer_is_unavailable_rather_than_an_empty_completion() -> None:
respx.post(OLLAMA_URL).mock(
return_value=httpx.Response(
- 200, json={"model": "llama3.1", "choices": [{"message": {"content": ""}}]}
+ 200, json={"model": "qwen3.5:9b", "choices": [{"message": {"content": ""}}]}
)
)
client = judge.JudgeClient(OLLAMA, None)
@@ -675,7 +694,7 @@ def test_review_sends_two_messages_and_a_strict_schema() -> None:
# The per-item field the template names is `verdict`: the local label and tier.
assert "verdict: NEI (low)" in sent["messages"][-1]["content"]
assert opinions["c1"] == judge.JudgeOpinion(
- label=Label.NEI, rationale='the passage says "faster"', model="ollama llama3.1"
+ label=Label.NEI, rationale='the passage says "faster"', model="ollama qwen3.5:9b"
)
assert reviewer.cost.calls == 1
assert not reviewer.unavailable and reviewer.skipped == []
@@ -686,7 +705,7 @@ def test_review_asks_for_low_reasoning_and_a_budget_it_can_answer_within() -> No
"""A reasoning model charges its thinking to ``max_tokens``: too small a budget
comes back ``finish_reason=length`` with no opinions in it at all."""
route = respx.post(OLLAMA_URL).mock(side_effect=_answers())
- reviewer = judge.Judge(judge.JudgeClient(OLLAMA, None))
+ reviewer = judge.Judge(judge.JudgeClient(PASSTHROUGH, None))
reviewer.review([_item("c1")])
sent = json.loads(route.calls.last.request.content)
@@ -845,9 +864,9 @@ def test_review_never_raises_when_the_client_misbehaves() -> None:
"""Nothing the optional layer does may reach the run (spec section 11.1)."""
class Broken:
- cost = judge.JudgeCost(model="llama3.1")
+ cost = judge.JudgeCost(model="qwen3.5:9b")
provider = "ollama"
- model = "llama3.1"
+ model = "qwen3.5:9b"
def complete(self, *args: object, **kwargs: object) -> judge.Completion:
raise ZeroDivisionError("a bug in the adapter")
@@ -859,7 +878,7 @@ def complete(self, *args: object, **kwargs: object) -> judge.Completion:
def test_the_judge_names_itself_by_provider_and_model() -> None:
reviewer = judge.Judge(judge.JudgeClient(OLLAMA, None))
- assert reviewer.name == "ollama llama3.1"
+ assert reviewer.name == "ollama qwen3.5:9b"
reviewer.close()
@@ -880,7 +899,7 @@ def test_closing_the_judge_closes_the_client_it_was_given() -> None:
@respx.mock
def test_summarize_sends_the_finished_report_as_plain_text() -> None:
route = respx.post(OLLAMA_URL).mock(return_value=_ok(f" {PARAGRAPH}\n"))
- reviewer = judge.Judge(judge.JudgeClient(OLLAMA, None))
+ reviewer = judge.Judge(judge.JudgeClient(PASSTHROUGH, None))
text = reviewer.summarize(REPORT)
sent = json.loads(route.calls.last.request.content)
diff --git a/tests/test_judge_fallback.py b/tests/test_judge_fallback.py
new file mode 100644
index 0000000..000b193
--- /dev/null
+++ b/tests/test_judge_fallback.py
@@ -0,0 +1,859 @@
+"""The local fallback judge (spec section 11, Task 11 and its Amendment B).
+
+Groq's free tier runs out; the run then switches, at once, to qwen3.5:9b in the
+user's own Ollama (or the model ``judge.fallback`` names), and says so. Every failure mode the user
+has seen in a fallback like this before has a test here: a primary that still retries or waits, a
+lost batch, a switch that races across threads, a notice that only comes after the
+model has loaded, thinking output that breaks the JSON, and a new run that stays on
+the local model. Nothing here opens a socket: every endpoint is a respx route.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from collections.abc import Callable, Iterator
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+import respx
+from typer.testing import CliRunner
+
+from proofpath import judge
+from proofpath import verify as verify_mod
+from proofpath.cli import app
+from proofpath.config import Config, JudgeConfig
+from proofpath.events import Event, Note
+from proofpath.models import Label
+from proofpath.report import render_footer, render_markdown
+from proofpath.secrets import ApiKey
+from tests.test_verify_search import CLAIM, StubSearcher, searching
+
+GROQ = JudgeConfig()
+KEY = ApiKey("gsk_fallback_test_secret", source="test")
+GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
+TAGS_URL = "http://localhost:11434/api/tags"
+WARM_URL = "http://localhost:11434/api/generate"
+LOCAL_URL = "http://localhost:11434/v1/chat/completions"
+LOCAL = "qwen3.5:9b" # the default ``judge.fallback``
+# The user's own `ollama list` on 2026-09-28, after the clean-up, in Ollama's order.
+INSTALLED = [
+ "qwen3-embedding:8b",
+ "nomic-embed-text:latest",
+ "qwen3-embedding:0.6b",
+ "gemma4:12b",
+ "qwen3.5:9b",
+ "qwen3.6:35b-a3b",
+ "qwen2.5-coder:7b-instruct-q4_K_M",
+]
+MESSAGES = [
+ {"role": "system", "content": "Answer only from the passage."},
+ {"role": "user", "content": "- id: c1\n claim: x\n passage: y\n verdict: NEI (low)"},
+]
+SCHEMA = {"type": "object", "properties": {"opinions": {"type": "array"}}}
+OPINION = json.dumps({"opinions": [{"id": "c1", "label": "NEI", "rationale": "silent"}]})
+LIMIT = f"Groq limit reached — judging with local ollama {LOCAL}"
+
+
+@pytest.fixture
+def mock() -> Iterator[respx.MockRouter]:
+ """Every endpoint as a route; a route a test does not reach is not an error."""
+ with respx.mock(assert_all_called=False) as router:
+ yield router
+
+
+def _tags(*names: str) -> httpx.Response:
+ return httpx.Response(200, json={"models": [{"name": name} for name in names]})
+
+
+def _ok(content: str, model: str = LOCAL) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "model": model,
+ "choices": [{"message": {"content": content}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 50, "completion_tokens": 5},
+ },
+ )
+
+
+class Sleeps:
+ """The client's ``sleep``, recorded: a fallback must never be reached by waiting."""
+
+ def __init__(self) -> None:
+ self.calls: list[float] = []
+
+ def __call__(self, seconds: float) -> None:
+ self.calls.append(seconds)
+
+
+def fallback(
+ *,
+ setting: str = LOCAL,
+ sleep: Sleeps | None = None,
+ on_switch: Callable[[str], None] | None = None,
+) -> judge.FallbackClient:
+ """A Groq primary behind the fallback, exactly as ``build_client`` wires it."""
+ primary = judge.JudgeClient(GROQ, KEY, sleep=sleep or Sleeps())
+ return judge.FallbackClient(primary, fallback=setting, on_switch=on_switch)
+
+
+def routes(
+ mock: respx.MockRouter,
+ *,
+ groq: httpx.Response | Exception | list[httpx.Response] | None = None,
+ tags: httpx.Response | Exception | None = None,
+ local: httpx.Response | None = None,
+) -> tuple[respx.Route, respx.Route, respx.Route, respx.Route]:
+ """Groq, ``/api/tags``, the warm-up and the local chat route, in that order."""
+ groq_route = mock.post(GROQ_URL)
+ if isinstance(groq, (list, Exception)):
+ groq_route.mock(side_effect=groq)
+ else:
+ groq_route.mock(return_value=groq or httpx.Response(429))
+ tags_route = mock.get(TAGS_URL)
+ if isinstance(tags, Exception):
+ tags_route.mock(side_effect=tags)
+ else:
+ tags_route.mock(return_value=tags or _tags(*INSTALLED))
+ warm = mock.post(WARM_URL).mock(return_value=httpx.Response(200, json={"done": True}))
+ local_route = mock.post(LOCAL_URL).mock(return_value=local or _ok(OPINION))
+ return groq_route, tags_route, warm, local_route
+
+
+# --- which model ------------------------------------------------------------------
+
+
+def test_the_default_fallback_is_qwen3_5() -> None:
+ """The user's choice (2026-09-28): Groq, then qwen3.5:9b, and nothing else."""
+ assert JudgeConfig().fallback == "qwen3.5:9b" == judge.LOCAL_JUDGE_MODEL
+
+
+def test_the_default_config_falls_back_to_qwen3_5(mock: respx.MockRouter) -> None:
+ _, tags, _, local = routes(mock)
+ client = judge.build_client(GROQ, KEY)
+ client.complete(MESSAGES)
+ assert tags.call_count == 1 # only to confirm it is installed
+ assert json.loads(local.calls.last.request.content)["model"] == "qwen3.5:9b"
+ assert client.switched == "Groq limit reached — judging with local ollama qwen3.5:9b"
+ client.close()
+
+
+def test_the_local_model_is_always_asked_not_to_think(mock: respx.MockRouter) -> None:
+ _, _, _, local = routes(mock)
+ fallback().complete(MESSAGES, reasoning_effort="high")
+ assert json.loads(local.calls.last.request.content)["reasoning_effort"] == "none"
+
+
+def test_a_local_model_that_refuses_the_effort_is_still_downgraded(
+ mock: respx.MockRouter,
+) -> None:
+ """The 400 safety stays: a model that names ``reasoning_effort`` in a 400 is asked
+ again without it, and that retry is not a failure."""
+ _, _, _, local = routes(mock)
+ local.mock(
+ side_effect=[
+ httpx.Response(400, json={"error": {"message": "invalid reasoning_effort"}}),
+ _ok(OPINION),
+ ]
+ )
+ assert fallback().complete(MESSAGES).text == OPINION
+ assert "reasoning_effort" not in json.loads(local.calls.last.request.content)
+
+
+def test_installed_models_reads_the_tags_from_the_root_not_from_v1(mock: respx.MockRouter) -> None:
+ route = mock.get(TAGS_URL).mock(return_value=_tags("qwen3.5:9b", "gemma4:12b"))
+ names = judge.installed_models("http://localhost:11434/v1", timeout=2.0)
+ assert names == ["qwen3.5:9b", "gemma4:12b"]
+ assert route.call_count == 1
+
+
+@pytest.mark.parametrize(
+ "answer",
+ [httpx.ConnectError("refused"), httpx.Response(500), httpx.Response(200, text="not json")],
+)
+def test_installed_models_is_empty_on_any_error_and_never_raises(
+ mock: respx.MockRouter,
+ answer: httpx.Response | Exception,
+) -> None:
+ route = mock.get(TAGS_URL)
+ if isinstance(answer, Exception):
+ route.mock(side_effect=answer)
+ else:
+ route.mock(return_value=answer)
+ assert judge.installed_models("http://localhost:11434/v1", timeout=2.0) == []
+
+
+# --- the switch -------------------------------------------------------------------
+
+
+def test_a_429_switches_at_once_and_the_opinion_names_the_local_model(
+ mock: respx.MockRouter,
+) -> None:
+ """Amendment B 1, 2 and 4: one Groq call, no sleep, the batch answered locally."""
+ sleep = Sleeps()
+ seen: list[str] = []
+ groq, _, warm, local = routes(mock, groq=httpx.Response(429, headers={"Retry-After": "60"}))
+ client = fallback(sleep=sleep, on_switch=seen.append)
+ opinions = judge.Judge(client).review([judge.JudgeItem("c1", "x", "y", Label.NEI, "low")])
+
+ assert opinions["c1"].model == f"ollama {LOCAL}"
+ assert client.switched == LIMIT
+ assert seen == [LIMIT]
+ assert groq.call_count == 1 # no retry
+ assert sleep.calls == [] # and no Retry-After wait
+ assert warm.call_count == 1
+ assert local.call_count == 1
+ assert (client.provider, client.model) == ("ollama", LOCAL)
+
+
+def test_the_failed_request_is_sent_again_to_the_local_model_whole(mock: respx.MockRouter) -> None:
+ """Amendment B 2: the caller sees one completion, for the very request that failed."""
+ groq, _, _, local = routes(mock)
+ client = fallback()
+ done = client.complete(MESSAGES, json_schema=SCHEMA, max_tokens=4096, reasoning_effort="low")
+
+ assert done.text == OPINION
+ first = json.loads(groq.calls.last.request.content)
+ again = json.loads(local.calls.last.request.content)
+ assert again["messages"] == first["messages"] == MESSAGES
+ assert again["max_tokens"] == 4096
+ assert again["model"] == LOCAL
+ assert again["response_format"]["json_schema"]["schema"] == SCHEMA
+
+
+def test_after_the_switch_groq_is_never_asked_again(mock: respx.MockRouter) -> None:
+ groq, _, warm, local = routes(mock)
+ client = fallback()
+ client.complete(MESSAGES)
+ client.complete(MESSAGES)
+ client.complete(MESSAGES)
+
+ assert groq.call_count == 1
+ assert local.call_count == 3
+ assert warm.call_count == 1 # loaded once, kept warm after
+
+
+@pytest.mark.parametrize(
+ ("failure", "notice"),
+ [
+ (httpx.Response(413), LIMIT),
+ (
+ httpx.Response(401),
+ f"Groq rejected the key (HTTP 401) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.Response(403),
+ f"Groq rejected the key (HTTP 403) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.Response(500),
+ f"Groq did not answer (HTTP 500) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.Response(503),
+ f"Groq did not answer (HTTP 503) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.ReadTimeout("slow"),
+ f"Groq did not answer (timeout) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.ConnectError("down"),
+ f"Groq did not answer (connection failed) — judging with local ollama {LOCAL}",
+ ),
+ (
+ httpx.Response(
+ 200,
+ json={"choices": [{"message": {"content": ""}, "finish_reason": "length"}]},
+ ),
+ f"Groq did not answer (empty answer) — judging with local ollama {LOCAL}",
+ ),
+ ],
+)
+def test_the_notice_names_the_real_cause_of_the_switch(
+ mock: respx.MockRouter,
+ failure: httpx.Response | Exception,
+ notice: str,
+) -> None:
+ """Fix round 1 B: every failure switches, and the notice says which one it was --
+ never a body, never the key."""
+ sleep = Sleeps()
+ groq, _, _, local = routes(mock, groq=failure)
+ client = fallback(sleep=sleep)
+ client.complete(MESSAGES)
+
+ assert client.switched == notice
+ assert KEY.value not in notice
+ assert groq.call_count == 1
+ assert local.call_count == 1
+ assert sleep.calls == []
+
+
+def test_a_hanging_groq_switches_on_a_short_connect_timeout(mock: respx.MockRouter) -> None:
+ """Fix round 1 D3: with a fallback, the primary gets five seconds to connect."""
+ groq, _, _, local = routes(mock, groq=httpx.ConnectTimeout("no route"))
+ client = fallback()
+ client.complete(MESSAGES)
+
+ timeout = groq.calls.last.request.extensions["timeout"]
+ assert timeout["connect"] == judge.PRIMARY_CONNECT_TIMEOUT == 5.0
+ assert timeout["read"] == 60.0
+ assert client.switched == f"Groq did not answer (timeout) — judging with local ollama {LOCAL}"
+ assert local.call_count == 1
+
+
+def test_without_a_fallback_the_primary_keeps_its_own_timeout(mock: respx.MockRouter) -> None:
+ groq, _, _, _ = routes(mock, groq=httpx.Response(401))
+ client = fallback(setting="off")
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ assert groq.calls.last.request.extensions["timeout"]["connect"] == 60.0
+
+
+def test_a_listener_that_raises_does_not_lose_the_batch(mock: respx.MockRouter) -> None:
+ """Fix round 1 D1: the notice is best effort; the answer is not."""
+
+ def broken(text: str) -> None:
+ raise RuntimeError("listener bug")
+
+ _, _, _, local = routes(mock)
+ client = fallback(on_switch=broken)
+ assert client.complete(MESSAGES).text == OPINION
+ assert client.switched == LIMIT
+ assert local.call_count == 1
+
+
+def test_a_local_model_that_fails_is_not_asked_again_this_run(mock: respx.MockRouter) -> None:
+ """Fix round 1 D4: a dead Ollama costs one wait, not one per batch."""
+ groq, _, _, local = routes(mock, local=httpx.Response(500))
+ client = fallback(sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable) as first:
+ client.complete(MESSAGES)
+ asked = local.call_count
+ with pytest.raises(judge.JudgeUnavailable) as second:
+ client.complete(MESSAGES)
+
+ assert str(second.value) == str(first.value)
+ assert str(first.value).startswith(f"HTTP 429 from {GROQ_URL}; local ollama {LOCAL}: ")
+ assert local.call_count == asked # the second call asked nobody
+ assert groq.call_count == 1
+
+
+def test_a_local_timeout_is_not_retried(mock: respx.MockRouter) -> None:
+ """Fix round 1 D4: one 180 s wait at most, not three."""
+ _, _, _, local = routes(mock)
+ local.mock(side_effect=httpx.ReadTimeout("loading"))
+ client = fallback(sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable) as caught:
+ client.complete(MESSAGES)
+ assert local.call_count == 1
+ assert "request failed: ReadTimeout" in str(caught.value)
+
+
+def test_reset_forgets_a_dead_local_model(mock: respx.MockRouter) -> None:
+ groq, _, _, local = routes(mock, groq=[httpx.Response(429), httpx.Response(429)])
+ local.mock(side_effect=[httpx.Response(404), _ok(OPINION)])
+ client = fallback(sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ client.reset()
+ assert client.complete(MESSAGES).text == OPINION
+ assert groq.call_count == 2
+
+
+def test_one_url_for_tags_warm_up_and_chat(mock: respx.MockRouter) -> None:
+ """Fix round 1 D5: ``ollama_url`` reaches the local client too."""
+ other = "http://ollama.test:1234"
+ tags = mock.get(f"{other}/api/tags").mock(return_value=_tags(LOCAL))
+ warm = mock.post(f"{other}/api/generate").mock(return_value=httpx.Response(200, json={}))
+ chat = mock.post(f"{other}/v1/chat/completions").mock(return_value=_ok(OPINION))
+ mock.post(GROQ_URL).mock(return_value=httpx.Response(429))
+ primary = judge.JudgeClient(GROQ, KEY, sleep=Sleeps())
+ client = judge.FallbackClient(primary, ollama_url=f"{other}/v1")
+ client.complete(MESSAGES)
+ assert (tags.call_count, warm.call_count, chat.call_count) == (1, 1, 1)
+
+
+@pytest.mark.parametrize("setting", ["off", "Off", "OFF", " off "])
+def test_off_turns_the_fallback_off_in_any_case(mock: respx.MockRouter, setting: str) -> None:
+ """Fix round 1 D6: the keyword in any case; Ollama is never asked."""
+ _, tags, _, local = routes(mock)
+ client = fallback(setting=setting, sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable, match=r"judge\.fallback is off"):
+ client.complete(MESSAGES)
+ assert tags.call_count == local.call_count == 0
+
+
+def test_the_notice_fires_before_the_model_is_loaded_or_asked(mock: respx.MockRouter) -> None:
+ """Amendment B 4: the user reads the line while the model loads, not after."""
+ _, _, warm, local = routes(mock)
+ counts: list[tuple[int, int]] = []
+ client = fallback(on_switch=lambda text: counts.append((warm.call_count, local.call_count)))
+ client.complete(MESSAGES)
+
+ assert counts == [(0, 0)]
+
+
+def test_the_local_request_turns_thinking_off_and_the_warm_up_keeps_the_model_loaded(
+ mock: respx.MockRouter,
+) -> None:
+ """Amendment B 5 and 6, as Ollama 0.34 honours them: ``reasoning_effort: none`` on
+ the OpenAI route, and ``keep_alive`` on the native one (the OpenAI route ignores
+ it, measured 2026-09-28)."""
+ _, _, warm, local = routes(mock)
+ fallback().complete(MESSAGES, reasoning_effort="low")
+
+ sent = json.loads(local.calls.last.request.content)
+ assert sent["reasoning_effort"] == "none"
+ loaded = json.loads(warm.calls.last.request.content)
+ assert loaded == {"model": LOCAL, "keep_alive": judge.LOCAL_KEEP_ALIVE}
+ assert "authorization" not in local.calls.last.request.headers
+ assert "authorization" not in warm.calls.last.request.headers
+
+
+def test_the_local_client_waits_long_enough_for_a_cold_load() -> None:
+ client = judge.local_client(LOCAL)
+ assert client.timeout == judge.LOCAL_TIMEOUT == 180.0
+ assert (client.provider, client.model) == ("ollama", LOCAL)
+ client.close()
+
+
+def test_a_warm_up_that_fails_does_not_cost_the_answer(mock: respx.MockRouter) -> None:
+ _, _, warm, local = routes(mock)
+ warm.mock(side_effect=httpx.ConnectError("slow"))
+ done = fallback().complete(MESSAGES)
+ assert done.text == OPINION
+ assert local.call_count == 1
+
+
+def test_a_leading_think_block_never_reaches_the_json_reader(mock: respx.MockRouter) -> None:
+ """Amendment B 5: a qwen that thinks out loud, braces and all, still parses."""
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": "OpenAI went bankrupt.", "queries": ["OpenAI bankrupt"]}]}
+ )
+ routes(
+ mock, local=_ok(f"\nThe user wants {{json}}. Let me think.\n\n\n{answer}")
+ )
+ written = judge.Judge(fallback()).queries(["OpenAI battı."]) # noqa: RUF001 - Turkish, verbatim
+
+ assert written == {
+ 0: judge.JudgedClaim(queries=("OpenAI bankrupt",), english="OpenAI went bankrupt.")
+ }
+
+
+def test_an_orphan_closing_think_tag_is_stripped_too(mock: respx.MockRouter) -> None:
+ """Fix round 1 D2: some templates open the block in the prompt, so the answer
+ carries only its end -- with a brace in the reasoning to trip the JSON reader."""
+ answer = json.dumps({"items": [{"id": 0, "english": "Paris is big.", "queries": ["Paris"]}]})
+ routes(mock, local=_ok(f"The user wants {{items}} in JSON, fine.\n\n{answer}"))
+ written = judge.Judge(fallback()).queries(["Paris is big."])
+ assert written == {0: judge.JudgedClaim(queries=("Paris",), english="Paris is big.")}
+
+
+def test_two_threads_against_a_rate_limited_primary_switch_once(mock: respx.MockRouter) -> None:
+ """Amendment B 3: one switch, one notice, at most one Groq call per thread."""
+ barrier = threading.Barrier(2, timeout=5)
+
+ def limited(request: httpx.Request) -> httpx.Response:
+ barrier.wait() # both threads are inside the primary before either switches
+ return httpx.Response(429)
+
+ groq, _, warm, local = routes(mock)
+ groq.mock(side_effect=limited)
+ seen: list[str] = []
+ client = fallback(on_switch=seen.append)
+ done: list[str] = []
+ errors: list[BaseException] = []
+
+ def work() -> None:
+ try:
+ done.append(client.complete(MESSAGES).text)
+ except BaseException as exc: # pragma: no cover - reported below
+ errors.append(exc)
+
+ threads = [threading.Thread(target=work) for _ in range(2)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=10)
+
+ assert errors == []
+ assert done == [OPINION, OPINION]
+ assert seen == [LIMIT]
+ assert groq.call_count == 2
+ assert warm.call_count == 1
+ assert local.call_count == 2
+
+
+def test_calls_after_a_threaded_switch_go_straight_to_the_local_model(
+ mock: respx.MockRouter,
+) -> None:
+ groq, _, _, local = routes(mock)
+ client = fallback()
+ client.complete(MESSAGES)
+ threads = [threading.Thread(target=client.complete, args=(MESSAGES,)) for _ in range(4)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=10)
+ assert groq.call_count == 1
+ assert local.call_count == 5
+
+
+# --- no fallback --------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+ ("tags", "why"),
+ [
+ (httpx.ConnectError("refused"), "ollama is not reachable at http://localhost:11434"),
+ (_tags("gemma4:12b", "qwen3-embedding:8b"), "qwen3.5:9b is not installed"),
+ (_tags(), "qwen3.5:9b is not installed"),
+ ],
+)
+def test_without_a_local_model_the_original_error_says_why(
+ mock: respx.MockRouter, tags: httpx.Response | Exception, why: str
+) -> None:
+ """No fallback means today's behaviour: the retries, then ``JudgeUnavailable``."""
+ sleep = Sleeps()
+ groq, _, warm, local = routes(mock, tags=tags)
+ client = fallback(sleep=sleep)
+ with pytest.raises(judge.JudgeUnavailable) as caught:
+ client.complete(MESSAGES)
+
+ assert str(caught.value) == f"HTTP 429 from {GROQ_URL}; no local fallback: {why}"
+ assert groq.call_count == 3 # the normal retrying client
+ assert len(sleep.calls) == 2
+ assert client.switched is None
+ assert warm.call_count == local.call_count == 0
+
+
+def test_fallback_off_never_asks_ollama_and_keeps_the_retries(mock: respx.MockRouter) -> None:
+ sleep = Sleeps()
+ groq, tags, _, local = routes(mock)
+ client = fallback(setting="off", sleep=sleep)
+ with pytest.raises(judge.JudgeUnavailable) as caught:
+ client.complete(MESSAGES)
+
+ assert str(caught.value).endswith("; no local fallback: judge.fallback is off")
+ assert groq.call_count == 3
+ assert tags.call_count == 0
+ assert local.call_count == 0
+
+
+def test_a_users_own_override_is_honoured_as_written(mock: respx.MockRouter) -> None:
+ """``judge.fallback`` names a model: it is checked against the tags and used as
+ written. The name here is a stand-in, not a model proofpath knows about."""
+ _, tags, _, local = routes(mock, tags=_tags("my-judge:latest"))
+ client = fallback(setting="my-judge:latest")
+ client.complete(MESSAGES)
+
+ assert tags.call_count == 1
+ assert json.loads(local.calls.last.request.content)["model"] == "my-judge:latest"
+ assert client.switched == "Groq limit reached — judging with local ollama my-judge:latest"
+
+
+def test_an_untagged_override_matches_its_latest_tag(mock: respx.MockRouter) -> None:
+ _, _, _, local = routes(mock, tags=_tags("my-judge:latest"))
+ fallback(setting="my-judge").complete(MESSAGES)
+ assert json.loads(local.calls.last.request.content)["model"] == "my-judge"
+
+
+def test_the_model_is_never_pulled(mock: respx.MockRouter) -> None:
+ """Rule 5: a model that is not installed is reported, never fetched."""
+ pull = mock.post("http://localhost:11434/api/pull").mock(return_value=httpx.Response(200))
+ routes(mock, tags=_tags("gemma4:12b"))
+ with pytest.raises(judge.JudgeUnavailable, match=r"qwen3\.5:9b is not installed"):
+ fallback(sleep=Sleeps()).complete(MESSAGES)
+ assert pull.call_count == 0
+
+
+def test_a_local_model_that_fails_too_keeps_the_primarys_reason(mock: respx.MockRouter) -> None:
+ """The search's ``fallback_notice`` reads ``HTTP 429`` out of this detail."""
+ routes(mock, local=httpx.Response(404))
+ client = fallback()
+ with pytest.raises(judge.JudgeUnavailable) as caught:
+ client.complete(MESSAGES)
+ text = str(caught.value)
+ assert text.startswith(f"HTTP 429 from {GROQ_URL}; local ollama {LOCAL}: HTTP 404")
+
+
+def test_a_primary_that_is_already_ollama_gets_no_wrapper() -> None:
+ built = judge.build_client(judge.provider_defaults("ollama"), None)
+ assert isinstance(built, judge.JudgeClient)
+ built.close()
+
+
+def test_a_groq_primary_is_wrapped_with_the_configured_setting() -> None:
+ built = judge.build_client(GROQ, KEY)
+ assert isinstance(built, judge.FallbackClient)
+ assert (built.provider, built.model) == ("groq", GROQ.model)
+ built.close()
+
+
+def test_no_key_reaches_a_repr() -> None:
+ client = fallback()
+ assert KEY.value not in repr(client)
+ assert KEY.value not in str(client)
+ client.close()
+
+
+def test_no_key_reaches_an_error(mock: respx.MockRouter) -> None:
+ routes(mock, tags=httpx.ConnectError("refused"))
+ client = fallback(sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable) as caught:
+ client.complete(MESSAGES)
+ assert KEY.value not in str(caught.value)
+
+
+# --- cost, and a new run -------------------------------------------------------------
+
+
+def test_local_answers_are_counted_apart_from_api_calls(mock: respx.MockRouter) -> None:
+ """Amendment B 7: ``calls`` counts every answer, ``local_calls`` the local ones."""
+ groq, _, _, _ = routes(mock, groq=[_ok(OPINION, model=GROQ.model), httpx.Response(429)])
+ client = fallback()
+ client.complete(MESSAGES)
+ client.complete(MESSAGES)
+ client.complete(MESSAGES)
+
+ assert groq.call_count == 2
+ assert client.cost.calls == 3
+ assert client.cost.local_calls == 2
+ assert client.cost.model == LOCAL
+
+
+def test_reset_goes_back_to_the_primary_and_forgets_the_notice(mock: respx.MockRouter) -> None:
+ """Amendment B 9: the switch lasts one run; the next one tries the API first."""
+ groq, _, _, local = routes(mock, groq=[httpx.Response(429), _ok(OPINION, model=GROQ.model)])
+ seen: list[str] = []
+ client = fallback(on_switch=seen.append)
+ client.complete(MESSAGES)
+ assert client.switched == LIMIT
+
+ client.reset()
+ assert client.switched is None
+ assert (client.provider, client.model) == ("groq", GROQ.model)
+ client.complete(MESSAGES)
+
+ assert groq.call_count == 2
+ assert local.call_count == 1
+ assert seen == [LIMIT]
+ # What the local model already answered is not forgotten by the reset.
+ assert (client.cost.calls, client.cost.local_calls) == (2, 1)
+
+
+# --- the run: one note, the footer, the CLI ------------------------------------------
+
+QUERIES = json.dumps(
+ {"items": [{"id": 0, "english": CLAIM, "queries": ["ChatGPT shut down March 2025"]}]}
+)
+
+
+def test_a_run_that_switches_emits_one_notice_and_the_footer_carries_it(
+ mock: respx.MockRouter,
+) -> None:
+ routes(mock, local=_ok(QUERIES))
+ built = searching(StubSearcher())
+ built.judge = judge.Judge(judge.build_client(GROQ, KEY))
+ events: list[Event] = []
+ report = verify_mod.verify(CLAIM, built, on_event=events.append)
+
+ notices = [event for event in events if isinstance(event, Note) and event.notice]
+ assert [note.text for note in notices] == [LIMIT]
+ assert report.judge_notice == LIMIT
+ assert report.api_calls == 0 # Groq answered nothing; the one answer was local
+ assert report.judge_cost is not None and report.judge_cost.local_calls == 1
+ footer = render_footer(report)
+ assert footer.judge_notice == LIMIT
+ assert footer.local_calls == 1
+ markdown = render_markdown(report)
+ assert f"- judge fallback: {LIMIT}" in markdown
+ assert "- local calls: 1" in markdown
+
+
+def test_a_second_run_on_the_same_engine_starts_on_the_primary_again(
+ mock: respx.MockRouter,
+) -> None:
+ groq, _, _, local = routes(mock, groq=[httpx.Response(429), _ok(QUERIES, model=GROQ.model)])
+ local.mock(return_value=_ok(QUERIES))
+ built = searching(StubSearcher())
+ built.judge = judge.Judge(judge.build_client(GROQ, KEY))
+ first = verify_mod.verify(CLAIM, built)
+ events: list[Event] = []
+ second = verify_mod.verify(CLAIM, built, on_event=events.append)
+
+ assert first.judge_notice == LIMIT
+ assert second.judge_notice is None
+ assert not [event for event in events if isinstance(event, Note) and event.notice]
+ assert groq.call_count == 2
+
+
+runner = CliRunner()
+
+
+def test_the_cli_prints_the_notice_on_stderr(
+ mock: respx.MockRouter, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ routes(mock, local=_ok(QUERIES))
+ monkeypatch.setenv("GROQ_API_KEY", KEY.value)
+ monkeypatch.setenv("PROOFPATH_CONFIG_DIR", str(tmp_path / "conf"))
+ built = searching(StubSearcher())
+
+ def fake_default(config: Config, **kwargs: Any) -> verify_mod.Engine:
+ built.judge = kwargs["judge"]
+ built.escalate = kwargs.get("escalate", True)
+ return built
+
+ monkeypatch.setattr(verify_mod.Engine, "default", staticmethod(fake_default))
+ target = tmp_path / "claim.txt"
+ target.write_text(CLAIM, encoding="utf-8")
+ result = runner.invoke(app, ["check", str(target), "--judge", "--format", "json"])
+
+ assert LIMIT in result.stderr
+ # stdout is the one JSON document and nothing else; the notice rides inside it.
+ assert json.loads(result.stdout)["judge_notice"] == LIMIT
+ assert KEY.value not in result.output
+
+
+# --- the setting and the cost line ---------------------------------------------------
+
+
+def test_judge_fallback_is_a_setting_that_survives_a_provider_switch(tmp_path: Path) -> None:
+ from proofpath import commands
+ from proofpath.config import load_config, set_value
+
+ path = tmp_path / "config.toml"
+ assert load_config(path).judge.fallback == "qwen3.5:9b"
+ set_value("judge.fallback", "off", path)
+ commands.config_set("judge.provider", "gemini", path)
+ # A provider preset carries its model, URL and key variable, not this choice.
+ assert load_config(path).judge.fallback == "off"
+
+
+def test_the_cost_line_names_the_local_answers_only_when_there_are_some() -> None:
+ from proofpath.report import calls_text
+
+ assert calls_text(3) == "3 API calls"
+ assert calls_text(0, 2) == "0 API calls + 2 local"
+
+
+# --- fix round 2 ----------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+ "first",
+ [
+ httpx.Response(
+ 200, json={"choices": [{"message": {"content": ""}, "finish_reason": "length"}]}
+ ),
+ httpx.Response(200, text="not a completion"),
+ httpx.Response(400, json={"error": {"message": "prompt is too long"}}),
+ ],
+)
+def test_one_bad_local_answer_does_not_disable_the_model(
+ mock: respx.MockRouter, first: httpx.Response
+) -> None:
+ """Fix round 2, 1: only a dead model (no answer, 5xx, 404) is latched. A cut-off,
+ unreadable or refused answer costs that call; the next call is still asked."""
+ item = judge.JudgeItem("c1", "x", "y", Label.NEI, "low")
+ _, _, _, local = routes(mock)
+ local.mock(side_effect=[first, _ok(OPINION)])
+ j = judge.Judge(fallback(sleep=Sleeps()))
+
+ assert j.queries(["OpenAI battı."]) == {} # noqa: RUF001 - Turkish, verbatim
+ assert j.unavailable
+ opinions = j.review([item])
+
+ assert not j.unavailable
+ assert opinions["c1"].model == f"ollama {LOCAL}"
+ assert local.call_count == 2
+
+
+@pytest.mark.parametrize(
+ "dead", [httpx.Response(404), httpx.Response(500), httpx.ConnectError("gone")]
+)
+def test_a_dead_local_model_is_latched(mock: respx.MockRouter, dead: Any) -> None:
+ _, _, _, local = routes(mock)
+ # Every attempt dead: the local client's own retries are not what is tested here.
+ if isinstance(dead, Exception):
+ local.mock(side_effect=dead)
+ else:
+ local.mock(return_value=dead)
+ client = fallback(sleep=Sleeps())
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ asked = local.call_count
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ assert local.call_count == asked
+
+
+def test_a_second_thread_waits_for_the_warm_up_before_its_request(
+ mock: respx.MockRouter,
+) -> None:
+ """Fix round 2, 2: a thread that did not switch must not start its 180 s read
+ timer while the model is still loading; it waits for the warm-up, however long."""
+ warming, release = threading.Event(), threading.Event()
+ warm_done: list[float] = []
+ asked: list[float] = []
+
+ def slow_warm(request: httpx.Request) -> httpx.Response:
+ warming.set()
+ assert release.wait(timeout=10)
+ warm_done.append(time.monotonic())
+ return httpx.Response(200, json={})
+
+ def answer(request: httpx.Request) -> httpx.Response:
+ asked.append(time.monotonic())
+ return _ok(OPINION)
+
+ _, _, warm, local = routes(mock)
+ warm.mock(side_effect=slow_warm)
+ local.mock(side_effect=answer)
+ client = fallback()
+ results: list[str] = []
+ first = threading.Thread(target=lambda: results.append(client.complete(MESSAGES).text))
+ first.start()
+ assert warming.wait(timeout=5)
+ second = threading.Thread(target=lambda: results.append(client.complete(MESSAGES).text))
+ second.start()
+ time.sleep(0.3)
+ assert local.call_count == 0 # neither thread asked while the model was loading
+ release.set()
+ first.join(timeout=10)
+ second.join(timeout=10)
+
+ assert results == [OPINION, OPINION]
+ assert warm.call_count == 1
+ assert all(when >= warm_done[0] for when in asked)
+
+
+def test_the_warm_up_wait_is_bounded_by_the_warm_ups_own_timeout() -> None:
+ assert judge.WARM_WAIT_S > judge.LOCAL_TIMEOUT
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ '{"items": [{"id": 0, "english": "He wrote {x}", "queries": ["q"]}]}',
+ '[{"note": " [not a prefix]"}]',
+ ],
+)
+def test_json_that_mentions_the_closing_tag_is_left_intact(text: str) -> None:
+ """Fix round 2, 3: an orphan ```` is a prefix only when the answer does not
+ already start as JSON and JSON follows it."""
+ assert judge._strip_think(text) == text
+
+
+def test_an_orphan_closing_tag_before_prose_is_not_stripped() -> None:
+ assert judge._strip_think("I think so.") == "I think so."
+
+
+def test_an_orphan_closing_tag_before_json_is_stripped() -> None:
+ assert judge._strip_think('reasoning {a}\n\n{"items": []}') == '{"items": []}'
+ assert judge._strip_think('r {"items": []}') == '{"items": []}'
diff --git a/tests/test_polite.py b/tests/test_polite.py
index 9ec9909..d862ca5 100644
--- a/tests/test_polite.py
+++ b/tests/test_polite.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
import threading
import httpx
@@ -266,3 +267,25 @@ def call() -> None:
for thread in threads:
thread.join(timeout=5.0)
assert sorted(waits) == [0.0, pytest.approx(pl.MIN_INTERVAL[host], abs=0.05)]
+
+
+@respx.mock
+def test_post_sends_a_json_body_and_keeps_headers_out_of_the_url() -> None:
+ route = respx.post("https://api.test/search").mock(
+ return_value=httpx.Response(200, json={"results": []})
+ )
+ client = pl.PoliteClient(throttle=pl.HostThrottle())
+ response = client.post(
+ "https://api.test/search",
+ json_body={"query": "a claim", "max_results": 3},
+ headers={"Authorization": "Bearer secret"},
+ )
+ assert response.status_code == 200
+ sent = route.calls.last.request
+ assert json.loads(sent.content) == {"query": "a claim", "max_results": 3}
+ assert sent.headers["Authorization"] == "Bearer secret"
+ assert "secret" not in str(sent.url)
+
+
+def test_tavily_has_a_polite_interval() -> None:
+ assert pl.MIN_INTERVAL["api.tavily.com"] == 0.5
diff --git a/tests/test_report.py b/tests/test_report.py
index f891682..f4fc0df 100644
--- a/tests/test_report.py
+++ b/tests/test_report.py
@@ -15,6 +15,7 @@
from proofpath.report import (
BROWSER_SKIPPED_REASON,
JUDGE_DETAIL_PREFIX,
+ LANGUAGE_UNSUPPORTED,
LEVELS,
NO_REASON_RECORDED,
SNIPPET_LIMIT,
@@ -25,6 +26,7 @@
Finding,
Kind,
Report,
+ SearchSummary,
SourceStatus,
Stage,
coverage_reasons,
@@ -36,6 +38,8 @@
render_diagnostics,
render_footer,
render_markdown,
+ search_experimental,
+ search_lines,
summary_silence,
summary_unavailable,
)
@@ -76,7 +80,9 @@ def claim_at(line: int, page: int | None = None) -> Claim:
def finding(kind: Kind, line: int, page: int | None = None, column: int = 1) -> Finding:
"""A minimally valid Finding of ``kind``, positioned where the test needs it."""
- needs_passage = kind in {Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH}
+ # EVIDENCE_FOUND asserts support too (rule 1), so it needs a passage just like the
+ # other two asserting kinds -- see ``_ASSERTING`` in ``proofpath.report``.
+ needs_passage = kind in {Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND}
return Finding(
kind=kind,
level=LEVELS[kind],
@@ -164,7 +170,7 @@ def test_kind_values_are_the_diagnostic_codes() -> None:
# --- Finding invariants ----------------------------------------------------------
-@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH])
+@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND])
def test_an_asserting_finding_without_a_verdict_raises(kind: Kind) -> None:
with pytest.raises(ValueError, match="passage"):
Finding(
@@ -182,7 +188,7 @@ def test_an_asserting_finding_without_a_verdict_raises(kind: Kind) -> None:
)
-@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH])
+@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND])
def test_an_asserting_finding_without_a_passage_raises(kind: Kind) -> None:
with pytest.raises(ValueError, match="passage"):
Finding(
@@ -200,7 +206,7 @@ def test_an_asserting_finding_without_a_passage_raises(kind: Kind) -> None:
)
-@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH])
+@pytest.mark.parametrize("kind", [Kind.NOT_SUPPORTED, Kind.NUMERIC_MISMATCH, Kind.EVIDENCE_FOUND])
def test_an_asserting_finding_with_a_passage_is_valid(kind: Kind) -> None:
assert finding(kind, line=260, page=9).verdict is REFUTED
@@ -1593,3 +1599,74 @@ def test_the_footer_carries_the_reasons_so_the_terminal_can_print_them() -> None
(CREDENTIALS_MISSING, 1),
)
assert render_footer(covered(4, 4, 0, 0)).reasons == ()
+
+
+# --- evidence search (OPEN-ITEMS 17.1a) -------------------------------------------
+
+
+def test_the_two_search_kinds_carry_their_section_15_words() -> None:
+ assert STATE_WORDS[Kind.EVIDENCE_FOUND] == "SUPPORTED (found by proofpath)"
+ assert LEVELS[Kind.EVIDENCE_FOUND] == "note"
+ assert STATE_WORDS[Kind.NO_EVIDENCE] == "NO EVIDENCE FOUND (searched)"
+ assert LEVELS[Kind.NO_EVIDENCE] == "warning"
+
+
+def test_evidence_found_needs_a_passage() -> None:
+ with pytest.raises(ValueError, match="passage"):
+ Finding(
+ kind=Kind.EVIDENCE_FOUND,
+ level="note",
+ locator=Locator(line=1),
+ title="t",
+ state=STATE_WORDS[Kind.EVIDENCE_FOUND],
+ reference=None,
+ claim=None,
+ verdict=None,
+ source_id=None,
+ fetch_step=None,
+ tier=None,
+ )
+
+
+def test_a_reference_is_the_authors_unless_the_search_found_it() -> None:
+ assert Reference(1, "x", Locator(line=1)).origin == "author"
+ assert Reference(1, "x", Locator(line=1), origin="search").origin == "search"
+
+
+def test_language_unsupported_is_worded_as_the_unverified_family() -> None:
+ assert LANGUAGE_UNSUPPORTED == "UNVERIFIED (language not supported)"
+
+
+SUMMARY = SearchSummary(
+ by="tavily",
+ queries_by="sentence",
+ sentences=7,
+ eligible=4,
+ searched=3,
+ pages_found=9,
+ pages_read=6,
+ notices=("LLM limit reached — searched with the sentence text",),
+)
+
+
+def test_search_lines_state_what_was_searched_and_what_was_not() -> None:
+ lines = search_lines(dataclasses.replace(report_with(), search=SUMMARY))
+ assert lines[0] == search_experimental()
+ assert "evidence search by tavily, queries by sentence" in lines
+ assert "claims searched 3 of 4 check-worthy (7 sentences)" in lines
+ assert "claims not searched 4" in lines
+ assert "pages found / read 9 / 6" in lines
+ assert lines[-1].startswith("LLM limit reached")
+
+
+def test_a_run_that_did_not_search_has_no_search_lines() -> None:
+ assert search_lines(report_with()) == ()
+ assert render_footer(report_with()).search == ()
+
+
+def test_the_footer_and_the_markdown_carry_the_search_block() -> None:
+ report = dataclasses.replace(report_with(), search=SUMMARY)
+ assert render_footer(report).search == search_lines(report)
+ markdown = render_markdown(report)
+ assert "## Evidence search" in markdown
+ assert "- claims not searched 4" in markdown
diff --git a/tests/test_sarif.py b/tests/test_sarif.py
index 0d8e8ce..84d07fb 100644
--- a/tests/test_sarif.py
+++ b/tests/test_sarif.py
@@ -30,6 +30,7 @@
Finding,
Kind,
Report,
+ SearchSummary,
judge_detail,
)
from proofpath.sarif import (
@@ -406,3 +407,19 @@ def test_a_judged_finding_still_validates_and_keeps_its_local_verdict(
result = results_by_kind(payload)[Kind.NOT_SUPPORTED.value]
assert result["level"] == "error"
assert PASSAGE.text in result["message"]["text"]
+
+
+# --- evidence search (OPEN-ITEMS 17.1a) -------------------------------------------
+
+
+def test_every_kind_has_a_rule_description() -> None:
+ assert set(RULE_DESCRIPTIONS) == set(Kind)
+
+
+def test_a_search_run_states_its_search_in_the_run_properties() -> None:
+ report = dataclasses.replace(
+ report_with(), search=SearchSummary("tavily", "sentence", 2, 2, 2, 3, 1)
+ )
+ props = to_sarif(report, artifact="draft.md")["runs"][0]["properties"]
+ assert props["evidenceSearch"]["pagesRead"] == 1
+ assert props["evidenceSearch"]["experimental"] is True
diff --git a/tests/test_search.py b/tests/test_search.py
new file mode 100644
index 0000000..d443d6b
--- /dev/null
+++ b/tests/test_search.py
@@ -0,0 +1,144 @@
+"""Evidence search providers, offline: respx over fixtures, no socket (OPEN-ITEMS 17.1a)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+import respx
+
+from proofpath import polite as pl
+from proofpath.config import SearchConfig
+from proofpath.polite import ProviderError
+from proofpath.search import SearchHit, canonical, keep_hits
+from proofpath.search.providers import (
+ TAVILY_QUERY_LIMIT,
+ TAVILY_URL,
+ SearxngSearcher,
+ TavilySearcher,
+ build_searcher,
+)
+from proofpath.secrets import ApiKey
+
+FIX = Path(__file__).parent / "fixtures" / "search"
+SEARXNG = "http://localhost:8888"
+
+
+def fixture(name: str) -> Any:
+ return json.loads((FIX / name).read_text(encoding="utf-8"))
+
+
+def client() -> pl.PoliteClient:
+ return pl.PoliteClient(throttle=pl.HostThrottle(), retries=0)
+
+
+@respx.mock
+def test_tavily_sends_the_key_as_a_bearer_header_and_reads_addresses_only() -> None:
+ route = respx.post(TAVILY_URL).mock(
+ return_value=httpx.Response(200, json=fixture("tavily.json"))
+ )
+ searcher = TavilySearcher(client(), ApiKey("tvly-secret", source="test"))
+
+ hits = searcher.search("ChatGPT shut down 2025", 5)
+
+ sent = route.calls.last.request
+ assert sent.headers["Authorization"] == "Bearer tvly-secret"
+ body = json.loads(sent.content)
+ assert body["query"] == "ChatGPT shut down 2025"
+ assert body["max_results"] == 5
+ assert body["include_raw_content"] is False
+ # The entry with no address is dropped; nothing of ``content`` survives (rule 1).
+ assert [hit.url for hit in hits] == [
+ "https://news.test/openai-statement?utm_source=x",
+ "https://news.test/openai-statement",
+ "https://x.com/someone/status/1",
+ "https://wiki.test/ChatGPT",
+ ]
+ assert [hit.rank for hit in hits] == [1, 2, 3, 4]
+ assert "SNIPPET" not in repr(hits)
+
+
+@respx.mock
+def test_tavily_cuts_a_long_query_to_its_limit() -> None:
+ route = respx.post(TAVILY_URL).mock(return_value=httpx.Response(200, json={"results": []}))
+ TavilySearcher(client(), ApiKey("k", source="test")).search("word " * 200, 3)
+ assert len(json.loads(route.calls.last.request.content)["query"]) <= TAVILY_QUERY_LIMIT
+
+
+@respx.mock
+def test_a_provider_that_refuses_is_a_provider_error_without_the_body() -> None:
+ respx.post(TAVILY_URL).mock(return_value=httpx.Response(401, json={"detail": "tvly-secret"}))
+ with pytest.raises(ProviderError) as caught:
+ TavilySearcher(client(), ApiKey("tvly-secret", source="test")).search("q", 3)
+ assert "tvly-secret" not in str(caught.value)
+
+
+@respx.mock
+def test_a_non_json_answer_is_a_provider_error() -> None:
+ respx.get(f"{SEARXNG}/search").mock(return_value=httpx.Response(200, text=""))
+ with pytest.raises(ProviderError, match="non-JSON"):
+ SearxngSearcher(client(), SEARXNG).search("q", 3)
+
+
+@respx.mock
+def test_searxng_asks_for_json_and_keeps_only_web_addresses() -> None:
+ route = respx.get(f"{SEARXNG}/search").mock(
+ return_value=httpx.Response(200, json=fixture("searxng.json"))
+ )
+ hits = SearxngSearcher(client(), SEARXNG + "/").search("ChatGPT shut down 2025", 3)
+ params = route.calls.last.request.url.params
+ assert params["format"] == "json"
+ assert params["q"] == "ChatGPT shut down 2025"
+ assert "mailto" not in params
+ assert hits == [SearchHit("https://wiki.test/ChatGPT", "ChatGPT", 1)]
+
+
+def test_canonical_drops_the_fragment_and_tracking_parameters() -> None:
+ assert canonical("HTTPS://News.Test/a?utm_source=x&id=2#top") == "https://news.test/a?id=2"
+ assert canonical("https://news.test") == "https://news.test/"
+
+
+def test_keep_hits_dedupes_and_drops_posts_and_the_documents_own_host() -> None:
+ hits = [
+ SearchHit("https://news.test/a?utm_source=x", "", 1),
+ SearchHit("https://news.test/a", "", 2),
+ SearchHit("https://x.com/u/status/1", "", 3),
+ SearchHit("https://www.blog.test/post", "", 4),
+ SearchHit("https://wiki.test/A", "", 5),
+ SearchHit("https://other.test/B", "", 6),
+ ]
+ kept = keep_hits(hits, own_host="blog.test", limit=2)
+ assert [hit.url for hit in kept] == ["https://news.test/a?utm_source=x", "https://wiki.test/A"]
+
+
+def test_build_searcher_is_off_by_default() -> None:
+ setup = build_searcher(SearchConfig(), client())
+ assert setup.searcher is None
+ assert setup.problem == ""
+
+
+def test_build_searcher_names_the_missing_key_and_never_a_value() -> None:
+ setup = build_searcher(SearchConfig(provider="tavily"), client(), resolve_key=lambda _: None)
+ assert setup.searcher is None
+ assert setup.problem == "set TAVILY_API_KEY in .env"
+
+
+def test_build_searcher_builds_tavily_with_a_key() -> None:
+ setup = build_searcher(
+ SearchConfig(provider="tavily"),
+ client(),
+ resolve_key=lambda _: ApiKey("tvly-SECRET", source="t"),
+ )
+ assert setup.searcher is not None and setup.searcher.name == "tavily"
+ assert "tvly-SECRET" not in repr(setup)
+
+
+def test_build_searcher_needs_a_searxng_address() -> None:
+ assert build_searcher(SearchConfig(provider="searxng"), client()).problem == (
+ "set search.base_url to your SearXNG instance"
+ )
+ built = build_searcher(SearchConfig(provider="searxng", base_url=SEARXNG), client())
+ assert built.searcher is not None and built.searcher.name == "searxng"
diff --git a/tests/test_search_queries.py b/tests/test_search_queries.py
new file mode 100644
index 0000000..af2bd17
--- /dev/null
+++ b/tests/test_search_queries.py
@@ -0,0 +1,215 @@
+"""Query building for the evidence search: the sentence, the judge, and the fallback."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import pytest
+
+from proofpath.judge import Completion, Judge, JudgeCost, JudgeUnavailable
+from proofpath.search.language import looks_english
+from proofpath.search.queries import (
+ LLM_LIMIT,
+ LLM_UNAVAILABLE,
+ QUERY_LIMIT,
+ SENTENCE_BY,
+ plan_queries,
+ sentence_query,
+)
+
+
+class ScriptedClient:
+ """``JudgeClient`` without a socket: one scripted answer or exception."""
+
+ provider = "fake"
+ model = "q-1"
+
+ def __init__(self, answer: str | Exception) -> None:
+ self.answer = answer
+ self.cost = JudgeCost(model=self.model)
+ self.prompts: list[str] = []
+
+ def complete(self, messages: list[dict[str, str]], **_: Any) -> Completion:
+ self.prompts.append(messages[-1]["content"])
+ if isinstance(self.answer, Exception):
+ raise self.answer
+ return Completion(text=self.answer, prompt_tokens=1, completion_tokens=1, model=self.model)
+
+ def close(self) -> None:
+ pass
+
+
+def judge(answer: str | Exception) -> Judge:
+ return Judge(ScriptedClient(answer)) # type: ignore[arg-type]
+
+
+def test_sentence_query_strips_links_handles_hashtag_signs_and_emoji() -> None:
+ text = "🚨 @newsbot says #ChatGPT was shut down https://t.co/xyz today!"
+ assert sentence_query(text) == "says ChatGPT was shut down today!"
+
+
+def test_sentence_query_cuts_at_a_word_boundary() -> None:
+ query = sentence_query("word " * 200)
+ assert len(query) <= QUERY_LIMIT
+ assert not query.endswith(" ")
+
+
+def test_without_a_judge_the_sentence_is_the_query() -> None:
+ plan = plan_queries(["ChatGPT was shut down in 2025."], None)
+ assert plan.queries == (("ChatGPT was shut down in 2025.",),)
+ assert plan.by == SENTENCE_BY
+ assert plan.notice is None
+
+
+def test_the_judge_writes_decontextualised_queries() -> None:
+ answer = json.dumps(
+ {
+ "items": [
+ {
+ "id": 0,
+ "english": "He shut it down in 2025.",
+ "queries": ["OpenAI ChatGPT shutdown 2025"],
+ }
+ ]
+ }
+ )
+ plan = plan_queries(["He shut it down in 2025."], judge(answer))
+ assert plan.queries == (("OpenAI ChatGPT shutdown 2025",),)
+ assert plan.by == "fake q-1"
+ assert plan.notice is None
+
+
+def test_a_rate_limited_judge_falls_back_to_the_sentence_and_says_so() -> None:
+ plan = plan_queries(
+ ["ChatGPT was shut down in 2025."],
+ judge(JudgeUnavailable("HTTP 429 from https://api.test/v1/chat/completions")),
+ )
+ assert plan.queries == (("ChatGPT was shut down in 2025.",),)
+ assert plan.by == SENTENCE_BY
+ assert plan.notice is not None and plan.notice.startswith(LLM_LIMIT)
+
+
+def test_a_judge_that_is_down_falls_back_with_its_own_words() -> None:
+ plan = plan_queries(["ChatGPT was shut down in 2025."], judge(JudgeUnavailable("HTTP 503")))
+ assert plan.notice is not None and plan.notice.startswith(LLM_UNAVAILABLE)
+
+
+def test_a_claim_the_judge_skipped_keeps_its_sentence() -> None:
+ answer = json.dumps(
+ {
+ "items": [
+ {"id": 1, "english": "Paris is the capital.", "queries": ["Paris capital France"]}
+ ]
+ }
+ )
+ plan = plan_queries(["NASA landed in 1969.", "Paris is the capital."], judge(answer))
+ assert plan.queries == (("NASA landed in 1969.",), ("Paris capital France",))
+
+
+def test_judge_queries_never_raises_on_an_unreadable_answer() -> None:
+ built = judge("not json at all")
+ assert built.queries(["a claim"]) == {}
+ assert built.unavailable is False
+
+
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("OpenAI shut down ChatGPT in 2025.", True),
+ ("NASA confirms water on Mars", True),
+ ("OpenAI battı.", False), # noqa: RUF001 - real Turkish letters
+ ("OpenAI batti ve kapandi.", False), # ASCII Turkish, "ve"
+ ("El gobierno cerró la empresa.", False),
+ ("Die Firma ist pleite.", False), # "ist"
+ ("André said the deal is done.", True),
+ ("Beyoncé released a new album.", True),
+ ("Müller won the race in Berlin.", True),
+ ("Путин подписал закон.", False), # Cyrillic
+ ("İstanbul büyük bir şehir.", False), # Turkish-only letters, and "bir" too
+ ],
+)
+def test_looks_english(text: str, expected: bool) -> None:
+ assert looks_english(text) is expected
+
+
+def test_an_english_claim_is_checked_as_written() -> None:
+ plan = plan_queries(["OpenAI shut down ChatGPT in 2025."], None)
+ assert plan.hypotheses == ("OpenAI shut down ChatGPT in 2025.",)
+
+
+def test_without_a_judge_a_turkish_claim_has_nothing_to_check() -> None:
+ plan = plan_queries(["OpenAI battı."], None) # noqa: RUF001 - real Turkish letters
+ assert plan.hypotheses == (None,)
+
+
+def test_the_judge_translates_a_turkish_claim_for_the_check_and_the_queries() -> None:
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": "OpenAI went bankrupt.", "queries": ["OpenAI bankruptcy"]}]}
+ )
+ plan = plan_queries(["OpenAI battı."], judge(answer)) # noqa: RUF001 - real Turkish letters
+ assert plan.hypotheses == ("OpenAI went bankrupt.",)
+ assert plan.queries == (("OpenAI bankruptcy",),)
+
+
+def test_a_rate_limited_judge_leaves_a_turkish_claim_untranslated() -> None:
+ plan = plan_queries(
+ ["OpenAI battı."], # noqa: RUF001 - real Turkish letters
+ judge(JudgeUnavailable("HTTP 429 from https://api.test")),
+ )
+ assert plan.hypotheses == (None,)
+ assert plan.notice is not None and plan.notice.startswith(LLM_LIMIT)
+
+
+def test_a_judge_that_translates_but_writes_no_query_keeps_the_sentence_query() -> None:
+ answer = json.dumps({"items": [{"id": 0, "english": "OpenAI went bankrupt.", "queries": []}]})
+ plan = plan_queries(["OpenAI battı."], judge(answer)) # noqa: RUF001 - real Turkish letters
+ assert plan.hypotheses == ("OpenAI went bankrupt.",)
+ assert plan.queries == ((sentence_query("OpenAI battı."),),) # noqa: RUF001 - real Turkish letters
+
+
+@pytest.mark.parametrize(
+ "english",
+ [
+ "OpenAI battı.", # noqa: RUF001 - the claim echoed back, real Turkish letters
+ "OpenAI iflas etti ve kapandı.", # noqa: RUF001 - still Turkish, just other words
+ ],
+)
+def test_an_english_that_is_not_english_is_no_translation(english: str) -> None:
+ """Fix round 1 A: a judge that echoes the claim (a local qwen does) must not have
+ Turkish checked by the English-only models. Its queries are still kept."""
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": english, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ plan = plan_queries(["OpenAI battı."], judge(answer)) # noqa: RUF001 - real Turkish letters
+ assert plan.hypotheses == (None,)
+ assert plan.queries == (("OpenAI bankruptcy",),)
+
+
+def test_an_unchanged_echo_of_a_non_english_claim_is_no_translation() -> None:
+ """Any language, not only Turkish: a French claim echoed back is still French."""
+ claim = "Le chat est noir."
+ assert not looks_english(claim)
+ answer = json.dumps({"items": [{"id": 0, "english": claim, "queries": ["black cat"]}]})
+ assert plan_queries([claim], judge(answer)).hypotheses == (None,)
+
+
+def test_the_queries_prompt_carries_worked_examples_that_cannot_be_answered() -> None:
+ """Fix round 1 (controller, 2026-09-28): a lone non-English claim was echoed by
+ qwen3.5:9b, so the prompt shows one translation that changes the verb and one
+ English claim kept as is. The examples use generic subjects and ids that are not
+ integers, so an echoed example is dropped by the reader, never taken for a claim."""
+ from proofpath.judge import _queries_from, _split_prompt, load_prompt
+
+ system, user = _split_prompt(load_prompt("queries").substitute(items="- id: 0\n claim: x"))
+ assert "Examples" in user
+ assert "The company went bankrupt." in user
+ assert "The Eiffel Tower is in Paris." in user
+ assert "OpenAI" not in system + user
+ # Only the real item uses the ``- id:`` shape the fakes and the batch counter read.
+ assert user.count("id: ") == 1
+ example = (
+ '{"items": [{"id": "E1", "english": "The company went bankrupt.",'
+ ' "queries": ["company went bankrupt"]}]}'
+ )
+ assert _queries_from(example, 1) == {}
diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py
index f803d70..6e50c53 100644
--- a/tests/test_tui_app.py
+++ b/tests/test_tui_app.py
@@ -46,6 +46,7 @@
CommandBlock,
CoverageFooter,
FindingLine,
+ JudgeNotice,
KvLine,
NoteLine,
PermissionPrompt,
@@ -1302,6 +1303,35 @@ def test_the_footer_hint_for_skipped_sources_names_the_setting() -> None:
assert ui.BROWSER_SETTING == "permissions.install_browser ask"
+def test_the_footer_hints_show_the_first_three_search_lines() -> None:
+ """The docked caveat row carries only the experimental banner and the searched
+ counts (the first three of ``search_lines``); the full evidence-search block is
+ in the written report, not the fixed-height docked footer."""
+ from proofpath.report import SearchSummary, render_footer, search_experimental
+ from proofpath.tui.widgets.footer import _hints
+
+ plain = a_report()
+ summary = SearchSummary(
+ by="tavily",
+ queries_by="sentence",
+ sentences=5,
+ eligible=3,
+ searched=1,
+ pages_found=2,
+ pages_read=1,
+ )
+ item = render_footer(replace(plain, search=summary))
+ hints = _hints(item)
+ # ``search_lines`` has five lines here (plus the banner); only the first three
+ # reach the docked footer, so this is an exact match, not a subset check.
+ assert hints == [
+ search_experimental(),
+ "evidence search by tavily, queries by sentence",
+ "claims searched 1 of 3 check-worthy (5 sentences)",
+ ]
+ assert not any("pages found" in line or "not searched" in line for line in hints)
+
+
def test_the_fetch_block_skipped_line_names_the_setting() -> None:
"""The ``/fetch`` block's line says which setting lets the browser run next time."""
from proofpath.report import BROWSER_SKIPPED_REASON
@@ -3019,21 +3049,40 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside
assert [row.key for row in panel.rows] == [
"permissions.install_browser",
"permissions.network",
+ "permissions.web_search",
"fetch.respect_robots",
"judge.provider",
"judge.model",
"judge.base_url",
"judge.api_key_env",
+ "judge.fallback",
+ "search.provider",
+ "search.base_url",
+ "search.api_key_env",
"contact.email",
]
- assert [row.kind for row in panel.rows] == ["choice"] * 4 + ["text"] * 4
+ assert [row.kind for row in panel.rows] == [
+ "choice",
+ "choice",
+ "choice",
+ "choice",
+ "choice",
+ "text",
+ "text",
+ "text",
+ "text",
+ "choice",
+ "text",
+ "text",
+ "text",
+ ]
assert panel.selected == 0
assert _panel_row(panel, "install_browser").startswith("> install_browser")
- await pilot.press(*["down"] * 8)
- assert panel.selected == 7
+ await pilot.press(*["down"] * 12)
+ assert panel.selected == 12
assert _panel_row(panel, "email").startswith("> email")
assert _panel_row(panel, "install_browser").startswith(" install_browser")
- await pilot.press(*["up"] * 9)
+ await pilot.press(*["up"] * 13)
assert panel.selected == 0
# The file was only read: moving through the rows writes nothing.
assert not config_path().exists()
@@ -3098,7 +3147,7 @@ async def test_a_bool_row_round_trips_through_the_file() -> None:
app, _ = build_app()
async with app.run_test(size=SIZE) as pilot:
panel = await open_panel(pilot)
- await pilot.press("down", "down", "right")
+ await pilot.press("down", "down", "down", "right")
await written(pilot, "fetch.respect_robots", "false")
assert load_config().fetch.respect_robots is False
assert panel.value("fetch.respect_robots") == "false"
@@ -3113,8 +3162,8 @@ async def test_a_provider_switch_writes_four_and_moves_the_key_line() -> None:
app, _ = build_app()
async with app.run_test(size=SIZE) as pilot:
panel = await open_panel(pilot)
- assert panel.rows[3].values == ("gemini", "groq", "ollama")
- await pilot.press("down", "down", "down", "left") # groq -> gemini
+ assert panel.rows[4].values == ("gemini", "groq", "ollama")
+ await pilot.press("down", "down", "down", "down", "left") # groq -> gemini
await written(pilot, "judge.api_key_env", "GEMINI_API_KEY")
gemini = provider_defaults("gemini")
assert load_config().judge == gemini
@@ -3142,7 +3191,7 @@ async def test_a_provider_outside_the_known_choices_is_drawn_honestly() -> None:
panel = await open_panel(pilot)
row = _panel_row(panel, "provider")
assert row == " provider gemini groq ollama [openrouter]"
- await pilot.press("down", "down", "down") # install_browser -> ... -> provider
+ await pilot.press("down", "down", "down", "down") # install_browser -> ... -> provider
await pilot.press("right") # cycles from the first choice, not from "nowhere"
await written(pilot, "judge.provider", "gemini")
@@ -3152,7 +3201,7 @@ async def test_a_provider_without_a_key_variable_says_none_is_needed() -> None:
app, _ = build_app()
async with app.run_test(size=SIZE) as pilot:
panel = await open_panel(pilot)
- await pilot.press("down", "down", "down", "right") # groq -> ollama
+ await pilot.press("down", "down", "down", "down", "right") # groq -> ollama
await written(pilot, "judge.provider", "ollama")
text = _panel_text(panel)
assert " key not needed" in text
@@ -3165,7 +3214,7 @@ async def test_a_text_row_is_edited_in_place() -> None:
default = JudgeConfig().model
async with app.run_test(size=SIZE) as pilot:
panel = await open_panel(pilot)
- await pilot.press("down", "down", "down", "down", "enter")
+ await pilot.press("down", "down", "down", "down", "down", "enter")
await pilot.pause()
assert panel.editing
edit = panel.query_one(Input)
@@ -3199,7 +3248,7 @@ async def test_an_empty_text_row_shows_unset_and_its_note() -> None:
assert _panel_row(panel, "email") == (
" email (unset) , optional, for the Crossref / OpenAlex polite pools"
)
- await pilot.press(*["down"] * 7, "enter")
+ await pilot.press(*["down"] * 12, "enter")
await pilot.pause()
await until(pilot, lambda: app.focused is panel.query_one(Input), "the edit")
assert panel.query_one(Input).value == ""
@@ -3220,7 +3269,7 @@ async def test_backspace_resets_a_row_to_its_default() -> None:
await written(pilot, "permissions.install_browser", "ask")
assert load_config().permissions.install_browser == "ask"
assert panel.value("permissions.install_browser") == "ask"
- await pilot.press("down", "down", "down", "left")
+ await pilot.press("down", "down", "down", "down", "left")
await written(pilot, "judge.provider", "gemini")
before = _notes(app).count("judge.")
await pilot.press("backspace")
@@ -3289,7 +3338,8 @@ async def test_escape_collapses_the_panel_to_one_line_and_focuses_the_bar() -> N
await pilot.pause()
assert panel.collapsed
assert _panel_text(panel) == (
- " config install_browser=ask network=ask respect_robots=true judge=groq"
+ " config install_browser=ask network=ask respect_robots=true"
+ " judge=groq search=off"
)
await until(pilot, lambda: app.focused is app.query_one(Prompt), "the bar to take focus")
# A collapsed panel is a record: the keys do nothing to it any more.
@@ -3413,3 +3463,200 @@ async def test_ctrl_l_clears_a_collapsed_panel_and_keeps_an_open_one() -> None:
await pilot.pause()
assert list(app.query(ConfigPanel)) == [second]
assert not second.collapsed
+
+
+# --- the local fallback judge's notice (spec section 11, Task 11) --------------------
+
+SWITCHED = "Groq limit reached — judging with local ollama qwen3.5:9b"
+
+
+def _bottom_order(app: ProofpathApp) -> list[str]:
+ return [child.id or type(child).__name__ for child in app.query_one("#bottom").children]
+
+
+async def test_the_judge_notice_sits_right_aligned_directly_above_the_prompt() -> None:
+ app, schedulers = build_app()
+ async with app.run_test(size=SIZE) as pilot:
+ notice = app.query_one(JudgeNotice)
+ frame = app.query_one(PromptFrame)
+ footer = app.query_one(CoverageFooter)
+ # Hidden with ``display: none``, never an empty line: no row is spent on it.
+ assert not notice.display
+ assert notice.region.height == 0
+ assert footer.region.bottom == frame.region.y
+ order = _bottom_order(app)
+ assert order.index("judge-notice") == order.index("prompt-frame") - 1
+
+ scheduler = schedulers[0]
+ await submit(pilot, "/check draft.md")
+ run = scheduler.runs[0]
+ scheduler.push(run, Note(SWITCHED, notice=True))
+ await pilot.pause()
+
+ assert notice.display
+ assert notice.region.height == 1
+ assert notice.region.bottom == frame.region.y # directly above the bar
+ assert notice.styles.text_align == "right"
+ assert notice.region.width == app.query_one("#bottom").region.width
+ assert notice.text == SWITCHED
+ # The footer keeps its fixed height and sits above the notice (rule 6).
+ assert footer.region.height == 3
+ assert footer.region.bottom == notice.region.y
+ # It is still a note of the run, too, where the run's other lines are.
+ assert SWITCHED in _notes(app)
+
+ scheduler.move(run, "done", report=a_report())
+ await pilot.pause()
+ assert notice.display # it stays until the next run starts
+
+ await submit(pilot, "/check other.md")
+ await pilot.pause()
+ assert not notice.display
+ assert notice.region.height == 0
+ assert footer.region.bottom == frame.region.y
+
+
+async def test_an_ordinary_note_never_shows_the_judge_notice() -> None:
+ app, schedulers = build_app()
+ async with app.run_test(size=SIZE) as pilot:
+ await submit(pilot, "/check draft.md")
+ schedulers[0].push(schedulers[0].runs[0], Note("judgement not cached: OSError"))
+ await pilot.pause()
+ assert not app.query_one(JudgeNotice).display
+
+
+async def test_the_rich_footer_keeps_its_height_under_the_notice() -> None:
+ app, schedulers = build_app(theme=RICH)
+ async with app.run_test(size=(100, 30)) as pilot:
+ await submit(pilot, "/check draft.md")
+ schedulers[0].push(schedulers[0].runs[0], Note(SWITCHED, notice=True))
+ await pilot.pause()
+ footer = app.query_one(CoverageFooter)
+ notice = app.query_one(JudgeNotice)
+ assert footer.region.height == 4
+ assert notice.region.height == 1
+ assert notice.region.bottom <= app.query_one(PromptFrame).region.y
+
+
+async def test_each_run_tries_the_api_first_and_the_old_notice_is_cleared() -> None:
+ """Amendment B 9, end to end on the real scheduler and the real pipeline halves.
+
+ Run 1's Groq answers 429 and the run switches to the local model; run 2's Groq
+ answers, so run 2 stays on Groq and shows no notice. Every endpoint is a respx
+ route: nothing leaves the process.
+ """
+ import httpx
+ import respx
+
+ from proofpath import judge as judge_mod
+ from proofpath.secrets import ApiKey
+ from tests.test_verify_search import CLAIM, StubSearcher, searching
+
+ queries = '{"items": [{"id": 0, "english": "x", "queries": ["ChatGPT shut down March 2025"]}]}'
+
+ def ok(model: str) -> httpx.Response:
+ return httpx.Response(
+ 200, json={"model": model, "choices": [{"message": {"content": queries}}]}
+ )
+
+ engines: list[Engine] = []
+
+ def engine_factory(run: Run) -> Engine:
+ built = searching(StubSearcher())
+ built.judge = judge_mod.Judge(
+ judge_mod.build_client(JudgeConfig(), ApiKey("gsk_tui_test", source="test"))
+ )
+ built._closers.append(built.judge.close)
+ engines.append(built)
+ return built
+
+ with respx.mock(assert_all_called=False) as mock:
+ groq = mock.post("https://api.groq.com/openai/v1/chat/completions").mock(
+ side_effect=[httpx.Response(429), ok("openai/gpt-oss-120b")]
+ )
+ mock.get("http://localhost:11434/api/tags").mock(
+ return_value=httpx.Response(200, json={"models": [{"name": "qwen3.5:9b"}]})
+ )
+ mock.post("http://localhost:11434/api/generate").mock(
+ return_value=httpx.Response(200, json={"done": True})
+ )
+ local = mock.post("http://localhost:11434/v1/chat/completions").mock(
+ return_value=ok("qwen3.5:9b")
+ )
+ app = ProofpathApp(
+ Config(), ui.build(force_terminal=True), engine_factory=engine_factory, theme=PLAIN
+ )
+ async with app.run_test(size=SIZE) as pilot:
+ notice = app.query_one(JudgeNotice)
+ await submit(pilot, CLAIM)
+ first = lambda: app._scheduler.runs[0] # noqa: E731 - read after it exists
+ await until(pilot, lambda: first().state in ("done", "failed"), "run 1")
+ assert first().state == "done", first().error
+ assert first().report is not None and first().report.judge_notice == SWITCHED
+ assert notice.display and notice.text == SWITCHED
+
+ await submit(pilot, CLAIM)
+ second = lambda: app._scheduler.runs[1] # noqa: E731
+ assert not notice.display # cleared the moment run 2 started
+ await until(pilot, lambda: second().state in ("done", "failed"), "run 2")
+ assert second().state == "done", second().error
+ assert second().report is not None and second().report.judge_notice is None
+ assert not notice.display
+ await pilot.press("ctrl+c")
+
+ assert groq.call_count == 2 # each run asked Groq first
+ assert local.call_count == 1 # and only run 1 needed the local model
+ assert len(engines) == 2
+
+
+async def test_summarize_shows_the_switch_notice_and_still_writes_the_summary(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Fix round 1 C: ``/summarize`` through the app's own judge, on its worker
+ thread. Groq answers 429, the local model writes the paragraph, and the notice
+ reaches the row above the bar through the loop, never from the worker."""
+ import httpx
+ import respx
+
+ from proofpath.judge import Judge
+
+ monkeypatch.setenv("GROQ_API_KEY", "gsk_summarize_test_not_real")
+ made: list[Judge] = []
+ app: ProofpathApp | None = None
+
+ def factory() -> Judge:
+ assert app is not None
+ made.append(app._default_judge()) # the real builder, with the real hook
+ return made[-1]
+
+ paragraph = "Two sources say something else."
+ with respx.mock(assert_all_called=False) as mock:
+ groq = mock.post("https://api.groq.com/openai/v1/chat/completions").mock(
+ return_value=httpx.Response(429)
+ )
+ mock.get("http://localhost:11434/api/tags").mock(
+ return_value=httpx.Response(200, json={"models": [{"name": "qwen3.5:9b"}]})
+ )
+ mock.post("http://localhost:11434/api/generate").mock(
+ return_value=httpx.Response(200, json={})
+ )
+ local = mock.post("http://localhost:11434/v1/chat/completions").mock(
+ return_value=httpx.Response(
+ 200,
+ json={"model": "qwen3.5:9b", "choices": [{"message": {"content": paragraph}}]},
+ )
+ )
+ app, schedulers = build_app(judge_factory=factory)
+ async with app.run_test(size=SIZE) as pilot:
+ await finished_run(pilot, schedulers[0])
+ await submit(pilot, "/summarize")
+ await until(pilot, lambda: paragraph in _kv_text(app), "the local summary")
+ notice = app.query_one(JudgeNotice)
+ await until(pilot, lambda: notice.display, "the notice")
+ assert notice.text == SWITCHED
+ text = _kv_text(app)
+
+ assert f"(model-written, ollama qwen3.5:9b) {paragraph}" in text
+ assert groq.call_count == 1
+ assert local.call_count == 1
+ assert "gsk_summarize_test_not_real" not in text
diff --git a/tests/test_ui.py b/tests/test_ui.py
index c86ae6d..b6a4db1 100644
--- a/tests/test_ui.py
+++ b/tests/test_ui.py
@@ -685,6 +685,29 @@ def test_footer_names_the_judge_tokens_on_their_own_line() -> None:
]
+def test_footer_prints_the_search_lines_after_search() -> None:
+ instance, out, _ = _build()
+ ui.footer(
+ instance,
+ Footer(
+ counts="4 refs: 4 ok",
+ coverage=(100, 0, 0),
+ api_calls=0,
+ elapsed=1.0,
+ written=None,
+ weak=False,
+ cancelled=False,
+ search=(
+ "evidence search is experimental — …",
+ "claims not searched 2",
+ ),
+ ),
+ )
+ lines = out.getvalue().splitlines()
+ assert "search evidence search is experimental — …" in lines
+ assert "search claims not searched 2" in lines
+
+
def test_footer_says_nothing_about_a_judge_that_never_ran() -> None:
instance, out, _ = _build()
ui.footer(
diff --git a/tests/test_verify_search.py b/tests/test_verify_search.py
new file mode 100644
index 0000000..3c5506f
--- /dev/null
+++ b/tests/test_verify_search.py
@@ -0,0 +1,477 @@
+"""The Searching stage end to end, offline (OPEN-ITEMS 17.1a, spec 2026-09-28).
+
+A stub searcher answers by substring, pages come from ``StubFetcher``, and both models
+are the fakes ``tests/test_verify.py`` uses. Nothing here opens a socket.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from proofpath import oa
+from proofpath.cache import Cache, claim_hash
+from proofpath.config import Config, Permissions, SearchConfig
+from proofpath.fetch import Fetched, Outcome
+from proofpath.judge import Judge, JudgeUnavailable
+from proofpath.polite import ProviderError
+from proofpath.report import (
+ FOUND_BY_PROOFPATH,
+ LANGUAGE_UNSUPPORTED,
+ SEARCH_UNAVAILABLE,
+ Kind,
+)
+from proofpath.search import SearchHit
+from proofpath.search.queries import LLM_LIMIT, sentence_query
+from proofpath.secrets import CREDENTIALS_MISSING
+from proofpath.verify import (
+ NO_SOURCE_HINT,
+ NO_SOURCE_IN_TEXT,
+ SEARCH_CANNOT_RUN,
+ SEARCHING,
+ Engine,
+ decide_all,
+ prepare,
+ verify,
+)
+from tests.fakes import TableScorer, WordEmbedder
+from tests.test_verify import (
+ LOW_REFUTED_ROW,
+ NEI_ROW,
+ REFUTED_ROW,
+ SUPPORTED_ROW,
+ FakeJudgeClient,
+ StubFetcher,
+ _opinion,
+ engine,
+ fetched,
+ words,
+)
+
+CLAIM = "OpenAI shut down ChatGPT in March 2025."
+PAGE_A = "https://news.test/openai"
+PAGE_B = "https://wiki.test/ChatGPT"
+BACKING = "OpenAI shut down ChatGPT in March 2025 after a vote."
+# A sentence of its own, so the page splits into the backing line and the rest, and
+# long enough for the page to count as full text.
+FILLER = f"The rest is padding: {words(oa.FULLTEXT_MIN_WORDS + 10)}."
+
+
+class StubSearcher:
+ name = "stub"
+
+ def __init__(
+ self, answers: dict[str, list[str]] | None = None, *, error: ProviderError | None = None
+ ) -> None:
+ self.answers = answers or {}
+ self.error = error
+ self.queries: list[str] = []
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ self.queries.append(query)
+ if self.error is not None:
+ raise self.error
+ for key, urls in self.answers.items():
+ if key in query:
+ return [SearchHit(url, "", rank) for rank, url in enumerate(urls, start=1)]
+ return []
+
+
+def searching(
+ searcher: StubSearcher | None,
+ *,
+ pages: dict[str, str] | None = None,
+ table: dict[str, tuple[float, float, float]] | None = None,
+ config: Config | None = None,
+ network_allowed: bool = True,
+) -> Engine:
+ fetcher = StubFetcher(
+ {url: fetched(url, text) for url, text in (pages or {}).items()},
+ network_allowed=network_allowed,
+ network_note="" if network_allowed else "network not permitted",
+ )
+ built = engine(
+ fetcher=fetcher,
+ config=config,
+ embedder=WordEmbedder(),
+ scorer=TableScorer(table or {BACKING: SUPPORTED_ROW}),
+ )
+ built.searcher = searcher
+ return built
+
+
+def test_a_bare_claim_is_searched_and_the_backing_page_is_reported_as_found() -> None:
+ searcher = StubSearcher({"ChatGPT": [PAGE_A]})
+ report = verify(CLAIM, searching(searcher, pages={PAGE_A: f"{BACKING} {FILLER}"}))
+
+ assert searcher.queries == [sentence_query(CLAIM)]
+ assert report.counts() == {Kind.EVIDENCE_FOUND: 1}
+ found = report.findings[0]
+ assert found.verdict is not None and found.verdict.passage is not None
+ assert found.verdict.passage.text == BACKING
+ assert found.detail[-1] == FOUND_BY_PROOFPATH
+ assert report.sources[0].reference.origin == "search"
+ assert report.sources[0].reference.raw == PAGE_A
+ assert SEARCHING in [stage.name for stage in report.stages]
+ assert report.search is not None
+ assert (report.search.searched, report.search.pages_found, report.search.pages_read) == (
+ 1,
+ 1,
+ 1,
+ )
+
+
+def test_a_search_that_finds_nothing_is_no_evidence_never_refuted() -> None:
+ report = verify(CLAIM, searching(StubSearcher()))
+
+ assert report.counts() == {Kind.NO_EVIDENCE: 1}
+ assert report.findings[0].claim is not None
+ assert report.findings[0].claim.text == CLAIM
+
+
+def test_an_unreadable_found_page_keeps_its_state_and_says_who_found_it() -> None:
+ report = verify(CLAIM, searching(StubSearcher({"ChatGPT": [PAGE_A]})))
+
+ assert report.counts() == {Kind.UNVERIFIED: 1, Kind.NO_EVIDENCE: 1}
+ unread = next(f for f in report.findings if f.kind is Kind.UNVERIFIED)
+ assert unread.detail[-1] == FOUND_BY_PROOFPATH
+
+
+def test_a_page_that_says_nothing_either_way_is_nei_and_no_evidence() -> None:
+ report = verify(
+ CLAIM,
+ searching(
+ StubSearcher({"ChatGPT": [PAGE_A]}),
+ pages={PAGE_A: f"{BACKING} {FILLER}"},
+ table={BACKING: NEI_ROW},
+ ),
+ )
+ assert Kind.NOT_SUPPORTED not in report.counts()
+ assert report.counts()[Kind.NO_EVIDENCE] == 1
+ assert report.counts()[Kind.NEI] == 1
+ nei = next(f for f in report.findings if f.kind is Kind.NEI)
+ assert nei.detail[-1] == FOUND_BY_PROOFPATH
+
+
+def test_a_provider_that_fails_is_reported_and_the_claim_is_not_called_empty() -> None:
+ searcher = StubSearcher(error=ProviderError("HTTP 503", 503, None))
+ report = verify(CLAIM, searching(searcher))
+
+ assert report.counts() == {Kind.UNVERIFIED: 1}
+ assert report.findings[0].state == SEARCH_UNAVAILABLE
+ assert report.search is not None and report.search.searched == 0
+
+
+def test_without_a_searcher_the_old_parse_error_stands_with_the_setup_hint() -> None:
+ ready = prepare(CLAIM, searching(None))
+ errors = [f for f in ready.findings if f.kind is Kind.PARSE_ERROR]
+ assert [f.title for f in errors] == [NO_SOURCE_IN_TEXT]
+ assert errors[0].detail == (NO_SOURCE_HINT,)
+ assert "search.provider tavily" in NO_SOURCE_HINT
+ assert "search.provider searxng" in NO_SOURCE_HINT
+ assert "TAVILY_API_KEY" in NO_SOURCE_HINT
+ assert "search.base_url" in NO_SOURCE_HINT
+
+
+def test_no_search_and_a_denied_network_search_nothing() -> None:
+ off = StubSearcher({"ChatGPT": [PAGE_A]})
+ built = searching(off)
+ built.search = False
+ prepare(CLAIM, built)
+ denied = StubSearcher({"ChatGPT": [PAGE_A]})
+ prepare(CLAIM, searching(denied, network_allowed=False))
+ assert off.queries == [] and denied.queries == []
+
+
+def test_a_configured_engine_with_search_off_gives_the_parse_error_with_no_hint() -> None:
+ """A searcher is already set up; telling the user to configure it is wrong."""
+ built = searching(StubSearcher({"ChatGPT": [PAGE_A]}))
+ built.search = False
+ ready = prepare(CLAIM, built)
+ errors = [f for f in ready.findings if f.kind is Kind.PARSE_ERROR]
+ assert [f.title for f in errors] == [NO_SOURCE_IN_TEXT]
+ assert errors[0].detail == ()
+
+
+def test_a_configured_provider_that_cannot_run_says_why() -> None:
+ built = searching(None)
+ built.search_problem = "set TAVILY_API_KEY in .env"
+ ready = prepare(CLAIM, built)
+ [item] = ready.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == CREDENTIALS_MISSING
+ assert item.detail == ("set TAVILY_API_KEY in .env",)
+
+
+def test_a_page_with_no_text_and_a_broken_provider_keeps_the_parse_error() -> None:
+ """Controller decision, fix round 1b (2026-09-28): a page with nothing to search
+ keeps its plain parse error even when a provider is configured and broken -- there
+ is no sentence to search, so pointing at the search credentials would mislead."""
+ empty_url = "https://empty.test/story"
+ fetcher = StubFetcher({empty_url: own_page(empty_url, "")})
+ built = engine(fetcher=fetcher, embedder=WordEmbedder(), scorer=TableScorer({}))
+ built.searcher = None
+ built.search_problem = "set TAVILY_API_KEY in .env"
+ ready = prepare(empty_url, built)
+ assert ready.findings
+ assert all(f.kind is Kind.PARSE_ERROR for f in ready.findings)
+ assert CREDENTIALS_MISSING not in {f.state for f in ready.findings}
+ assert SEARCH_CANNOT_RUN not in {f.title for f in ready.findings}
+
+
+def test_search_off_suppresses_the_credential_finding_too() -> None:
+ """``--no-search`` means no search, not even the "cannot run" finding (rule 3)."""
+ built = searching(None)
+ built.search_problem = "set TAVILY_API_KEY in .env"
+ built.search = False
+ ready = prepare(CLAIM, built)
+ assert [f.title for f in ready.findings] == [NO_SOURCE_IN_TEXT]
+ assert SEARCH_CANNOT_RUN not in [f.title for f in ready.findings]
+
+
+def test_a_refuted_result_from_a_found_page_says_so() -> None:
+ report = verify(
+ CLAIM,
+ searching(
+ StubSearcher({"ChatGPT": [PAGE_A]}),
+ pages={PAGE_A: f"{BACKING} {FILLER}"},
+ table={BACKING: REFUTED_ROW},
+ ),
+ )
+ assert report.counts() == {Kind.NOT_SUPPORTED: 1}
+ finding = report.findings[0]
+ assert finding.title == "claim is not supported by a page proofpath found"
+ assert finding.detail[-1] == FOUND_BY_PROOFPATH
+
+
+def own_page(url: str, text: str) -> Fetched:
+ """A page fetched as the *target* itself, read straight from its text (no HTML
+ parsing needed): its own host must never be counted as a source for it."""
+ return Fetched(
+ url=url,
+ final_url=url,
+ step=1,
+ outcome=Outcome.OK,
+ status=200,
+ content_type="text/plain",
+ kind="text",
+ body=b"",
+ text=text,
+ notes=[],
+ )
+
+
+def test_a_hit_on_the_input_pages_own_host_is_dropped() -> None:
+ own_url = "https://own.test/story"
+ own_host_hit = "https://own.test/other-page"
+ searcher = StubSearcher({"ChatGPT": [own_host_hit, PAGE_A]})
+ fetcher = StubFetcher(
+ {own_url: own_page(own_url, CLAIM), PAGE_A: fetched(PAGE_A, f"{BACKING} {FILLER}")}
+ )
+ built = engine(
+ fetcher=fetcher, embedder=WordEmbedder(), scorer=TableScorer({BACKING: SUPPORTED_ROW})
+ )
+ built.searcher = searcher
+ report = verify(own_url, built)
+ assert report.search is not None and report.search.pages_found == 1
+ assert [source.reference.raw for source in report.sources] == [PAGE_A]
+
+
+class SecondCallFails:
+ """A searcher that answers the first query and then goes down."""
+
+ name = "stub"
+
+ def __init__(self, first: list[str], error: ProviderError) -> None:
+ self.first = first
+ self.error = error
+ self.queries: list[str] = []
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ self.queries.append(query)
+ if len(self.queries) == 1:
+ return [SearchHit(url, "", rank) for rank, url in enumerate(self.first, start=1)]
+ raise self.error
+
+
+def test_a_provider_error_on_the_second_claim_keeps_the_first_claims_results() -> None:
+ second_claim = "Google shut down Bard in June 2025."
+ body = f"{CLAIM} {second_claim}"
+ searcher = SecondCallFails([PAGE_A], ProviderError("HTTP 503", 503, None))
+ report = verify(body, searching(searcher, pages={PAGE_A: f"{BACKING} {FILLER}"}))
+
+ assert len(searcher.queries) == 2
+ assert report.search is not None and report.search.searched == 1
+ found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND)
+ assert found.claim is not None and found.claim.text == CLAIM
+ assert report.counts()[Kind.UNVERIFIED] == 1
+ unavailable = next(f for f in report.findings if f.state == SEARCH_UNAVAILABLE)
+ assert unavailable.detail == ("stub: HTTP 503",)
+
+
+def test_a_text_that_cites_something_is_never_searched() -> None:
+ searcher = StubSearcher({"ChatGPT": [PAGE_A]})
+ prepare(f"{CLAIM} See {PAGE_B}", searching(searcher))
+ assert searcher.queries == []
+
+
+def test_a_text_with_no_sentence_to_search_keeps_its_parse_error() -> None:
+ """A search over nothing would end on no finding at all and read as clean (rule 6)."""
+ searcher = StubSearcher()
+ report = verify("???", searching(searcher))
+ assert searcher.queries == []
+ assert report.search is None
+ assert [f.title for f in report.findings] == [NO_SOURCE_IN_TEXT]
+
+
+def test_the_cap_limits_the_search_and_the_rest_is_counted() -> None:
+ body = " ".join(f"Company{i} sold {i} million units in 2020." for i in range(7))
+ searcher = StubSearcher()
+ report = verify(body, searching(searcher, config=Config(search=SearchConfig(max_claims=5))))
+ assert len(searcher.queries) == 5
+ assert report.search is not None
+ assert (report.search.eligible, report.search.searched) == (7, 5)
+
+
+def test_a_rate_limited_judge_falls_back_to_the_sentence_and_the_report_says_so() -> None:
+ searcher = StubSearcher()
+ built = searching(searcher)
+ built.judge = Judge(FakeJudgeClient([JudgeUnavailable("HTTP 429 from https://api.test")])) # type: ignore[arg-type]
+ report = verify(CLAIM, built)
+ assert searcher.queries == [sentence_query(CLAIM)]
+ assert report.search is not None
+ assert report.search.notices[0].startswith(LLM_LIMIT)
+
+
+@pytest.mark.parametrize("web_search", ["ask", "deny"])
+def test_ask_or_deny_turns_the_search_off_in_the_default_engine(web_search: str) -> None:
+ config = Config(
+ permissions=Permissions(web_search=web_search), # type: ignore[arg-type]
+ search=SearchConfig(provider="searxng", base_url="http://localhost:8888"),
+ )
+ with Engine.default(config, interactive=True, no_cache=True) as built:
+ assert built.searcher is not None
+ assert built.search is False
+ allowed = Config(search=SearchConfig(provider="searxng", base_url="http://localhost:8888"))
+ with Engine.default(allowed, interactive=False, no_cache=True) as built:
+ assert built.search is True
+
+
+# --- Amendment A: short text, and claims that are not in English --------------------
+
+TURKISH = "OpenAI battı." # noqa: RUF001 - a real Turkish letter, not a typo
+ENGLISH = "OpenAI went bankrupt."
+BANKRUPT = "OpenAI went bankrupt in 2026, the court said."
+
+
+def test_a_one_line_post_is_searched_even_when_nothing_in_it_looks_checkworthy() -> None:
+ searcher = StubSearcher()
+ report = verify("OpenAI shut down", searching(searcher))
+ assert searcher.queries == ["OpenAI shut down"]
+ assert report.counts() == {Kind.NO_EVIDENCE: 1}
+
+
+def test_without_a_judge_a_turkish_claim_is_reported_not_searched() -> None:
+ searcher = StubSearcher({"OpenAI": [PAGE_A]})
+ report = verify(TURKISH, searching(searcher))
+ assert searcher.queries == []
+ [item] = report.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == LANGUAGE_UNSUPPORTED
+ assert report.search is not None and report.search.searched == 0
+
+
+def test_the_judge_translates_a_turkish_claim_and_the_report_shows_both() -> None:
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": ENGLISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ searcher = StubSearcher({"bankruptcy": [PAGE_A]})
+ built = searching(
+ searcher,
+ pages={PAGE_A: f"{BANKRUPT} {FILLER}"},
+ table={BANKRUPT: SUPPORTED_ROW},
+ )
+ built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type]
+ report = verify(TURKISH, built)
+ assert searcher.queries == ["OpenAI bankruptcy"]
+ found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND)
+ assert found.claim is not None and found.claim.text == TURKISH
+ assert f"checked as: {ENGLISH}" in found.detail
+ assert found.detail[-1] == FOUND_BY_PROOFPATH
+
+
+def test_a_judge_that_echoes_a_turkish_claim_leaves_it_language_unsupported() -> None:
+ """Fix round 1 A: Turkish never reaches the English-only models as if translated."""
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": TURKISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ searcher = StubSearcher({"bankruptcy": [PAGE_A]})
+ built = searching(searcher, pages={PAGE_A: f"{BANKRUPT} {FILLER}"})
+ built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type]
+ report = verify(TURKISH, built)
+ [item] = report.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == LANGUAGE_UNSUPPORTED
+ assert searcher.queries == []
+
+
+def test_a_translated_claim_the_search_finds_nothing_for_says_what_was_checked() -> None:
+ """Fix round 1, item 1: ``_no_evidence`` names the English text too, not just the
+ reader's own sentence, when the claim was translated."""
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": ENGLISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ built = searching(StubSearcher())
+ built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type]
+ report = verify(TURKISH, built)
+ assert report.counts() == {Kind.NO_EVIDENCE: 1}
+ finding = report.findings[0]
+ assert finding.claim is not None and finding.claim.text == TURKISH
+ assert f"checked as: {ENGLISH}" in finding.detail
+
+
+def test_a_translated_claims_judgement_is_cached_under_the_english_hash(tmp_path: Path) -> None:
+ """The judgement cache is keyed the same way the verdict cache is: a translated
+ claim's opinion belongs to the English text the judge actually read, not to the
+ reader's own sentence (item 5, controller decision 2026-09-28)."""
+ translate = json.dumps(
+ {"items": [{"id": 0, "english": ENGLISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ # A low-tier REFUTED (spec section 9 step 8): a real passage, so it escalates.
+ opinion = _opinion("c0", "REFUTED")
+ searcher = StubSearcher({"bankruptcy": [PAGE_A]})
+ with Cache(tmp_path / "c.sqlite3") as db:
+ built = engine(
+ fetcher=StubFetcher({PAGE_A: fetched(PAGE_A, f"{BANKRUPT} {FILLER}")}),
+ embedder=WordEmbedder(),
+ scorer=TableScorer({BANKRUPT: LOW_REFUTED_ROW}),
+ cache=db,
+ )
+ built.searcher = searcher
+ judge = Judge(FakeJudgeClient([translate, opinion])) # type: ignore[arg-type]
+ built.judge = judge
+ ready = prepare(TURKISH, built)
+ # The ``sources`` row a real fetch would have written (``tests.test_verify``'s
+ # ``source_row``, inlined here: the searching stage's page is found by URL,
+ # not by DOI, so ``StubFetcher`` never registers it itself).
+ db.add_source(
+ f"url:{PAGE_A}",
+ scheme="url",
+ title="",
+ url=PAGE_A,
+ text_kind="fulltext",
+ raw_text=None,
+ )
+ report = decide_all(ready, built)
+
+ not_supported = next(f for f in report.findings if f.kind is Kind.NOT_SUPPORTED)
+ assert not_supported.source_id is not None
+ assert not_supported.judge is not None
+
+ stored = db.get_judgement(claim_hash(ENGLISH), not_supported.source_id, judge.name)
+ assert stored is not None
+ # Never under the reader's own sentence: that would reuse this opinion for a
+ # differently translated run of the same Turkish claim later on.
+ assert db.get_judgement(claim_hash(TURKISH), not_supported.source_id, judge.name) is None
From a0c6a86c2ffc01b768bf6da538dace616e50a7e9 Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Mon, 28 Sep 2026 16:53:59 +0300
Subject: [PATCH 2/7] docs(search): document evidence search and the local
judge fallback
The README gains a section on searching claims with no source, a row in
the What it reads table, and a note on the judge's fallback to a local
qwen3.5:9b. The landing page's reads list and the X hint name the new
path. The AVeriTeC score is a marker until the corrected eval runs.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
README.md | 42 ++++++++++++++++++++++++-
docs/eval/2026-09-28-averitec-search.md | 37 ++++++++++++++++++++++
docs/superpowers/OPEN-ITEMS.md | 4 +--
site/src/data/reads.ts | 5 +++
src/proofpath/providers/social.py | 5 ++-
5 files changed, 89 insertions(+), 4 deletions(-)
create mode 100644 docs/eval/2026-09-28-averitec-search.md
diff --git a/README.md b/README.md
index 393b25f..a387bb9 100644
--- a/README.md
+++ b/README.md
@@ -113,6 +113,7 @@ and the shared browser cache, never system-wide — and remembers the answer.
| 🔑 | Reddit | a free Reddit app of your own |
| 🔑 | LLM second opinion (`--judge`, `--summarize`) | a free Groq key, or Ollama with no key |
| 🔑 | Sites that block robots | your OK for a 280 MB browser, asked first |
+| 🔑 | Claims with no source, searched on the web (experimental) | a free Tavily key, or your own SearXNG |
| ❌ | Scanned PDFs | no OCR yet |
| ❌ | Old formats: `.doc`, `.odt`, `.rtf` | save as `.docx` or PDF |
| ❌ | Paywalls | reported as unread |
@@ -299,12 +300,45 @@ own words are never allowed to stand as their own evidence.
| Reddit | with a **free app you register yourself**: put `REDDIT_CLIENT_ID` and `REDDIT_CLIENT_SECRET` in `.env`. Without them the run says `UNVERIFIED (credentials missing)` and names both variables — it never quietly skips the post |
| Mastodon | best effort, per instance. Many instances now require a login for the public API, and that answer is reported as `UNVERIFIED (blocked)`, not as a missing post |
| Lemmy | best effort, per instance, through the `/api/v3` every instance serves without a login; a private instance's `not_logged_in` is reported as `UNVERIFIED (blocked)`, not as a missing post. A post's URL and the links in its body or in a comment. Known by its host — `lemmy.*` and the big instances without that prefix (`lemm.ee`, `sh.itjust.works`, `beehaw.org`, `programming.dev`, `feddit.org`) — never by `/post/` alone, which is any blog's address; a post on an instance outside that rule is read as a page |
-| X / Twitter | cannot be read at all. `check --url` says so and asks you to paste the text; the links inside it are then verified normally |
+| X / Twitter | cannot be read at all. Paste the text instead: a link inside it is verified normally, and with evidence search set up a text with no link is searched (experimental) |
The Reddit path is built against Reddit's documented shapes and covered by fixtures, but it
has never run against Reddit on this machine — nobody here has an app to register. Bluesky,
Hacker News, Mastodon, Lobste.rs and Lemmy were each read live before release.
+## Claims with no source (experimental)
+
+A plain text that cites nothing — no link, no `[12]` marker, no bibliography — has
+nothing to check it against. When a search provider is configured, proofpath searches
+the web for it instead of stopping at a parse error:
+
+```bash
+proofpath config set search.provider tavily
+echo 'TAVILY_API_KEY=tvly-…' >> .env
+```
+
+or `search.provider searxng` plus `search.base_url` for your own instance. Setting
+the provider **is** the consent — search then runs automatically on source-less text;
+`--no-search` turns it off for one run, and `permissions.web_search = deny` turns it
+off for good.
+
+Up to 5 check-worthy sentences are searched, up to 3 pages each. Each page is fetched
+and read like any other source — the search result's own snippet is never used as
+evidence (rule 1). A claim a page supports is reported `SUPPORTED (found by
+proofpath)`, with its passage; a claim nothing could confirm is `NO EVIDENCE FOUND
+(searched)`, never `REFUTED` (rule 2). Every such finding and source carries a
+`FOUND BY PROOFPATH (not cited by the author)` tag, so it is never mistaken for
+something the author themselves pointed at.
+
+With `--judge`, the model rewrites each claim into a search query (and translates it,
+if it is not in English) before searching. If the judge is rate-limited, the local
+fallback below takes over; only if that fails too is the plain sentence searched, and
+the report says so.
+
+**This is experimental.** On AVeriTeC's search scenario, proofpath scores
+`<>` against a 0.708 majority baseline, and every report from a
+run that searched prints that line at the top.
+
## Optional LLM judge (v0.3)
Everything above runs locally, and the default run makes **zero** LLM calls. Two flags
@@ -354,6 +388,12 @@ fixture-tested and were not exercised live in v0.3.0. All three speak
the OpenAI `chat/completions` shape. The key comes from `GROQ_API_KEY` / `GEMINI_API_KEY`
in the environment or a `.env` file, never from config, and is never printed.
+**When the judge stops answering.** If the configured judge hits its rate limit or
+otherwise stops answering, the run switches at once to `qwen3.5:9b` in your local
+Ollama, if it is installed (`ollama pull qwen3.5:9b`; proofpath never pulls it). It says so on stderr, in the TUI
+above the prompt, and in the report. The next run tries the configured API again first.
+`proofpath config set judge.fallback off` turns this off.
+
## Measured
| What | Set | Result |
diff --git a/docs/eval/2026-09-28-averitec-search.md b/docs/eval/2026-09-28-averitec-search.md
new file mode 100644
index 0000000..e4f178c
--- /dev/null
+++ b/docs/eval/2026-09-28-averitec-search.md
@@ -0,0 +1,37 @@
+# AVeriTeC dev — 2026-09-28
+
+- claims: 100 (limit=100)
+- dataset: `https://raw.githubusercontent.com/MichSchli/AVeriTeC/main/data/dev.json` (sha256 499793726b4a…)
+- one run of the whole product: real claims, real source pages, real fetch ladder.
+
+## Headline
+
+3-way accuracy **0.326** vs majority baseline 0.708, over 89 of 100 claims.
+
+The 3-way number excludes the Conflicting Evidence/Cherrypicking rows, which proofpath has no verdict for; counting them as wrong gives a 4-way accuracy of 0.290.
+
+## Per label
+
+| label | n | correct | accuracy |
+|---|---|---|---|
+| Supported | 19 | 9 | 0.474 |
+| Refuted | 63 | 18 | 0.286 |
+| Not Enough Evidence | 7 | 2 | 0.286 |
+| Conflicting Evidence/Cherrypicking | 11 | 0 | 0.000 |
+
+## Source coverage
+
+| state | count |
+|---|---|
+| ok | 178 |
+| UNVERIFIED (blocked, browser not permitted) | 80 |
+| UNVERIFIED (blocked, robots.txt) | 36 |
+| UNVERIFIED (unreachable) | 2 |
+| UNVERIFIED (provider unavailable) | 1 |
+
+0 of 297 source URLs are web.archive.org snapshots (0.0 %).
+
+## Notes
+
+
+
diff --git a/docs/superpowers/OPEN-ITEMS.md b/docs/superpowers/OPEN-ITEMS.md
index ca2f253..48d3e83 100644
--- a/docs/superpowers/OPEN-ITEMS.md
+++ b/docs/superpowers/OPEN-ITEMS.md
@@ -542,8 +542,8 @@ against. "Paste the text" is only an answer for a tweet that links its source.
| # | Item | Note |
|---|---|---|
| 17.1 | **Evidence search for a source-less claim** (decided 2026-09-18: this is the fix, path B) | An opt-in stage for a document that cites nothing: query a web search provider with the user's own key (Tavily / Brave / SearXNG; consent per §7.1, never on by default), take the top N pages and run them through the existing fetch → retrieval → entailment path. The report must carry a **distinct state** for such a source — `found by proofpath, not cited by the author` — on every finding and in coverage; rules 1–2 stay: passage or no verdict, unreachable stays unreachable. **Gate:** AVeriTeC (§15.1) measures exactly this scenario at 0.270 vs a 0.708 majority baseline, so the stage ships only once a web-claim eval is above baseline, or it ships flagged experimental with that number printed in the report. Not the judge: a judge answering with no source is the truth oracle §3 forbids |
-| 17.1a | **How 17.1 searches** (decided 2026-09-28) | Tavily + SearXNG behind one `Searcher`; query = the cleaned sentence, or 1–2 judge-written queries when `--judge` is set, falling back to the sentence with a visible notice on a rate limit / quota; heuristic check-worthiness filter, cap 5 claims × 3 pages. **Revises "never on by default":** runs automatically on source-less text once a provider is configured (configuring it is the consent); `--no-search` / `permissions.web_search = deny` turn it off. Design: `specs/2026-09-28-evidence-search-design.md` |
-| 17.2 | README:280 and the `check --url` X message overstate "paste the text" | both should say a tweet with no link cannot be checked, until 17.1 lands |
+| 17.1a | **How 17.1 searches** (decided 2026-09-28) — **Implemented 2026-09-28** (`plans/2026-09-28-evidence-search.md`) | Tavily + SearXNG behind one `Searcher`; query = the cleaned sentence, or 1–2 judge-written queries when `--judge` is set, falling back to the sentence with a visible notice on a rate limit / quota; heuristic check-worthiness filter, cap 5 claims × 3 pages. **Revises "never on by default":** runs automatically on source-less text once a provider is configured (configuring it is the consent); `--no-search` / `permissions.web_search = deny` turn it off. Design: `specs/2026-09-28-evidence-search-design.md` |
+| 17.2 | ~~README:280 and the `check --url` X message overstate "paste the text"~~ **Closed 2026-09-28**: fixed by the README X row and `X_HINT`, 2026-09-28 | — |
| 17.3 | The "cites nothing" finding fires for a whole document, never per paragraph | a long pasted post where only some paragraphs link is fine today (it has references); a `.txt` mixing cited and uncited claims reports nothing about the uncited ones — that is the existing per-claim coverage's job, but worth checking it says so |
## 18. Lobste.rs and Lemmy — v0.4.7, 2026-09-18
diff --git a/site/src/data/reads.ts b/site/src/data/reads.ts
index 81474e4..c74cfe3 100644
--- a/site/src/data/reads.ts
+++ b/site/src/data/reads.ts
@@ -48,6 +48,11 @@ export const readColumns: readonly ReadColumn[] = [
name: 'Sites that block robots',
note: 'your OK for a 280 MB browser, asked first',
},
+ {
+ icon: 'search',
+ name: 'Claims with no source',
+ note: 'searched on the web: a free Tavily key, or your own SearXNG',
+ },
],
},
{
diff --git a/src/proofpath/providers/social.py b/src/proofpath/providers/social.py
index e923d88..b362171 100644
--- a/src/proofpath/providers/social.py
+++ b/src/proofpath/providers/social.py
@@ -88,7 +88,10 @@
#: tool never asked anyone about must not be reported as one that could not be
#: reached (product rule 2) — the post is there, this reader may not have it, which
#: is what ``BLOCKED`` means in spec section 15.
-X_HINT = "X cannot be read; paste the post text — the links inside it will be verified"
+X_HINT = (
+ "X cannot be read; paste the post text — its links are verified, and with "
+ "search.provider set a text with no link is searched"
+)
#: What a comment permalink is answered with when the thread came back without that
#: comment anywhere in it. ``UNAVAILABLE`` and not ``UNREACHABLE``: the provider
From 1a5826f2b9ceaa4365fcd4ac9ea10f224d14ebd4 Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Tue, 29 Sep 2026 11:11:22 +0300
Subject: [PATCH 3/7] fix(search): close the gaps the final review found before
merge
A page the search found through a DOI or arXiv URL was reported as a
GHOST REFERENCE, because the resolver compared a title against a URL.
Found pages now go straight to the web reader, and an entry that is only
an identifier is never called a ghost: it resolves, or it is AMBIGUOUS.
A v5 cache migration drops GHOST resolutions cached under the old rule.
The English detector flagged ordinary English ("MIT", "et al.", "Los
Angeles") as foreign, then missed short Spanish, French and German
sentences. It now ignores capitalised tokens, counts accented lowercase
words and a fixed set of function words, and a judge's translation is
used unless the original is certainly English.
A run that hit the claim cap now exits 1 even when every searched claim
was supported, so a partly searched run cannot look clean. The AVeriTeC
eval measures the product's own search path, and pages proofpath found
on its own are fetched without the contact address.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
scripts/eval_averitec.py | 164 +++++++++++++++----
src/proofpath/cache.py | 16 +-
src/proofpath/cli.py | 5 +
src/proofpath/fetch.py | 68 +++++---
src/proofpath/judge.py | 103 +++++++++---
src/proofpath/providers/__init__.py | 1 +
src/proofpath/providers/web.py | 5 +-
src/proofpath/report.py | 40 ++++-
src/proofpath/resolve.py | 77 +++++++++
src/proofpath/sarif.py | 4 +-
src/proofpath/search/__init__.py | 65 +++++++-
src/proofpath/search/language.py | 80 ++++++---
src/proofpath/search/providers.py | 55 +++++--
src/proofpath/search/queries.py | 70 ++++++--
src/proofpath/tui/widgets/footer.py | 7 +-
src/proofpath/verify.py | 59 +++++--
tests/test_cache.py | 45 +++++-
tests/test_check_cli.py | 1 +
tests/test_config_cli.py | 25 +++
tests/test_eval_averitec.py | 157 +++++++++++++-----
tests/test_fetch.py | 64 ++++++++
tests/test_judge.py | 31 ++++
tests/test_judge_fallback.py | 63 +++++++-
tests/test_providers.py | 1 +
tests/test_report.py | 25 ++-
tests/test_resolve.py | 106 ++++++++++++
tests/test_sarif.py | 2 +
tests/test_search.py | 101 +++++++++++-
tests/test_search_queries.py | 159 +++++++++++++++++-
tests/test_verify.py | 5 +
tests/test_verify_golden.py | 1 +
tests/test_verify_search.py | 243 +++++++++++++++++++++++++++-
32 files changed, 1636 insertions(+), 212 deletions(-)
diff --git a/scripts/eval_averitec.py b/scripts/eval_averitec.py
index 59570f8..425caa9 100644
--- a/scripts/eval_averitec.py
+++ b/scripts/eval_averitec.py
@@ -32,18 +32,25 @@
from dataclasses import dataclass, replace
from datetime import date
from pathlib import Path
+from typing import TYPE_CHECKING
from urllib.parse import urlsplit
-from proofpath import pipeline, retrieval
-from proofpath.config import load_config
+from proofpath import claims as claims_mod
+from proofpath import ingest, pipeline, retrieval
+from proofpath.config import SearchConfig, load_config
from proofpath.eval import averitec
from proofpath.fetch import Fetched
from proofpath.models import Label, Passage
from proofpath.paths import cache_dir
-from proofpath.search import Searcher, keep_hits
-from proofpath.search.queries import sentence_query
+from proofpath.polite import ProviderError
+from proofpath.report import LANGUAGE_UNSUPPORTED, SEARCH_UNAVAILABLE
+from proofpath.search import Searcher, SearchKeyError, canonical, search_claim
+from proofpath.search.queries import plan_queries
from proofpath.verify import NO_TEXT, Engine
+if TYPE_CHECKING:
+ from proofpath.judge import Judge
+
# The state a claim gets when the dataset itself names no page to read. It is not a
# fetch failure and must not be counted as one.
NO_SOURCE = "no source url"
@@ -54,6 +61,12 @@
# The host whose share of the source URLs the report has to state: a third of the dev
# split's evidence is already a snapshot, and that changes what the numbers measure.
ARCHIVE_HOST = "web.archive.org"
+# The state a claim gets in search mode when the product would not have searched it:
+# no sentence in it with a word to search for. Scored as NEI, never dropped.
+NOT_SEARCHED = "not searched (no sentence to search)"
+# What the report says was measured, one line (final review, Important 3).
+GOLD_PATH = "gold source URLs, no search"
+SENTENCE_PATH = "sentence queries, no judge"
RESULTS_NAME = "averitec_results.json"
SEARCH_RESULTS_NAME = "averitec_search_results.json"
@@ -93,35 +106,90 @@ def state_for(fetched: Fetched) -> str:
return fetched.outcome.value
-def search_urls(searcher: Searcher, claim: averitec.Claim, limit: int) -> tuple[str, ...]:
- """What the evidence search would read for this claim, minus the answer key.
+@dataclass(frozen=True)
+class Searched:
+ """What the product's search did with one claim.
+
+ ``state`` is ``None`` when it searched; otherwise it is the state the product
+ would have reported instead, and the claim is scored as NEI rather than dropped.
+ ``text`` is what the models check: the searched sentences, in English.
+ """
- The whole fact-checking site is excluded, not just the one article: a site that
- rated the claim quotes its own rating on every related page (the leakage the
- §17.1 gate must not measure). "Site" reaches across subdomains both ways, so a
- rating on ``factcheck.afp.com`` also rules out ``www.afp.com``. That errs towards
- a lower score, never a leaked one.
+ urls: tuple[str, ...]
+ text: str
+ state: str | None = None
+
+
+def search_urls(
+ searcher: Searcher,
+ claim: averitec.Claim,
+ limit: int,
+ *,
+ judge: Judge | None = None,
+ max_claims: int = SearchConfig().max_claims,
+) -> Searched:
+ """What the product's evidence search would read for this claim, minus the answer key.
+
+ The product's own path, step for step (final review, Important 3): the claim is
+ read as pasted text, ``checkworthy`` picks the sentences, ``plan_queries`` writes
+ the queries and applies the language gate, and ``search_claim`` asks the provider
+ for exactly ``limit`` hits per query, as ``verify`` does.
+
+ The one addition is the leakage exclusion: the whole fact-checking site is
+ excluded, not just the one article, because a site that rated the claim quotes its
+ own rating on every related page (the leakage the §17.1 gate must not measure). It
+ is passed to the helper as one more excluded host, which matches across
+ subdomains both ways, so a rating on ``factcheck.afp.com`` also rules out
+ ``www.afp.com``. That errs towards a lower score, never a leaked one.
"""
checker = _site(claim.fact_check_url)
- # Three times the limit, not a fixed few: a checker's own pages can crowd the top
- # of the results, and the claim should still get its full share of other pages.
- hits = [
- hit
- for hit in searcher.search(sentence_query(claim.text), limit * 3)
- if not checker or not _same_site(_site(hit.url), checker)
- ]
- return tuple(hit.url for hit in keep_hits(hits, own_host=None, limit=limit))
+ exclude = (checker,) if checker else ()
+ document = ingest.from_text(claim.text, name="averitec", kind="text")
+ chosen = claims_mod.checkworthy(document, limit=max_claims)
+ if not chosen.claims:
+ return Searched((), claim.text, NOT_SEARCHED)
+ plan = plan_queries([sentence.text for sentence in chosen.claims], judge)
+ urls: list[str] = []
+ seen: set[str] = set()
+ checked: list[str] = []
+ for queries, hypothesis in zip(plan.queries, plan.hypotheses, strict=True):
+ if hypothesis is None:
+ continue # not English and not translated: the product does not search it
+ checked.append(hypothesis)
+ try:
+ hits = search_claim(queries, searcher, limit, exclude=exclude)
+ except SearchKeyError:
+ # Not this claim's problem: every claim would fail the same way, so the
+ # run stops (``main``) rather than scoring them all unverified.
+ raise
+ except ProviderError:
+ # Per claim: this one is unverified, the run goes on (product rule 2).
+ return Searched((), claim.text, SEARCH_UNAVAILABLE)
+ for hit in hits:
+ key = canonical(hit.url)
+ if key not in seen:
+ seen.add(key)
+ urls.append(hit.url)
+ if not checked:
+ return Searched((), claim.text, LANGUAGE_UNSUPPORTED)
+ return Searched(tuple(urls), " ".join(checked))
+
+
+def unsearched_row(claim: averitec.Claim, state: str) -> Row:
+ """A claim the product would not have searched, as the product would report it:
+ no verdict (scored as NEI), and the state that says why."""
+ return Row(claim_id=claim.id, gold=claim.label, predicted=None, states=(state,), urls=())
+
+
+def measured_path(judge: Judge | None) -> str:
+ """The search path a run measured, in one line for the report."""
+ return SENTENCE_PATH if judge is None else f"judge queries ({judge.name})"
def _site(url: str) -> str:
return (urlsplit(url).hostname or "").removeprefix("www.")
-def _same_site(host: str, checker: str) -> bool:
- """One host is the other, or a subdomain of it, in either direction."""
- return host == checker or host.endswith(f".{checker}") or checker.endswith(f".{host}")
-
-
@dataclass(frozen=True)
class AveritecResult:
n: int
@@ -184,7 +252,7 @@ def _md_row(*cells: object) -> str:
return "| " + " | ".join(str(c) for c in cells) + " |"
-def render_report(result: AveritecResult, *, date: str, limit: int) -> str:
+def render_report(result: AveritecResult, *, date: str, limit: int, path: str = GOLD_PATH) -> str:
"""The markdown skeleton. ``## Notes`` is left for the controller to fill in."""
# ``result.counted`` is the denominator the 3-way accuracy and its baseline are
# read over. Printed beside `n`, because "0.80 over 100 claims" would otherwise
@@ -193,6 +261,7 @@ def render_report(result: AveritecResult, *, date: str, limit: int) -> str:
f"# AVeriTeC dev — {date}",
"",
f"- claims: {result.n} (limit={limit or 'none'})",
+ f"- measured path: {path}",
f"- dataset: `{averitec.URL}` (sha256 {averitec.SHA256[:12]}…)",
"- one run of the whole product: real claims, real source pages, real fetch ladder.",
"",
@@ -284,8 +353,13 @@ def _save_rows(path: Path, rows: Sequence[Row]) -> None:
temporary.replace(path)
-def _decide_claim(engine: Engine, claim: averitec.Claim, *, sleep: float) -> Row:
- """Fetch every source of one claim and keep the strongest non-NEI verdict."""
+def _decide_claim(
+ engine: Engine, claim: averitec.Claim, *, sleep: float, anonymous: bool = False
+) -> Row:
+ """Fetch every source of one claim and keep the strongest non-NEI verdict.
+
+ ``anonymous`` is for pages the search found: fetched without the contact address,
+ as the product fetches them (round 2)."""
states: list[str] = []
best: pipeline.Verdict | None = None
fetched_any = False
@@ -295,7 +369,7 @@ def _decide_claim(engine: Engine, claim: averitec.Claim, *, sleep: float) -> Row
# claims too, not just within one.
if sleep:
time.sleep(sleep)
- fetched = engine.fetcher.fetch(url)
+ fetched = engine.fetcher.fetch(url, anonymous=anonymous)
states.append(state_for(fetched))
if not fetched.ok or not fetched.text.strip():
continue
@@ -398,21 +472,34 @@ def main(argv: list[str] | None = None) -> int:
return 2
searcher = engine.searcher
+ # The judge the product would write queries with: the same rule ``verify`` uses.
+ judge = engine.judge if engine.escalate else None
+ path = measured_path(judge) if searcher is not None else GOLD_PATH
scored = 0
try:
for position, claim in enumerate(claims, start=1):
if claim.id in done:
continue
- if searcher is not None:
+ if searcher is None:
+ stored.append(_decide_claim(engine, claim, sleep=args.sleep))
+ else:
# The gold URLs are the answer key; search mode reads what evidence
# search would actually find, which is the §17.1 gate this run exists
# to measure. ``non_urls=0`` because a search hit is always a URL.
- claim = replace(
+ searched = search_urls(
+ searcher,
claim,
- source_urls=search_urls(searcher, claim, config.search.results_per_claim),
- non_urls=0,
+ config.search.results_per_claim,
+ judge=judge,
+ max_claims=config.search.max_claims,
)
- stored.append(_decide_claim(engine, claim, sleep=args.sleep))
+ if searched.state is not None:
+ stored.append(unsearched_row(claim, searched.state))
+ else:
+ found = replace(
+ claim, text=searched.text, source_urls=searched.urls, non_urls=0
+ )
+ stored.append(_decide_claim(engine, found, sleep=args.sleep, anonymous=True))
scored += 1
# After every claim, not at the end: a run this long is interrupted more
# often than it finishes, and the file has to survive that.
@@ -423,12 +510,21 @@ def main(argv: list[str] | None = None) -> int:
)
except KeyboardInterrupt:
print(f"\ninterrupted after {scored} new claims; {results} is usable", file=sys.stderr)
+ except SearchKeyError as exc:
+ # The message names the variable to check, never the key (round 2).
+ print(
+ f"error {exc}\n"
+ " The run stopped: every claim would fail the same way.\n"
+ f" {results} keeps what was scored.",
+ file=sys.stderr,
+ )
+ return 2
finally:
engine.close()
wanted = {claim.id for claim in claims}
rows = [row for row in stored if row.claim_id in wanted]
- report = render_report(score(rows), date=date.today().isoformat(), limit=args.limit)
+ report = render_report(score(rows), date=date.today().isoformat(), limit=args.limit, path=path)
print()
print(report)
suffix = "-search" if args.search else ""
diff --git a/src/proofpath/cache.py b/src/proofpath/cache.py
index fc0c5e5..9dfe390 100644
--- a/src/proofpath/cache.py
+++ b/src/proofpath/cache.py
@@ -28,7 +28,7 @@
from proofpath.pipeline import CUT_DECIMALS, Thresholds
from proofpath.resolve import Candidate, FieldMatch, ResolveResult, Retraction, State, strip_marker
-SCHEMA_VERSION = "4"
+SCHEMA_VERSION = "5"
RAW_TEXT_TTL_DAYS = 7
# A resolution is a statement about a published record, which does not change; the
# month is there so a reference an index had not yet ingested is looked at again.
@@ -191,6 +191,19 @@ def _migrate_to_v4(conn: sqlite3.Connection) -> None:
conn.execute(statement)
+def _migrate_to_v5(conn: sqlite3.Connection) -> None:
+ """v4 -> v5: every cached ``GHOST REFERENCE`` is forgotten, nothing else.
+
+ No table changes. Before v5 an entry that is nothing but an identifier (a bare
+ DOI, a ``doi.org`` or arXiv address) could be called a ghost, and a found page
+ with such an address was resolved as one (final review, C1). Those rows would be
+ served for up to a month under the old rule, and a ghost is the one verdict rule 3
+ cannot let stand on it: the rows are dropped and resolved again. Every other
+ state is still what the providers answered, and stays.
+ """
+ conn.execute("DELETE FROM resolutions WHERE state = ?", (State.GHOST.value,))
+
+
# One step per schema version, oldest first.
#
# Invariant, on which the self-healing in ``Cache.__init__`` rests: every step runs
@@ -202,6 +215,7 @@ def _migrate_to_v4(conn: sqlite3.Connection) -> None:
("2", _migrate_to_v2),
("3", _migrate_to_v3),
("4", _migrate_to_v4),
+ ("5", _migrate_to_v5),
)
diff --git a/src/proofpath/cli.py b/src/proofpath/cli.py
index a74149e..81833b8 100644
--- a/src/proofpath/cli.py
+++ b/src/proofpath/cli.py
@@ -632,6 +632,11 @@ def on_event(event: events.Event) -> None:
else:
written = _write_report(out, report, out_path, default=True)
ui.blank(out)
+ banner = report_mod.search_banner(report)
+ if banner is not None:
+ # At the top of the report, before the first finding (spec 2026-09-28
+ # section 5), and never suppressed: it qualifies everything below it.
+ ui.kv(out, "search", banner)
for item in report_mod.render_diagnostics(report):
ui.diagnostic(out, item)
ui.blank(out)
diff --git a/src/proofpath/fetch.py b/src/proofpath/fetch.py
index a75069d..bcadc95 100644
--- a/src/proofpath/fetch.py
+++ b/src/proofpath/fetch.py
@@ -23,6 +23,7 @@
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
+from functools import partial
from typing import Any, Literal, Protocol
import httpx
@@ -157,7 +158,8 @@ def fetch(self, url: str) -> tuple[int, bytes, str]:
WaybackLookup = Callable[[str], str | None] # url -> snapshot url or None
# The ladder's own lookup also says *why* there is no snapshot: (snapshot, error).
# An injected ``WaybackLookup`` is wrapped into this shape with error ``None``.
-_WaybackLookup = Callable[[str], tuple[str | None, str | None]]
+# The second argument is ``anonymous`` (see ``Fetcher.fetch``).
+_WaybackLookup = Callable[[str, bool], tuple[str | None, str | None]]
@dataclass(frozen=True)
@@ -419,6 +421,7 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched:
"""One climb. A bare URL is one source, so a browser refusal here is one
skipped source; the open-access chain fetches several locations for one
@@ -427,14 +430,22 @@ def fetch(
``use_cache=False`` climbs past a cached copy. A hit carries the text alone
(``body=b""``), which is all a *source* needs; a page read as the document
itself needs its markup back, because its links are its bibliography.
+
+ ``anonymous`` leaves the contact address out of every request of the climb
+ -- robots.txt, httpx, curl_cffi and the Wayback lookup: the User-Agent names
+ the tool and not the user. It is for pages the evidence search found, which
+ the user never chose to contact (final review, round 2). A cited source keeps
+ the address, as the polite pools it is configured for expect.
"""
- result = self._climb(url, text_kind, use_cache=use_cache)
+ result = self._climb(url, text_kind, use_cache=use_cache, anonymous=anonymous)
self._counts[result.outcome] += 1
if counts_as_source and result.outcome is Outcome.BLOCKED_NO_BROWSER:
self._gate.skipped += 1
return result
- def _climb(self, url: str, text_kind: str, *, use_cache: bool = True) -> Fetched:
+ def _climb(
+ self, url: str, text_kind: str, *, use_cache: bool = True, anonymous: bool = False
+ ) -> Fetched:
notes: list[str] = []
if self.network_note:
notes.append(self.network_note)
@@ -457,13 +468,13 @@ def _climb(self, url: str, text_kind: str, *, use_cache: bool = True) -> Fetched
notes=notes,
from_cache=True,
)
- if self._config.fetch.respect_robots and not self._robots_allow(url):
+ if self._config.fetch.respect_robots and not self._robots_allow(url, anonymous):
notes.append(f"step 1 httpx: robots.txt disallows {PRODUCT_TOKEN}")
return _failed(url, 1, Outcome.BLOCKED_ROBOTS, _no_response(url), notes)
# Step 1: httpx.
step = 1
- response, verdict, note = self._with_backoff(url)
+ response, verdict, note = self._with_backoff(url, anonymous)
notes.append(f"step 1 httpx: {note}")
if verdict == "ok":
return self._finish(url, step, response, text_kind, notes)
@@ -476,7 +487,7 @@ def _climb(self, url: str, text_kind: str, *, use_cache: bool = True) -> Fetched
if failure_verdict == "blocked":
# Step 2: TLS impersonation.
step = 2
- response, verdict, note = self._curl(url)
+ response, verdict, note = self._curl(url, anonymous)
notes.append(f"step 2 curl_cffi: {note}")
if verdict == "ok":
return self._finish(url, step, response, text_kind, notes)
@@ -502,7 +513,7 @@ def _climb(self, url: str, text_kind: str, *, use_cache: bool = True) -> Fetched
# Step 4: Wayback, for blocked and unreachable alike. A miss reports the
# last real failure, not the archive's.
step = 4
- snapshot, error = self._wayback(url)
+ snapshot, error = self._wayback(url, anonymous)
if snapshot is None:
notes.append(
f"step 4 wayback: lookup failed ({error})"
@@ -510,7 +521,7 @@ def _climb(self, url: str, text_kind: str, *, use_cache: bool = True) -> Fetched
else "step 4 wayback: no snapshot"
)
else:
- response, verdict, note = self._with_backoff(snapshot)
+ response, verdict, note = self._with_backoff(snapshot, anonymous)
notes.append(f"step 4 wayback: {note}")
if verdict == "ok":
return self._finish(url, step, response, text_kind, notes)
@@ -560,10 +571,10 @@ def _finish(
# --- transports ---------------------------------------------------------
- def _httpx_get(self, url: str) -> _Response:
+ def _httpx_get(self, url: str, anonymous: bool = False) -> _Response:
# PoliteClient.get raises on 429/5xx; the ladder needs the status to classify.
try:
- response = self._polite.client.get(url, headers={"Accept": ACCEPT})
+ response = self._polite.client.get(url, headers={"Accept": ACCEPT, **_agent(anonymous)})
except httpx.HTTPError as exc:
return _Response(None, b"", "", url, error=type(exc).__name__)
retry_after = response.headers.get("Retry-After", "")
@@ -575,11 +586,11 @@ def _httpx_get(self, url: str) -> _Response:
retry_after=float(retry_after) if retry_after.isdigit() else None,
)
- def _with_backoff(self, url: str) -> tuple[_Response, Verdict, str]:
+ def _with_backoff(self, url: str, anonymous: bool = False) -> tuple[_Response, Verdict, str]:
"""An httpx GET with the PoliteClient backoff; the last response is returned, not raised."""
for attempt in range(RETRIES + 1):
self._polite.throttle(url)
- response = self._httpx_get(url)
+ response = self._httpx_get(url, anonymous)
verdict = response.verdict
if verdict != "retryable":
return response, verdict, response.explain(verdict)
@@ -596,9 +607,9 @@ def _with_backoff(self, url: str) -> tuple[_Response, Verdict, str]:
detail = f"retryable, gave up after {RETRIES + 1} attempts"
return response, "retryable", response.describe(detail)
- def _curl(self, url: str) -> tuple[_Response, Verdict, str]:
+ def _curl(self, url: str, anonymous: bool = False) -> tuple[_Response, Verdict, str]:
self._polite.throttle(url)
- get = self._curl_get or self._default_curl
+ get = self._curl_get or partial(self._default_curl, anonymous=anonymous)
try:
status, body, content_type, final_url = get(url)
except TransportFailure as exc:
@@ -612,11 +623,10 @@ def _curl(self, url: str) -> tuple[_Response, Verdict, str]:
response = _Response(status, body, content_type, final_url)
return response, response.verdict, response.explain(response.verdict)
- def _default_curl(self, url: str) -> tuple[int, bytes, str, str]:
+ def _default_curl(self, url: str, *, anonymous: bool = False) -> tuple[int, bytes, str, str]:
# Looked up at call time so tests can monkeypatch ``proofpath.fetch.default_curl_get``.
- return default_curl_get(
- url, timeout=self._timeout, contact_email=self._config.contact.email
- )
+ email = "" if anonymous else self._config.contact.email
+ return default_curl_get(url, timeout=self._timeout, contact_email=email)
def _browser(self, url: str) -> tuple[_Response, Verdict, str]:
self._polite.throttle(url)
@@ -626,12 +636,14 @@ def _browser(self, url: str) -> tuple[_Response, Verdict, str]:
response = _Response(status, body, content_type, url)
return response, response.verdict, response.explain(response.verdict)
- def _lookup_wayback(self, url: str) -> tuple[str | None, str | None]:
+ def _lookup_wayback(self, url: str, anonymous: bool = False) -> tuple[str | None, str | None]:
"""``(snapshot url, error)``. The error is set only when the call itself failed,
never for a well-formed response that simply has no snapshot (a plain miss)."""
self._polite.throttle(WAYBACK_AVAILABLE)
try:
- response = self._polite.client.get(WAYBACK_AVAILABLE, params={"url": url})
+ response = self._polite.client.get(
+ WAYBACK_AVAILABLE, params={"url": url}, headers=_agent(anonymous)
+ )
except httpx.HTTPError as exc:
return None, type(exc).__name__
if response.status_code != 200:
@@ -646,20 +658,20 @@ def _lookup_wayback(self, url: str) -> tuple[str | None, str | None]:
# --- robots -------------------------------------------------------------
- def _robots_allow(self, url: str) -> bool:
+ def _robots_allow(self, url: str, anonymous: bool = False) -> bool:
parsed = httpx.URL(url)
host = parsed.host
if host not in self._robots:
robots_url = str(parsed.copy_with(path="/robots.txt", query=None, fragment=None))
- self._robots[host] = self._load_robots(robots_url)
+ self._robots[host] = self._load_robots(robots_url, anonymous)
rules = self._robots[host]
return rules is None or bool(rules.can_fetch(url, PRODUCT_TOKEN))
- def _load_robots(self, robots_url: str) -> Protego | None:
+ def _load_robots(self, robots_url: str, anonymous: bool = False) -> Protego | None:
"""A missing, erroring or unreachable robots.txt allows everything."""
self._polite.throttle(robots_url)
try:
- response = self._polite.client.get(robots_url)
+ response = self._polite.client.get(robots_url, headers=_agent(anonymous))
except httpx.HTTPError:
return None
if response.status_code != 200:
@@ -667,6 +679,12 @@ def _load_robots(self, robots_url: str) -> Protego | None:
return Protego.parse(response.text)
+def _agent(anonymous: bool) -> dict[str, str]:
+ """The per-request header that replaces the client's User-Agent for an anonymous
+ climb: the tool's name and repository, and no contact address."""
+ return {"User-Agent": user_agent()} if anonymous else {}
+
+
def _no_response(url: str) -> _Response:
return _Response(None, b"", "", url)
@@ -680,7 +698,7 @@ def _transport_failed(url: str, error: str) -> tuple[_Response, Verdict, str]:
def _plain_lookup(lookup: WaybackLookup) -> _WaybackLookup:
"""An injected lookup has no failure channel: a ``None`` is a plain miss."""
- return lambda url: (lookup(url), None)
+ return lambda url, _anonymous: (lookup(url), None)
def _failed(
diff --git a/src/proofpath/judge.py b/src/proofpath/judge.py
index 66e0582..71ab70a 100644
--- a/src/proofpath/judge.py
+++ b/src/proofpath/judge.py
@@ -546,12 +546,19 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
#: is the name as a reader knows it (``Groq``), ``model`` the Ollama tag the run
#: switched to. Every failure switches; the notice says which one it was, so a wrong
#: key is never mistaken for a busy free tier.
-JUDGE_FALLBACK_LIMIT = "{provider} limit reached — judging with local ollama {model}"
-JUDGE_FALLBACK_KEY = (
- "{provider} rejected the key (HTTP {status}) — judging with local ollama {model}"
-)
-JUDGE_FALLBACK_DOWN = "{provider} did not answer ({cause}) — judging with local ollama {model}"
-_LIMIT_STATUSES = frozenset({429, 413}) # Groq answers 413 when a request busts TPM
+_LIMIT_HEAD = "{provider} limit reached"
+_KEY_HEAD = "{provider} rejected the key (HTTP {status})"
+_DOWN_HEAD = "{provider} did not answer ({cause})"
+_JUDGING = " — judging with local ollama {model}"
+JUDGE_FALLBACK_LIMIT = _LIMIT_HEAD + _JUDGING
+JUDGE_FALLBACK_KEY = _KEY_HEAD + _JUDGING
+JUDGE_FALLBACK_DOWN = _DOWN_HEAD + _JUDGING
+#: What ``switched`` says once the local model has failed too: the primary's reason,
+#: then the local one. "Judging with local ollama" would then be false (final review).
+JUDGE_FALLBACK_FAILED = "{head}, and local ollama {model} failed too ({cause})"
+#: The statuses that mean "over the provider's limit", read wherever a limit is named
+#: (the switch notice, the search's fallback notice), so both classify it one way.
+LIMIT_STATUSES = frozenset({429, 413}) # Groq answers 413 when a request busts TPM
_KEY_STATUSES = frozenset({401, 403})
#: ``judge.fallback``: the Ollama model to switch to (``LOCAL_JUDGE_MODEL`` unless the
#: user named another), or ``off``, in any case, for no fallback.
@@ -711,6 +718,10 @@ def __init__(
self._plan: _Plan | None = None
self._local: JudgeClient | None = None
self._primary_detail = ""
+ # The primary's failure and the notice it earned, kept for the moment the
+ # local model fails too and ``switched`` has to say so.
+ self._primary_failure: JudgeUnavailable | None = None
+ self._notice: str | None = None
#: The combined detail once the local model has failed too; ``None`` before.
self._dead: str | None = None
# Local answers from before a ``reset``: the client is gone, the spend is not.
@@ -765,6 +776,8 @@ def reset(self) -> None:
self._dead = None
self.switched = None
self._primary_detail = ""
+ self._primary_failure = None
+ self._notice = None
self._ready = threading.Event()
self._primary.fail_fast = False
self._primary.timeout = self._primary_timeout
@@ -826,16 +839,31 @@ def ask(client: JudgeClient) -> Completion:
if self._dead is not None:
raise JudgeUnavailable(self._dead)
try:
- return ask(local)
+ answer = ask(local)
except JudgeUnavailable as exc:
- # The primary's reason first: it is what the run switched away from, and
- # the search's own notice reads "HTTP 429" out of it (spec section 11).
+ # The primary's reason first: it is what the run switched away from
+ # (spec section 11).
detail = f"{self._primary_detail}; local ollama {local.model}: {exc}"
+ # "Judging with local ollama" is no longer true, so the notice the report
+ # carries says both failed instead (final review, minor).
+ self._both_failed(local.model, exc)
if exc.status is None or exc.status >= 500 or exc.status in _DEAD_STATUSES:
# Latched: a dead or stuck Ollama costs this run one wait, not one per
# batch. Anything else failed this request only (fix round 2).
self._dead = detail
raise JudgeUnavailable(detail, status=exc.status, cause=exc.cause) from None
+ if self._notice is not None:
+ # Answering again after a failure that was about one request only.
+ self.switched = self._notice
+ return answer
+
+ def _both_failed(self, model: str, failure: JudgeUnavailable) -> None:
+ """Say in ``switched`` that the local model failed too, after the primary."""
+ primary = self._primary_failure
+ if primary is None: # pragma: no cover - a local client exists only after a switch
+ return
+ head = _head(self._primary.provider, primary)
+ self.switched = JUDGE_FALLBACK_FAILED.format(head=head, model=model, cause=failure.cause)
def _announce(self, notice: str) -> None:
"""Hand the notice to the listener, once, and never at the answer's expense."""
@@ -899,19 +927,26 @@ def _switch(self, failure: JudgeUnavailable) -> tuple[JudgeClient, str | None]:
) from None
notice = _notice(self._primary.provider, plan.model, failure)
self._primary_detail = str(failure)
+ self._primary_failure = failure
+ self._notice = notice
self._local = self._local_factory(plan.model, self._ollama_url)
self.switched = notice
return self._local, notice
-def _notice(provider: str, model: str, failure: JudgeUnavailable) -> str:
- """The switch notice, naming why the primary was left (fix round 1, B)."""
+def _head(provider: str, failure: JudgeUnavailable) -> str:
+ """Why the primary was left, in the words every notice opens with (fix round 1, B)."""
name = _display(provider)
- if failure.status in _LIMIT_STATUSES:
- return JUDGE_FALLBACK_LIMIT.format(provider=name, model=model)
+ if failure.status in LIMIT_STATUSES:
+ return _LIMIT_HEAD.format(provider=name)
if failure.status in _KEY_STATUSES:
- return JUDGE_FALLBACK_KEY.format(provider=name, status=failure.status, model=model)
- return JUDGE_FALLBACK_DOWN.format(provider=name, cause=failure.cause, model=model)
+ return _KEY_HEAD.format(provider=name, status=failure.status)
+ return _DOWN_HEAD.format(provider=name, cause=failure.cause)
+
+
+def _notice(provider: str, model: str, failure: JudgeUnavailable) -> str:
+ """The switch notice: why the primary was left, and who judges now."""
+ return _head(provider, failure) + _JUDGING.format(model=model)
def build_client(
@@ -1016,7 +1051,18 @@ class JudgedClaim:
# Evidence search (OPEN-ITEMS 17.1a): the judge writes *queries*, never an answer.
# Deciding a claim without a passage is the truth oracle spec section 3 forbids.
-_QUERIES_TOKENS = 1500
+#
+# The answer budget grows with the batch (final review). ``gpt-oss-120b`` charges its
+# reasoning to ``max_tokens`` even at low effort; an item -- the claim in English and
+# two queries -- is ~170 tokens (``estimate_tokens``, a 300-character claim), so a
+# flat 1500 left five claims ~650 tokens to think in, and a pass that ran over came
+# back ``finish_reason=length`` with every query and translation lost. 1500 for the
+# thinking plus 250 per claim gives five claims 2750, and ~840 of prompt beside it
+# still fits Groq's 8k tokens a minute. The ceiling keeps a large ``search.max_claims``
+# under that budget too.
+_QUERIES_REASONING_TOKENS = 1500
+_QUERIES_TOKENS_PER_CLAIM = 250
+_QUERIES_MAX_TOKENS = 5000
_QUERIES_PER_CLAIM = 2
_QUERY_CHARS = 400
_QUERIES_SCHEMA: dict[str, Any] = {
@@ -1044,6 +1090,11 @@ class JudgedClaim:
_USER_HEADING = "## User"
+def _queries_budget(count: int) -> int:
+ """``max_tokens`` for one queries call over ``count`` claims."""
+ return min(_QUERIES_MAX_TOKENS, _QUERIES_REASONING_TOKENS + _QUERIES_TOKENS_PER_CLAIM * count)
+
+
def _split_prompt(text: str) -> tuple[str, str]:
"""A packaged prompt's ``## System`` and ``## User`` halves, as two messages.
@@ -1095,9 +1146,11 @@ def __init__(
self._queries_template = load_prompt("queries")
#: One line per opinion that could not be used, in the order they arrived.
self.skipped: list[str] = []
- #: True once the provider stopped answering; ``detail`` says why.
+ #: True once the provider stopped answering; ``detail`` says why, and
+ #: ``status`` is the HTTP status of that failure (``None`` when there was none).
self.unavailable = False
self.detail = ""
+ self.status: int | None = None
@property
def name(self) -> str:
@@ -1157,6 +1210,7 @@ def review(
self.skipped = []
self.unavailable = False
self.detail = ""
+ self.status = None
opinions: dict[str, JudgeOpinion] = {} if into is None else into
total = len(items)
done = 0
@@ -1169,7 +1223,7 @@ def review(
reasoning_effort=_REASONING_EFFORT,
)
except JudgeUnavailable as exc:
- self._give_up(str(exc))
+ self._give_up(str(exc), status=exc.status)
break
except Exception as exc:
# A bug in the adapter, a library that changed under us: whatever it
@@ -1200,6 +1254,7 @@ def summarize(self, report_markdown: str, *, max_tokens: int = _SUMMARY_TOKENS)
# one that goes down now must not be hidden by an earlier success.
self.unavailable = False
self.detail = ""
+ self.status = None
rendered = self._summary_template.substitute(report=report_markdown)
system, user = _split_prompt(rendered)
messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
@@ -1208,7 +1263,7 @@ def summarize(self, report_markdown: str, *, max_tokens: int = _SUMMARY_TOKENS)
messages, max_tokens=max_tokens, reasoning_effort=_REASONING_EFFORT
)
except JudgeUnavailable as exc:
- self._give_up(str(exc))
+ self._give_up(str(exc), status=exc.status)
return ""
except Exception as exc:
# As in ``review``: the type is named, the message is not, because a
@@ -1236,6 +1291,7 @@ def queries(self, claims: Sequence[str]) -> dict[int, JudgedClaim]:
"""
self.unavailable = False
self.detail = ""
+ self.status = None
if not claims:
return {}
items = "\n".join(
@@ -1247,11 +1303,13 @@ def queries(self, claims: Sequence[str]) -> dict[int, JudgedClaim]:
completion = self._client.complete(
messages,
json_schema=_QUERIES_SCHEMA,
- max_tokens=_QUERIES_TOKENS,
+ max_tokens=_queries_budget(len(claims)),
reasoning_effort=_REASONING_EFFORT,
)
except JudgeUnavailable as exc:
- self._give_up(str(exc))
+ # The status travels with the detail: the search names a limit from the
+ # status alone, never from a substring of the detail (final review).
+ self._give_up(str(exc), status=exc.status)
return {}
except Exception as exc:
# The type is named, the message is not: it can carry the key.
@@ -1259,9 +1317,10 @@ def queries(self, claims: Sequence[str]) -> dict[int, JudgedClaim]:
return {}
return _queries_from(completion.text, len(claims))
- def _give_up(self, detail: str) -> None:
+ def _give_up(self, detail: str, *, status: int | None = None) -> None:
self.unavailable = True
self.detail = detail
+ self.status = status
def _batches(self, items: Sequence[JudgeItem]) -> list[list[JudgeItem]]:
"""Items packed to the smaller of the item cap and the token budget.
diff --git a/src/proofpath/providers/__init__.py b/src/proofpath/providers/__init__.py
index d37b597..1e2e558 100644
--- a/src/proofpath/providers/__init__.py
+++ b/src/proofpath/providers/__init__.py
@@ -124,6 +124,7 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched: ...
def summary(self) -> FetchStats: ...
diff --git a/src/proofpath/providers/web.py b/src/proofpath/providers/web.py
index 882df8f..8544e36 100644
--- a/src/proofpath/providers/web.py
+++ b/src/proofpath/providers/web.py
@@ -64,7 +64,10 @@ def fetch(self, ref: Reference, resolved: ResolveResult) -> list[EvidenceDoc]:
url = find_url(ref.raw) or _address(resolved)
if not url:
return []
- return [_from_page(url, self._fetcher.fetch(url))]
+ # A page the search found is one the user never chose to contact, so it is
+ # fetched without their contact address; a cited one keeps it (round 2).
+ fetched = self._fetcher.fetch(url, anonymous=ref.origin == "search")
+ return [_from_page(url, fetched)]
def _address(resolved: ResolveResult) -> str | None:
diff --git a/src/proofpath/report.py b/src/proofpath/report.py
index 8224bcc..8dd72ce 100644
--- a/src/proofpath/report.py
+++ b/src/proofpath/report.py
@@ -361,12 +361,25 @@ def exit_code(self) -> int:
Spec section 13.3: every reported state counts, including the UNVERIFIED and
LOW CONFIDENCE ones. A run that could not read its sources is not a clean run.
+ The one finding that does not count is ``EVIDENCE_FOUND``: a page the search
+ found, read, and quoted, that supports the claim. It is printed only because
+ a searched run's answer has to be shown (spec 2026-09-28 section 4), where a
+ supported *cited* claim is no finding at all and exits 0. The spec is silent
+ on its exit code, so it exits like the cited run it mirrors (final review).
+ Everything else a search can end in -- REFUTED, NO EVIDENCE FOUND, an unread
+ page, an unavailable search -- is still a finding and still exits 1. So does a
+ search the cap cut short (``searched < eligible``): the check-worthy claims it
+ never reached were not checked at all, and a run that left claims unchecked
+ must not read as a clean one (product rule 6; final review, round 2).
+
``cancelled`` is folded in so that a stopped run is never reported as clean by
a caller that only looks at this number. The CLI does not use that branch: it
exits 2 for a cancelled run, because stopping early is the tool not finishing
rather than a verdict on the document, and ``Report.cancelled`` is what says so.
"""
- return 1 if self.findings or self.cancelled else 0
+ counted = [item for item in self.findings if item.kind is not Kind.EVIDENCE_FOUND]
+ capped = self.search is not None and self.search.searched < self.search.eligible
+ return 1 if counted or capped or self.cancelled else 0
def counts(self) -> dict[Kind, int]:
"""How many findings of each kind, in ``Kind`` declaration order.
@@ -629,6 +642,9 @@ class Footer:
reasons: tuple[tuple[str, int], ...] = ()
# ``search_lines``: empty unless the run searched.
search: tuple[str, ...] = ()
+ # ``search_banner``: the footer's surfaces that cannot print it at the top (the
+ # TUI's docked footer) print it first among the search lines.
+ banner: str | None = None
# Of the judge's answers, those the local fallback gave; beside ``api_calls``,
# which counts provider requests only (spec section 11).
local_calls: int = 0
@@ -684,6 +700,7 @@ def render_footer(report: Report, *, written: str | None = None) -> Footer:
judge_tokens=_judge_tokens(report),
reasons=coverage_reasons(report.coverage),
search=search_lines(report),
+ banner=search_banner(report),
local_calls=0 if report.judge_cost is None else report.judge_cost.local_calls,
judge_notice=report.judge_notice,
)
@@ -715,6 +732,11 @@ def render_markdown(report: Report, *, written_at: datetime | None = None) -> st
"""The whole run as one markdown document. The caller writes it, UTF-8 encoded."""
when = written_at if written_at is not None else datetime.now()
lines: list[str] = [f"# proofpath report — {report.document.name}", ""]
+ banner = search_banner(report)
+ if banner is not None:
+ # First, before anything the run found: a searched run is experimental, and
+ # its findings are read in that light (spec 2026-09-28 section 5).
+ lines.extend([f"> **{banner}**", ""])
lines.append(f"- date: {when:%Y-%m-%d %H:%M:%S}")
lines.extend(f"- {key}: {value}" for key, value in report.models.items())
if report.tier_note:
@@ -1073,13 +1095,25 @@ def _markdown_coverage(report: Report) -> list[str]:
return lines
+def search_banner(report: Report) -> str | None:
+ """The experimental banner, for the top of a run that searched; ``None`` otherwise.
+
+ Apart from ``search_lines`` because it goes somewhere else: at the top of the
+ report on every surface that has one (spec 2026-09-28 section 5), not in the
+ evidence-search block at its end.
+ """
+ return None if report.search is None else search_experimental()
+
+
def search_lines(report: Report) -> tuple[str, ...]:
- """The evidence-search block, one line each, for every surface to print as is."""
+ """The evidence-search block, one line each, for every surface to print as is.
+
+ The experimental banner is not in it: ``search_banner`` opens the report instead.
+ """
item = report.search
if item is None:
return ()
return (
- search_experimental(),
f"evidence search by {item.by}, queries by {item.queries_by}",
f"claims searched {item.searched} of {item.eligible} check-worthy "
f"({item.sentences} sentences)",
diff --git a/src/proofpath/resolve.py b/src/proofpath/resolve.py
index 0060399..7b8bb01 100644
--- a/src/proofpath/resolve.py
+++ b/src/proofpath/resolve.py
@@ -244,6 +244,37 @@ def find_arxiv_id(raw: str) -> str | None:
return ident
+# The label a style prints before a bare identifier ("doi:", "arXiv:"), which is not
+# a title, an author or a year either.
+# The label is followed by a colon or a space, so "doi.org/…" keeps its host.
+_ID_LABEL = re.compile(r"^(?:doi|arxiv)(?:\s*:\s*|\s+)", re.I)
+
+
+def identifier_only(raw: str) -> bool:
+ """Whether an entry is an identifier and nothing else, once its marker is gone.
+
+ A bare DOI, a ``doi.org`` or publisher ``/doi/`` address, an arXiv id or its
+ ``abs``/``pdf`` address. Such an entry carries no title, author or year, so the
+ field checks of spec section 8 have nothing to agree or disagree with: the
+ resolver must not read the matcher's answer to a URL string as "nothing agrees".
+ """
+ text = strip_marker(raw).strip().rstrip(_URL_TAIL)
+ # "DOI: https://doi.org/…" and "": a label, or the angle brackets some
+ # styles print around an address, are no more a title than the identifier is.
+ body = _ID_LABEL.sub("", text, count=1).strip()
+ if body.startswith("<") and body.endswith(">"):
+ body = body[1:-1].strip().rstrip(_URL_TAIL)
+ if not body:
+ return False
+ url = find_url(body)
+ if url is not None:
+ return url == body and bool(find_doi(url) or find_arxiv_id(url))
+ doi = find_doi(body)
+ if doi is not None:
+ return body == doi
+ return _ARXIV_ID_BARE.fullmatch(body) is not None
+
+
_QUOTED = re.compile(r"[\"“]([^\"”]{10,}?)[,.]?[\"”]")
_SPLIT = re.compile(r"[.?!]\s+")
# Every marker form ingest._ENTRY_START accepts: "[12] ", "12. ", "12) " and the
@@ -536,6 +567,28 @@ def classify(candidates: Sequence[Candidate], raw: str) -> ResolveResult:
return ResolveResult(State.GHOST, None, list(candidates))
+IDENTIFIER_ONLY_NOTE = "the entry is only an identifier; no title, author or year to compare"
+IDENTIFIER_UNRESOLVED_NOTE = (
+ "the entry is only an identifier, and it does not resolve; nothing else to search by"
+)
+
+
+def _identified(direct: Candidate, raw: str, notes: list[str]) -> ResolveResult:
+ """An identifier-only entry, resolved to the record its identifier names.
+
+ Low confidence, not ``RESOLVED``: no field was checked, because there was none to
+ check. But the identifier is the reference, so the record it names is the work
+ cited -- and the one thing it can never be is a ghost (spec section 8, rule 3).
+ """
+ return ResolveResult(
+ State.RESOLVED_LOW,
+ direct,
+ [direct],
+ notes=[*notes, IDENTIFIER_ONLY_NOTE],
+ match=match_fields(direct, raw),
+ )
+
+
# --- provider records -> candidates -----------------------------------------------
@@ -814,15 +867,22 @@ def resolve(self, raw: str) -> ResolveResult:
raw = strip_marker(raw)
notes: list[str] = []
pool: list[Candidate] = []
+ # An entry that is nothing but its identifier is decided by the identifier
+ # alone: it has no fields to check a record against (final review, C1).
+ bare = identifier_only(raw)
+ lookup_down = False
arxiv_id = find_arxiv_id(raw)
if arxiv_id:
try:
direct = self.arxiv_id(arxiv_id)
except ProviderError as exc:
+ lookup_down = True
notes.append(f"arxiv unavailable for id lookup ({exc})")
else:
if direct is None:
notes.append(f"arXiv:{arxiv_id} does not resolve")
+ elif bare:
+ return _identified(direct, raw, [*notes, f"arXiv:{arxiv_id} resolved"])
else:
result = classify([direct], raw)
if result.state in (State.RESOLVED, State.RESOLVED_LOW):
@@ -859,10 +919,13 @@ def resolve(self, raw: str) -> ResolveResult:
try:
direct = self.crossref_doi(doi)
except ProviderError as exc:
+ lookup_down = True
notes.append(f"crossref unavailable for DOI lookup ({exc})")
else:
if direct is None:
notes.append(f"DOI {doi} does not resolve")
+ elif bare:
+ return _identified(direct, raw, [*notes, f"DOI {doi} resolved"])
else:
result = classify([direct], raw)
if result.state in (State.RESOLVED, State.RESOLVED_LOW):
@@ -894,6 +957,20 @@ def resolve(self, raw: str) -> ResolveResult:
notes.append(f"DOI {doi} resolves to a different work")
pool.append(direct)
+ if bare:
+ # Nothing but an identifier, and it did not resolve. The entry has no
+ # title to search by, and the bibliographic matcher's answer to a URL
+ # string is an unrelated paper whose fields cannot agree with a string
+ # that has none -- "nothing agrees" there is not evidence of fabrication.
+ # Crossref also 404s every DataCite DOI (Zenodo, arXiv's own), so an
+ # unresolved identifier is uncertainty, and uncertainty is AMBIGUOUS
+ # (spec section 8, product rule 3).
+ if lookup_down:
+ return ResolveResult(State.UNAVAILABLE, None, [], notes=notes)
+ return ResolveResult(
+ State.AMBIGUOUS, None, [], notes=[*notes, IDENTIFIER_UNRESOLVED_NOTE]
+ )
+
segments = title_segments(raw)[:2]
required_down: list[str] = []
diff --git a/src/proofpath/sarif.py b/src/proofpath/sarif.py
index efacc42..6a23dc4 100644
--- a/src/proofpath/sarif.py
+++ b/src/proofpath/sarif.py
@@ -35,7 +35,7 @@
# ``_one_line`` is private to ``report`` but it is *the* definition of flattening a
# quoted passage onto one line, and the markdown renderer already uses it. Reusing it
# keeps SARIF and markdown quoting the same passage the same way.
-from proofpath.report import LEVELS, Finding, Kind, Report, _one_line
+from proofpath.report import LEVELS, Finding, Kind, Report, _one_line, search_experimental
SARIF_VERSION = "2.1.0"
SCHEMA_URI = "https://json.schemastore.org/sarif-2.1.0.json"
@@ -246,6 +246,8 @@ def _run_properties(report: Report) -> dict[str, Any]:
"evidenceSearch": None
if report.search is None
else {
+ # The banner the other surfaces print at the top of a searched run.
+ "banner": search_experimental(),
"experimental": True,
"by": report.search.by,
"queriesBy": report.search.queries_by,
diff --git a/src/proofpath/search/__init__.py b/src/proofpath/search/__init__.py
index bfb6bef..f8935c4 100644
--- a/src/proofpath/search/__init__.py
+++ b/src/proofpath/search/__init__.py
@@ -8,11 +8,12 @@
from __future__ import annotations
import re
-from collections.abc import Sequence
+from collections.abc import Collection, Sequence
from dataclasses import dataclass
from typing import Protocol
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+from proofpath.polite import ProviderError
from proofpath.providers import is_social
@@ -25,9 +26,18 @@ class SearchHit:
rank: int # 1-based position in the provider's answer
+class SearchKeyError(ProviderError):
+ """The provider refused the key (HTTP 401/403): a credential to fix, not an outage.
+
+ A ``ProviderError`` still, so a caller that knows only outages stops the search
+ as before. The message names the variable to check and never the key.
+ """
+
+
class Searcher(Protocol):
"""A web search provider. ``search`` raises ``polite.ProviderError`` when the
- provider does not answer; an empty list means it answered with nothing."""
+ provider does not answer, ``SearchKeyError`` when it refuses the key; an empty
+ list means it answered with nothing."""
name: str # "tavily" | "searxng": what the Searching stage is attributed to
@@ -58,22 +68,61 @@ def _bare_host(url: str) -> str:
return (urlsplit(url).hostname or "").lower().removeprefix("www.")
-def keep_hits(hits: Sequence[SearchHit], *, own_host: str | None, limit: int) -> list[SearchHit]:
- """The hits worth reading, in order: one per page, no posts, not the document itself.
+def same_site(host: str, site: str) -> bool:
+ """One host is the other, or a subdomain of it, in either direction.
+
+ Both directions, so that excluding ``factcheck.afp.com`` also rules out
+ ``www.afp.com`` and excluding ``afp.com`` rules out ``factcheck.afp.com``. It errs
+ towards reading fewer pages, never towards a page vouching for its own site.
+ """
+ host = host.lower().removeprefix("www.")
+ site = site.lower().removeprefix("www.")
+ if not host or not site:
+ return False
+ return host == site or host.endswith(f".{site}") or site.endswith(f".{host}")
+
+
+def keep_hits(
+ hits: Sequence[SearchHit], *, exclude: Collection[str] = (), limit: int
+) -> list[SearchHit]:
+ """The hits worth reading, in order: one per page, no posts, no excluded site.
A post is dropped because a platform post is a claim, not a source. X, the case
- that brought this feature, cannot be read at all (spec section 3). The document's
- own host is dropped so a page is never found to support itself.
+ that brought this feature, cannot be read at all (spec section 3). ``exclude``
+ holds the document's own host, so a page is never found to support itself, and
+ whatever else a caller must not read (the eval's fact-checking site).
"""
- own = (own_host or "").lower().removeprefix("www.")
kept: list[SearchHit] = []
seen: set[str] = set()
for hit in hits:
key = canonical(hit.url)
- if key in seen or is_social(hit.url) or (own and _bare_host(hit.url) == own):
+ if key in seen or is_social(hit.url):
+ continue
+ if any(same_site(_bare_host(hit.url), site) for site in exclude):
continue
seen.add(key)
kept.append(hit)
if len(kept) >= limit:
break
return kept
+
+
+def search_claim(
+ queries: Sequence[str],
+ searcher: Searcher,
+ results: int,
+ *,
+ exclude: Collection[str] = (),
+) -> list[SearchHit]:
+ """The product's search for one claim: each query asked for ``results`` hits, and
+ at most ``results`` of them kept (final review, Important 3).
+
+ One helper for ``verify`` and for the AVeriTeC eval, so the eval measures the
+ search the product runs rather than a copy of it that drifts. Raises
+ ``polite.ProviderError`` when the provider does not answer; the caller decides
+ what that costs.
+ """
+ hits: list[SearchHit] = []
+ for query in queries:
+ hits.extend(searcher.search(query, results))
+ return keep_hits(hits, exclude=exclude, limit=results)
diff --git a/src/proofpath/search/language.py b/src/proofpath/search/language.py
index 1c02b87..ee856b0 100644
--- a/src/proofpath/search/language.py
+++ b/src/proofpath/search/language.py
@@ -1,10 +1,16 @@
"""Whether a claim reads as English, the only language the models were trained on.
Deliberately small and one-sided: English is the default, and a claim is called
-something else only on positive evidence -- a letter from another script (real
-Turkish, Cyrillic, CJK...), a letter only Turkish writes, or a common function word
-of another language. A wrong "English" costs a weaker check; a wrong "not English"
-would cost the claim its search without a judge, so the doubt goes to English.
+something else only on positive evidence. Decisive on its own: a word in another
+script (Cyrillic, CJK...) or a letter only Turkish writes. Otherwise it takes two
+signals, where a signal is a distinct foreign function word, or -- once, however many
+there are -- a word with an accented Latin letter ("subió", "président"). A wrong
+"English" costs a weaker check (and with a judge, ``certainly_english`` sends the
+claim to its translation anyway); a wrong "not English" would cost the claim its
+search without a judge, so the doubt goes to English.
+
+Capitalised words are never evidence: they are names and acronyms ("MIT", "Le Monde",
+"El Niño", "André", "Uğur Şahin"), and English text is full of foreign ones.
"""
from __future__ import annotations
@@ -12,21 +18,28 @@
import re
import unicodedata
-# Function words of languages a reader is likely to paste, none of them English words.
+# Function words of languages a reader is likely to paste. Only lower-case words are
+# read, so "Los Angeles", "Le Monde" and "Die Hard" never count. Left out on purpose,
+# because English text prints them lower-case too: "et" ("et al."), "de" ("de facto",
+# "de Gaulle"), "da" and "del" ("da Vinci", "del Toro"), "est" (the time zone), "y",
+# "ama", and the particles "della", "du", "den" ("van den Berg").
_FOREIGN_WORDS = frozenset(
{
# Turkish
"ve", "bir", "bu", "için", "ile", "değil", "çok", "daha", "gibi", "olan",
- "olarak", "ama", "şu", "mi", "mı", "mu", "mü", "da", "de", "ki", "yok", "var", # noqa: RUF001 - real Turkish letters
+ "olarak", "şu", "mi", "mı", "mu", "mü", "ki", "yok", "var", # noqa: RUF001 - real Turkish letters
# Spanish / Portuguese / Italian
- "el", "los", "las", "que", "y", "es", "por", "con", "una", "para", "del",
- "não", "uma", "il", "della", "che", "sono",
+ "el", "la", "los", "las", "que", "es", "por", "con", "una", "un", "para", "não",
+ "uma", "il", "ha", "che", "sono",
# German / Dutch
- "der", "das", "und", "ist", "nicht", "ein", "eine", "mit", "den", "het", "een",
+ "der", "die", "das", "und", "mit", "hat", "ist", "nicht", "ein", "eine", "het",
+ "een", "heeft",
# French
- "le", "les", "et", "est", "une", "des", "du", "pas", "pour", "dans",
+ "le", "les", "des", "une", "pas", "pour", "dans", "hier",
}
) # fmt: skip
+# One signal proves nothing: English borrows words ("que sera", "a con", "café").
+_MIN_SIGNALS = 2
_WORDS = re.compile(r"[^\W\d_]+")
# Letters only Turkish writes. ç, ö and ü are not here: they turn up in names
@@ -34,16 +47,47 @@
_TURKISH_ONLY = frozenset("ğĞıİşŞ")
-def _foreign_letter(char: str) -> bool:
- """A letter from another script, or one only Turkish uses. A Latin letter with
- an accent ("André", "Beyoncé") is not evidence: English borrows names."""
- if char in _TURKISH_ONLY:
+def _foreign_word(word: str) -> bool:
+ """A word in another script, or with a letter only Turkish uses.
+
+ A Latin letter with an accent ("André", "Beyoncé") is not evidence: English
+ borrows names. A single letter of another script is not either: it is a symbol
+ ("μ-opioid", "π"), where a sentence in that script has words.
+ """
+ if any(char in _TURKISH_ONLY for char in word):
return True
- return not unicodedata.name(char, "LATIN").startswith("LATIN")
+ return len(word) > 1 and any(
+ not unicodedata.name(char, "LATIN").startswith("LATIN") for char in word
+ )
+
+
+def _accented(word: str) -> bool:
+ """A Latin word with a letter outside ASCII: "subió", "président", "yüzde"."""
+ return any(not char.isascii() for char in word)
def looks_english(text: str) -> bool:
- letters = [char for char in text if char.isalpha()]
- if any(_foreign_letter(char) for char in letters):
+ # Only words that do not start with a capital: a script with no case (CJK) is
+ # never capitalised, so it is always read.
+ words = [word for word in _WORDS.findall(text) if not word[0].isupper()]
+ if any(_foreign_word(word) for word in words):
+ return False
+ signals = len({word.casefold() for word in words} & _FOREIGN_WORDS)
+ signals += any(_accented(word) for word in words)
+ return signals < _MIN_SIGNALS
+
+
+def certainly_english(text: str) -> bool:
+ """English with no doubt at all: no lower-case word with a letter outside ASCII,
+ and no foreign function word, capitalised or not.
+
+ Stricter than :func:`looks_english` on purpose. That one decides whether a claim
+ may be searched with no judge, so the doubt goes to English; this one decides
+ whether a judge's translation may be set aside, so the doubt goes to the judge
+ ("Las vacunas causan autismo." has one capitalised signal and reads as English,
+ but it is not certainly English).
+ """
+ words = _WORDS.findall(text)
+ if any(_accented(word) for word in words if not word[0].isupper()):
return False
- return not any(word.casefold() in _FOREIGN_WORDS for word in _WORDS.findall(text))
+ return not {word.casefold() for word in words} & _FOREIGN_WORDS
diff --git a/src/proofpath/search/providers.py b/src/proofpath/search/providers.py
index c03e91a..93aa00b 100644
--- a/src/proofpath/search/providers.py
+++ b/src/proofpath/search/providers.py
@@ -9,8 +9,8 @@
import httpx
from proofpath.config import SearchConfig
-from proofpath.polite import PoliteClient, ProviderError
-from proofpath.search import Searcher, SearchHit
+from proofpath.polite import PoliteClient, ProviderError, user_agent
+from proofpath.search import Searcher, SearchHit, SearchKeyError
from proofpath.secrets import ApiKey, missing_hint, resolve_api_key
TAVILY_URL = "https://api.tavily.com/search"
@@ -18,31 +18,50 @@
TAVILY_QUERY_LIMIT = 400
_WEB = ("http://", "https://")
SEARXNG_NEEDS_URL = "set search.base_url to your SearXNG instance"
+TAVILY_KEY_ENV = "TAVILY_API_KEY"
+# The statuses Tavily answers a missing, wrong or revoked key with.
+_KEY_STATUSES = frozenset({401, 403})
+# Every search request names the tool and never the user: the run's client carries
+# the contact address in its User-Agent for Crossref's and OpenAlex's polite pools,
+# and a search provider is owed neither it nor a ``mailto`` (final review, minor).
+_HEADERS = {"User-Agent": user_agent()}
class TavilySearcher:
name = "tavily"
- def __init__(self, client: PoliteClient, key: ApiKey) -> None:
+ def __init__(self, client: PoliteClient, key: ApiKey, key_env: str = TAVILY_KEY_ENV) -> None:
self._client = client
self._key = key
+ self._key_env = key_env
def __repr__(self) -> str:
return f"TavilySearcher(key={self._key!r})" # ApiKey's repr never shows the value
def search(self, query: str, max_results: int) -> list[SearchHit]:
- response = self._client.post(
- TAVILY_URL,
- json_body={
- "query": query[:TAVILY_QUERY_LIMIT],
- "max_results": max_results,
- "search_depth": "basic",
- "include_answer": False,
- "include_raw_content": False,
- "include_images": False,
- },
- headers={"Authorization": f"Bearer {self._key.value}"},
- )
+ try:
+ response = self._client.post(
+ TAVILY_URL,
+ json_body={
+ "query": query[:TAVILY_QUERY_LIMIT],
+ "max_results": max_results,
+ "search_depth": "basic",
+ "include_answer": False,
+ "include_raw_content": False,
+ "include_images": False,
+ },
+ headers={**_HEADERS, "Authorization": f"Bearer {self._key.value}"},
+ )
+ except ProviderError as exc:
+ if exc.status in _KEY_STATUSES:
+ # One line of .env fixes it, so it is reported as a credential, not
+ # as a provider that is down (final review, minor).
+ raise SearchKeyError(
+ f"tavily rejected the key (HTTP {exc.status}); check {self._key_env} in .env",
+ exc.status,
+ exc.code,
+ ) from None
+ raise
return _hits(_payload(response, "api.tavily.com").get("results"), max_results)
@@ -56,7 +75,9 @@ def __init__(self, client: PoliteClient, base_url: str) -> None:
def search(self, query: str, max_results: int) -> list[SearchHit]:
# ``format=json`` must be enabled in the instance's settings.yml; an instance
# that has not enabled it answers 403, which is reported, not guessed around.
- response = self._client.get(self._url, {"q": query, "format": "json"}, mailto=False)
+ response = self._client.get(
+ self._url, {"q": query, "format": "json"}, mailto=False, headers=_HEADERS
+ )
host = httpx.URL(self._url).host
return _hits(_payload(response, host).get("results"), max_results)
@@ -118,7 +139,7 @@ def build_searcher(
key = resolve_key(config.api_key_env)
if key is None:
return SearchSetup(None, missing_hint(config.api_key_env))
- return SearchSetup(TavilySearcher(client, key), "")
+ return SearchSetup(TavilySearcher(client, key, config.api_key_env), "")
if config.provider == "searxng":
if not config.base_url:
return SearchSetup(None, SEARXNG_NEEDS_URL)
diff --git a/src/proofpath/search/queries.py b/src/proofpath/search/queries.py
index 7d31f18..c97985a 100644
--- a/src/proofpath/search/queries.py
+++ b/src/proofpath/search/queries.py
@@ -17,7 +17,8 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING
-from proofpath.search.language import looks_english
+from proofpath.judge import LIMIT_STATUSES
+from proofpath.search.language import certainly_english, looks_english
if TYPE_CHECKING:
from proofpath.judge import Judge
@@ -52,8 +53,15 @@ class QueryPlan:
notice: str | None # set when the judge was asked and the sentence was used instead
-def fallback_notice(detail: str) -> str:
- head = LLM_LIMIT if "HTTP 429" in detail else LLM_UNAVAILABLE
+def fallback_notice(detail: str, status: int | None) -> str:
+ """The run's notice for a judge that did not write the queries.
+
+ A limit is named from the failure's status alone, the one rule the judge's own
+ switch notice uses (``LIMIT_STATUSES``). A detail can mention "HTTP 429" without
+ being one -- the primary's reason, quoted before a local model that failed for
+ another reason -- and then no model answered, which is what the notice says.
+ """
+ head = LLM_LIMIT if status in LIMIT_STATUSES else LLM_UNAVAILABLE
return f"{head} ({detail})"
@@ -66,30 +74,70 @@ def plan_queries(texts: Sequence[str], judge: Judge | None) -> QueryPlan:
return QueryPlan(fallback, as_written, SENTENCE_BY, None)
written = judge.queries(texts)
if judge.unavailable:
- return QueryPlan(fallback, as_written, SENTENCE_BY, fallback_notice(judge.detail))
+ return QueryPlan(
+ fallback, as_written, SENTENCE_BY, fallback_notice(judge.detail, judge.status)
+ )
queries = tuple(
(written[i].queries if i in written and written[i].queries else fallback[i])
for i in range(len(texts))
)
hypotheses = tuple(
- texts[i]
- if english[i]
- else (_translation(texts[i], written[i].english) if i in written else None)
+ _hypothesis(texts[i], english[i], written[i].english if i in written else None)
for i in range(len(texts))
)
return QueryPlan(queries, hypotheses, judge.name, None)
+def _hypothesis(text: str, english: bool, translated: str | None) -> str | None:
+ """What the models check for one claim, once a judge has answered.
+
+ The judge's English wins unless the claim is *certainly* English (round 2):
+ ``looks_english`` lets a short foreign sentence through as English ("Las vacunas
+ causan autismo."), and checking it as written would hand Spanish to English-only
+ models. An echo, or no valid translation at all, keeps the claim when it reads as
+ English and leaves nothing to check when it does not.
+ """
+ if translated is not None and not certainly_english(text):
+ checked = _translation(text, translated)
+ if checked is not None and checked.strip() != text.strip():
+ return checked
+ return text if english else None
+
+
+# The worked examples of ``prompts/queries.md``: each example answer's English, and
+# the example claim it translates. A model that copies an example into its answer
+# would otherwise have the example checked as if it were the claim (ledger T11).
+PROMPT_EXAMPLES = {
+ "The company went bankrupt.": "Şirket battı.", # noqa: RUF001 - real Turkish letters
+ "The Eiffel Tower is in Paris.": "The Eiffel Tower is in Paris.",
+}
+
+
+def _folded(text: str) -> str:
+ """One spelling for a comparison: case, spacing and the closing stop ignored."""
+ return " ".join(text.casefold().split()).rstrip(".!?")
+
+
+_EXAMPLES = {_folded(english): _folded(claim) for english, claim in PROMPT_EXAMPLES.items()}
+
+
def _translation(original: str, english: str) -> str | None:
"""The judge's English text, or ``None`` when it is not a translation at all.
A model can copy the claim back unchanged (a local qwen with thinking off does,
- measured 2026-09-28) or answer in the claim's own language. Either one handed to
- the English-only models would be checked as if it were English; ``None`` sends
- the claim down the no-translation path instead (``language not supported``).
+ measured 2026-09-28), answer in the claim's own language, or copy one of the
+ prompt's examples. Any of those handed to the English-only models would be
+ checked as if it were the claim; ``None`` sends the claim down the
+ no-translation path instead (``language not supported``). An echo of a claim
+ that already reads as English is the right answer, and is kept.
"""
if not english:
return None
- if english.strip() == original.strip() or not looks_english(english):
+ if english.strip() == original.strip():
+ return english if looks_english(original) else None
+ if not looks_english(english):
+ return None
+ example = _EXAMPLES.get(_folded(english))
+ if example is not None and example != _folded(original):
return None
return english
diff --git a/src/proofpath/tui/widgets/footer.py b/src/proofpath/tui/widgets/footer.py
index c2fe8ef..26ee22c 100644
--- a/src/proofpath/tui/widgets/footer.py
+++ b/src/proofpath/tui/widgets/footer.py
@@ -169,8 +169,11 @@ def _hints(item: Footer) -> list[str]:
f" — /config set {ui.BROWSER_SETTING}"
)
# The docked footer has one hints row: the experimental banner and the searched
- # line go there, and the full block is in the run's report.
- hints.extend(item.search[:3])
+ # line go there, and the full block is in the run's report. The banner cannot sit
+ # at the top of a docked footer, so it goes first among the search lines.
+ if item.banner is not None:
+ hints.append(item.banner)
+ hints.extend(item.search[:2])
if item.judge_notice:
# The notice above the prompt goes when the next run starts; the footer is
# this run's record, so it keeps saying who gave the second opinion.
diff --git a/src/proofpath/verify.py b/src/proofpath/verify.py
index 46aa8dd..ca3f320 100644
--- a/src/proofpath/verify.py
+++ b/src/proofpath/verify.py
@@ -113,7 +113,7 @@
)
from proofpath.resolve import Resolver, ResolveResult, Retraction, State
from proofpath.retrieval import Embedder, FastEmbedder, PassageIndex
-from proofpath.search import Searcher, SearchHit, canonical, keep_hits
+from proofpath.search import Searcher, SearchKeyError, canonical, search_claim
from proofpath.search.providers import build_searcher
from proofpath.search.queries import plan_queries
from proofpath.secrets import CREDENTIALS_MISSING
@@ -206,7 +206,13 @@
# A claim the models cannot read: not English, and no judge translated it. It is not
# searched, and it is not called NEI either (OPEN-ITEMS 17.1a, Amendment A).
LANGUAGE_TITLE = "the claim is not in English and was not translated"
-LANGUAGE_HINT = "the models read English only; --judge translates a claim before it is searched"
+# True whether or not a judge was on (it may have been down, or echoed the claim), and
+# neutral about where it is run: a TUI run builds no judge, so the hint names the one
+# switch that does, rather than a TUI setting that would not help (round 2).
+LANGUAGE_HINT = (
+ "the models read English only, and no judge translated this claim; run with a "
+ "judge (--judge) to translate a claim before it is searched"
+)
# What a translated claim's finding says was actually checked. After the numeric
# reason, which ``report`` reads back out of ``detail[0]``.
CHECKED_AS = "checked as: "
@@ -992,16 +998,25 @@ def closed(stage: str, by: str, summary: str, began: float) -> None:
continue
# Which family this entry belongs to, decided once and used by all three
# stages below. The core never learns what the answer means (spec 5.2).
- provider = provider_for(reference, engine.providers)
+ # A page the search found is a page: it goes to the web family whatever its
+ # address carries. ``provider_for`` would file a found ``/doi/`` or arXiv
+ # address as a bibliography entry, and the indexes' answer to a bare URL used
+ # to be a false GHOST (final review, C1; product rule 3).
+ found = reference.origin == "search"
+ provider = engine.providers.web if found else provider_for(reference, engine.providers)
chosen[number] = provider
# The cache answers for a reference, not for a source: a reference has no
- # source id until it resolves. A miss costs exactly what it used to.
- stored = None if engine.cache is None else engine.cache.get_resolution(reference.raw)
+ # source id until it resolves. A miss costs exactly what it used to. A found
+ # page skips it both ways: the web family asks nobody, and a stored record of
+ # the same address was about a bibliography entry, not about this page.
+ stored = (
+ None if engine.cache is None or found else engine.cache.get_resolution(reference.raw)
+ )
if stored is None:
result = provider.resolve(reference)
if provider.scheme == INDEXED_FAMILY:
resolved_from_indexes += 1
- if engine.cache is not None:
+ if engine.cache is not None and not found:
# ``put_resolution`` drops UNVERIFIED (provider unavailable) itself:
# an outage is not knowledge about the reference (product rule 2).
engine.cache.put_resolution(reference.raw, result)
@@ -1988,11 +2003,14 @@ def _finding(
source_id: str | None = None,
fetch_step: int | None = None,
detail: tuple[str, ...] = (),
+ claim: Claim | None = None,
) -> Finding:
- """A finding from a stage: never about a claim, so never carrying a verdict.
+ """A finding from a stage: never carrying a verdict.
``state`` defaults to the kind's own word; only the ``UNVERIFIED`` family passes
one, because which flavour applies depends on what failed (product rule 2).
+ ``claim`` is set by the searching stage, whose findings are about one sentence
+ and have no reference to point at: JSON and SARIF readers get the sentence.
"""
return Finding(
kind=kind,
@@ -2001,7 +2019,7 @@ def _finding(
title=title,
state=STATE_WORDS[kind] if state is None else state,
reference=reference,
- claim=None,
+ claim=claim,
verdict=None,
source_id=source_id,
fetch_step=fetch_step,
@@ -2284,15 +2302,31 @@ def _search(
LANGUAGE_TITLE,
state=LANGUAGE_UNSUPPORTED,
detail=(LANGUAGE_HINT,),
+ claim=claim,
)
)
emit(Progress(name=SEARCHING, done=index, total=total, detail=claim.locator.label()))
check()
continue
- hits: list[SearchHit] = []
try:
- for query in queries:
- hits.extend(searcher.search(query, settings.results_per_claim))
+ hits = search_claim(
+ queries, searcher, settings.results_per_claim, exclude=(own,) if own else ()
+ )
+ except SearchKeyError as exc:
+ # The provider answered, and refused the key: the fix is one line of .env,
+ # so it is the credential state a missing key already has, not an outage
+ # (spec 2026-09-28 section 4; final review, minor).
+ add(
+ _finding(
+ Kind.UNVERIFIED,
+ claim.locator,
+ "the evidence search refused the key",
+ state=CREDENTIALS_MISSING,
+ detail=(str(exc),),
+ claim=claim,
+ )
+ )
+ break
except ProviderError as exc:
add(
_finding(
@@ -2301,11 +2335,12 @@ def _search(
"the evidence search did not answer",
state=SEARCH_UNAVAILABLE,
detail=(f"{searcher.name}: {exc}",),
+ claim=claim,
)
)
break
cited: list[int] = []
- for hit in keep_hits(hits, own_host=own, limit=settings.results_per_claim):
+ for hit in hits:
key = canonical(hit.url)
if key not in numbers:
numbers[key] = len(references) + 1
diff --git a/tests/test_cache.py b/tests/test_cache.py
index d65d624..19706bc 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -57,7 +57,7 @@ def test_schema_is_plain_sqlite_readable_without_extensions(db: Cache) -> None:
"retractions",
"judgements",
} <= tables
- assert conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == ("4",)
+ assert conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == ("5",)
def test_chunks_round_trip_with_embeddings(db: Cache) -> None:
@@ -429,7 +429,7 @@ def test_opening_a_v1_file_migrates_it_and_keeps_sources_text_and_verdicts(
with Cache(path) as db:
assert db._conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == (
- "4",
+ "5",
)
columns = {r[1] for r in db._conn.execute("PRAGMA table_info(chunks)")}
assert "text_sha256" in columns
@@ -501,7 +501,7 @@ def broken_step(conn: sqlite3.Connection) -> None:
monkeypatch.undo()
with Cache(path) as db:
assert db._conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == (
- "4",
+ "5",
)
assert db.migrated_from == "1"
assert "text_sha256" in {r[1] for r in db._conn.execute("PRAGMA table_info(chunks)")}
@@ -775,7 +775,7 @@ def test_migrating_a_v3_file_adds_judgements_and_keeps_everything_else(tmp_path:
with Cache(path) as db:
assert db.migrated_from == "3"
assert db._conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == (
- "4",
+ "5",
)
# v4 is additive: the chunks stay, so a warm run stays free.
assert db.get_chunks("s", "bge@rev", text_sha256=RAW_SHA) is not None
@@ -795,3 +795,40 @@ def test_migrating_a_v1_file_runs_the_whole_chain_including_judgements(tmp_path:
db.put_judgement("h", "s", "groq gpt-oss", OPINION, now=NOW)
assert db.get_judgement("h", "s", "groq gpt-oss") == OPINION
assert db.get_verdict("h", "s", "m") is not None # nothing else was lost
+
+
+# --- schema v5: stale ghosts are forgotten (final review, round 2) --------------------
+
+GHOST_REFERENCE = "https://doi.org/10.1021/acs.est.0c01234"
+
+
+def _write_v4_database(path: Path) -> None:
+ """A v4 file holding one GHOST and one RESOLVED resolution.
+
+ v5 changes no table, so a v4 file is today's tables with ``schema_version`` 4.
+ """
+ with Cache(path) as db:
+ db.put_resolution(GHOST_REFERENCE, ResolveResult(State.GHOST, None, []), now=NOW)
+ db.put_resolution(RAW_REFERENCE, RESOLVED, now=NOW)
+ conn = sqlite3.connect(path)
+ with conn:
+ conn.execute("UPDATE meta SET value = '4' WHERE key = 'schema_version'")
+ conn.close()
+
+
+def test_migrating_a_v4_file_forgets_every_cached_ghost_and_keeps_the_rest(
+ tmp_path: Path,
+) -> None:
+ """A GHOST cached before the identifier-only rule (final review, C1) would be
+ served for a month; a ghost is the one verdict rule 3 cannot let stand on an old
+ rule, so v5 drops them and they are resolved again."""
+ path = tmp_path / "proofpath.sqlite3"
+ _write_v4_database(path)
+
+ with Cache(path) as db:
+ assert db.migrated_from == "4"
+ assert db._conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone() == (
+ "5",
+ )
+ assert db.get_resolution(GHOST_REFERENCE, now=NOW) is None
+ assert db.get_resolution(RAW_REFERENCE, now=NOW) == RESOLVED
diff --git a/tests/test_check_cli.py b/tests/test_check_cli.py
index bf3c5a6..ebe9d7e 100644
--- a/tests/test_check_cli.py
+++ b/tests/test_check_cli.py
@@ -108,6 +108,7 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched:
return self.pages.get(url, unreachable(url))
diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py
index 73de263..d54824e 100644
--- a/tests/test_config_cli.py
+++ b/tests/test_config_cli.py
@@ -141,3 +141,28 @@ def test_old_top_level_groups_are_gone(config_dir: Path, group: str) -> None:
result = runner.invoke(app, [group])
assert result.exit_code == 2
assert "No such command" in result.output
+
+
+# --- evidence search settings (ledger T1, final review) --------------------------------
+
+
+def test_config_renders_the_web_search_permission(config_dir: Path) -> None:
+ result = runner.invoke(app, ["config"])
+ assert result.exit_code == 0, result.output
+ assert 'web_search = "allow"' in result.stdout.splitlines()
+
+
+def test_the_search_provider_values_and_the_hinted_settings_are_the_real_ones(
+ config_dir: Path,
+) -> None:
+ from proofpath.config import SEARCH_PROVIDERS
+ from proofpath.settings_hints import SEARCH_SETTING, SEARXNG_SETTING
+
+ assert SEARCH_PROVIDERS == ("off", "tavily", "searxng")
+ assert SEARCH_SETTING == "search.provider tavily"
+ assert SEARXNG_SETTING == "search.provider searxng"
+ # The hints are commands a reader types, so each one has to be accepted as one.
+ for setting in (SEARCH_SETTING, SEARXNG_SETTING):
+ result = runner.invoke(app, ["config", "set", *setting.split()])
+ assert result.exit_code == 0, result.output
+ assert result.stdout.startswith(f"{setting.replace(' ', ' = ', 1)} (")
diff --git a/tests/test_eval_averitec.py b/tests/test_eval_averitec.py
index 038d93e..2239931 100644
--- a/tests/test_eval_averitec.py
+++ b/tests/test_eval_averitec.py
@@ -254,62 +254,101 @@ def test_the_report_ends_with_a_newline() -> None:
assert render_report(score(ROWS), date="2026-09-15", limit=100).endswith("\n")
-# --- search_urls -----------------------------------------------------------
+# --- search_urls: the product's search, gate and all ------------------------
-def test_search_urls_never_hand_back_the_fact_check_itself() -> None:
- class Stub:
- name = "stub"
+class _Stub:
+ name = "stub"
+
+ def __init__(self, urls: list[str], *, error: Exception | None = None) -> None:
+ self.urls = urls
+ self.error = error
+ self.asked: list[tuple[str, int]] = []
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ self.asked.append((query, max_results))
+ if self.error is not None:
+ raise self.error
+ return [SearchHit(url, "", rank) for rank, url in enumerate(self.urls, start=1)]
+
- def search(self, query: str, max_results: int) -> list[SearchHit]:
- return [
- SearchHit("https://factcheck.test/claim-1", "", 1),
- SearchHit("https://factcheck.test/other", "", 2),
- SearchHit("https://news.test/a", "", 3),
- ]
+def _claim(
+ text: str = "The senator voted against the bill in 2019.", **fields: object
+) -> averitec.Claim:
+ return averitec.Claim(id=0, text=text, label="Refuted", source_urls=(), **fields) # type: ignore[arg-type]
- claim = averitec.Claim(
- id=0, text="A claim.", label="Refuted", source_urls=(),
- fact_check_url="https://factcheck.test/claim-1",
- ) # fmt: skip
- assert search_urls(Stub(), claim, 3) == ("https://news.test/a",)
+
+def test_search_urls_never_hand_back_the_fact_check_itself() -> None:
+ stub = _Stub(
+ ["https://factcheck.test/claim-1", "https://factcheck.test/other", "https://news.test/a"]
+ )
+ claim = _claim(fact_check_url="https://factcheck.test/claim-1")
+ assert search_urls(stub, claim, 3).urls == ("https://news.test/a",)
def test_search_urls_exclude_the_checker_across_subdomains() -> None:
- class Stub:
- name = "stub"
-
- def search(self, query: str, max_results: int) -> list[SearchHit]:
- return [
- SearchHit("https://www.afp.com/en/story", "", 1),
- SearchHit("https://factcheck.afp.com/other", "", 2),
- SearchHit("https://news.afp.com.evil.test/a", "", 3),
- SearchHit("https://notafp.com/b", "", 4),
- ]
-
- claim = averitec.Claim(
- id=0, text="A claim.", label="Refuted", source_urls=(),
- fact_check_url="https://factcheck.afp.com/claim-1",
- ) # fmt: skip
- assert search_urls(Stub(), claim, 3) == (
+ stub = _Stub(
+ [
+ "https://www.afp.com/en/story",
+ "https://factcheck.afp.com/other",
+ "https://news.afp.com.evil.test/a",
+ "https://notafp.com/b",
+ ]
+ )
+ claim = _claim(fact_check_url="https://factcheck.afp.com/claim-1")
+ assert search_urls(stub, claim, 3).urls == (
"https://news.afp.com.evil.test/a",
"https://notafp.com/b",
)
-def test_search_urls_ask_for_enough_hits_to_fill_the_limit() -> None:
- asked: list[int] = []
+def test_search_urls_ask_for_exactly_the_products_share_of_hits() -> None:
+ """Final review, Important 3: the product asks for ``results_per_claim``, and so
+ does the eval -- no over-fetch the product never makes."""
+ stub = _Stub([])
+ searched = search_urls(stub, _claim(), 3)
+ assert [asked for _, asked in stub.asked] == [3]
+ assert searched.state is None
+
+
+def test_search_urls_query_with_the_products_sentence_query() -> None:
+ stub = _Stub([])
+ search_urls(stub, _claim("The senator voted against the #bill in 2019."), 3)
+ assert [query for query, _ in stub.asked] == ["The senator voted against the bill in 2019."]
+
+
+RUSSIAN = "Путин подписал закон о выборах." # noqa: RUF001 - real Cyrillic letters
+
+
+def test_a_claim_the_product_would_not_search_is_scored_not_dropped() -> None:
+ stub = _Stub(["https://news.test/a"])
+ searched = search_urls(stub, _claim(RUSSIAN), 3)
+ assert searched.state == eval_averitec.LANGUAGE_UNSUPPORTED
+ assert stub.asked == [] # never searched in a language the models cannot read
+ row = eval_averitec.unsearched_row(_claim(RUSSIAN), searched.state)
+ assert row.predicted is None and row.states == (eval_averitec.LANGUAGE_UNSUPPORTED,)
+ assert score([row]).n == 1 # counted, as NEI
+
- class Stub:
- name = "stub"
+def test_a_claim_with_no_sentence_to_search_is_reported_as_not_searched() -> None:
+ searched = search_urls(_Stub([]), _claim("???"), 3)
+ assert searched.state == eval_averitec.NOT_SEARCHED
- def search(self, query: str, max_results: int) -> list[SearchHit]:
- asked.append(max_results)
- return []
- claim = averitec.Claim(id=0, text="A claim.", label="Refuted", source_urls=())
- search_urls(Stub(), claim, 3)
- assert asked == [9]
+def test_a_provider_error_is_recorded_as_unverified_for_that_claim_only() -> None:
+ from proofpath.polite import ProviderError
+
+ searched = search_urls(_Stub([], error=ProviderError("HTTP 432", 432, None)), _claim(), 3)
+ assert searched.state == eval_averitec.SEARCH_UNAVAILABLE
+ assert search_urls(_Stub(["https://news.test/a"]), _claim(), 3).urls == ("https://news.test/a",)
+
+
+def test_the_report_states_the_measured_path() -> None:
+ assert eval_averitec.measured_path(None) == "sentence queries, no judge"
+ text = render_report(
+ score(ROWS), date="2026-09-29", limit=100, path=eval_averitec.measured_path(None)
+ )
+ assert "measured path: sentence queries, no judge" in text
# --- main ----------------------------------------------------------------
@@ -336,3 +375,39 @@ def test_a_fresh_run_refuses_to_overwrite_an_existing_results_file(
assert "--resume" in message
assert str(results) in message
assert results.read_text(encoding="utf-8") == "[]"
+
+
+def test_a_rejected_search_key_stops_the_run_at_the_first_claim(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+ """Round 2: a key the provider refuses would fail every claim the same way; the
+ run says so once and exits 2 instead of scoring a hundred claims unverified."""
+ from proofpath.config import Config
+ from proofpath.search import SearchKeyError
+
+ stub = _Stub(
+ [], error=SearchKeyError("tavily rejected the key (HTTP 401); check TAVILY_API_KEY", 401)
+ )
+ closed: list[bool] = []
+ engine = SimpleNamespace(
+ searcher=stub,
+ search_problem="",
+ judge=None,
+ escalate=True,
+ close=lambda: closed.append(True),
+ )
+ claims = [_claim(f"The senator voted against the bill in 201{n}.") for n in range(3)]
+ monkeypatch.setattr(eval_averitec, "cache_dir", lambda: tmp_path)
+ monkeypatch.setattr(eval_averitec.averitec, "ensure_downloaded", lambda _cache: tmp_path)
+ monkeypatch.setattr(eval_averitec.averitec, "load", lambda _path, limit=None: claims)
+ monkeypatch.setattr(eval_averitec, "load_config", Config)
+ monkeypatch.setattr(eval_averitec.Engine, "default", staticmethod(lambda *a, **k: engine))
+ out = tmp_path / "report.md"
+
+ assert eval_averitec.main(["--search", "--out", str(out)]) == 2
+
+ assert len(stub.asked) == 1 # the first claim, and no retry on the others
+ assert closed == [True]
+ assert not out.exists()
+ message = capsys.readouterr().err
+ assert "TAVILY_API_KEY" in message and "rejected" in message
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 791da07..a354148 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -925,3 +925,67 @@ def test_throttle_spaces_requests_to_one_host(client: httpx.Client, clock: list[
fetcher.fetch("https://x.test/a")
fetcher.fetch("https://x.test/b")
assert clock == [pytest.approx(DEFAULT_MIN_INTERVAL)]
+
+
+# --- a page nobody chose is fetched without the contact address (round 2) -------------
+
+CONTACT = "someone@example.org"
+
+
+def _contact_fetcher(*, curl: fx.CurlGet | None = curl_forbidden) -> fx.Fetcher:
+ """The run's fetcher as ``Engine.default`` builds it: its own client, whose
+ User-Agent carries the contact address, and robots.txt respected."""
+ from proofpath.config import Contact
+
+ config = Config(contact=Contact(email=CONTACT), fetch=FetchConfig(respect_robots=True))
+ return fx.Fetcher(config=config, curl_get=curl, wayback_lookup=None, interactive=False)
+
+
+def _agents(*routes: Any) -> list[str]:
+ return [call.request.headers["user-agent"] for route in routes for call in route.calls]
+
+
+@respx.mock
+def test_an_anonymous_fetch_sends_no_contact_address_on_any_request() -> None:
+ robots = respx.get("https://x.test/robots.txt").mock(return_value=httpx.Response(404))
+ page = serve()
+ fetcher = _contact_fetcher()
+ assert fetcher.fetch(PAGE, anonymous=True).outcome is fx.Outcome.OK
+ fetcher.close()
+ agents = _agents(robots, page)
+ assert len(agents) == 2
+ assert all(CONTACT not in agent and "mailto" not in agent for agent in agents)
+ assert all(agent == user_agent() for agent in agents)
+
+
+@respx.mock
+def test_a_cited_source_keeps_the_contact_address() -> None:
+ robots = respx.get("https://x.test/robots.txt").mock(return_value=httpx.Response(404))
+ page = serve()
+ fetcher = _contact_fetcher()
+ fetcher.fetch(PAGE)
+ fetcher.close()
+ assert all(agent == user_agent(CONTACT) for agent in _agents(robots, page))
+
+
+@respx.mock
+def test_an_anonymous_fetch_keeps_the_address_off_curl_and_the_wayback_lookup(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ respx.get("https://x.test/robots.txt").mock(return_value=httpx.Response(404))
+ serve(status=403, body=CHALLENGE)
+ lookup = respx.get(fx.WAYBACK_AVAILABLE).mock(
+ return_value=httpx.Response(200, json={"archived_snapshots": {}})
+ )
+ emails: list[str] = []
+
+ def fake_curl(url: str, *, timeout: float = 20.0, contact_email: str = "") -> Any:
+ emails.append(contact_email)
+ return 403, CHALLENGE, "text/html", url
+
+ monkeypatch.setattr(fx, "default_curl_get", fake_curl)
+ fetcher = _contact_fetcher(curl=None)
+ fetcher.fetch(PAGE, anonymous=True)
+ fetcher.close()
+ assert emails == [""]
+ assert _agents(lookup) == [user_agent()]
diff --git a/tests/test_judge.py b/tests/test_judge.py
index e7c6e6d..e234945 100644
--- a/tests/test_judge.py
+++ b/tests/test_judge.py
@@ -983,3 +983,34 @@ def test_known_providers_are_the_provider_table_sorted() -> None:
accepts, in the order its own error message lists them."""
assert judge.known_providers() == ("gemini", "groq", "ollama")
assert judge.known_providers() == tuple(sorted(judge._PROVIDERS))
+
+
+# --- the queries budget (final review, minor) ------------------------------------------
+
+
+@respx.mock
+def test_queries_leave_a_reasoning_model_room_to_think_and_answer_five_claims() -> None:
+ """``gpt-oss-120b`` charges its thinking to ``max_tokens``. Five claims answered in
+ full -- an English text and two queries each -- are ~170 tokens apiece, so 1500
+ left a low-effort reasoning pass ~650 tokens before ``finish_reason=length``
+ cost every query and every translation."""
+ route = respx.post(OLLAMA_URL).mock(return_value=_ok('{"items": []}'))
+ asker = judge.Judge(judge.JudgeClient(PASSTHROUGH, None))
+ claims = ["x" * 300] * 5
+ asker.queries(claims)
+
+ sent = json.loads(route.calls.last.request.content)
+ one_answer = judge.estimate_tokens(
+ json.dumps({"id": 4, "english": "x" * 300, "queries": ["y" * 150, "y" * 150]})
+ )
+ assert sent["max_tokens"] >= 5 * one_answer + judge._QUERIES_REASONING_TOKENS
+ assert sent["reasoning_effort"] == "low"
+ # Prompt and answer budget together still fit a free tier's 8k tokens a minute.
+ prompt = sum(judge.estimate_tokens(m["content"]) for m in sent["messages"])
+ assert prompt + sent["max_tokens"] <= 8000
+ asker.close()
+
+
+def test_the_queries_budget_grows_with_the_batch_and_stays_under_its_ceiling() -> None:
+ assert judge._queries_budget(1) < judge._queries_budget(5)
+ assert judge._queries_budget(1000) == judge._QUERIES_MAX_TOKENS
diff --git a/tests/test_judge_fallback.py b/tests/test_judge_fallback.py
index 000b193..248dcd0 100644
--- a/tests/test_judge_fallback.py
+++ b/tests/test_judge_fallback.py
@@ -573,7 +573,7 @@ def test_the_model_is_never_pulled(mock: respx.MockRouter) -> None:
def test_a_local_model_that_fails_too_keeps_the_primarys_reason(mock: respx.MockRouter) -> None:
- """The search's ``fallback_notice`` reads ``HTTP 429`` out of this detail."""
+ """The primary's reason comes first in the detail: it is what the run left."""
routes(mock, local=httpx.Response(404))
client = fallback()
with pytest.raises(judge.JudgeUnavailable) as caught:
@@ -857,3 +857,64 @@ def test_an_orphan_closing_tag_before_prose_is_not_stripped() -> None:
def test_an_orphan_closing_tag_before_json_is_stripped() -> None:
assert judge._strip_think('reasoning {a}\n\n{"items": []}') == '{"items": []}'
assert judge._strip_think('r {"items": []}') == '{"items": []}'
+
+
+# --- both failed (final review, minor: stale notice) ---------------------------------
+
+BOTH_FAILED = f"Groq limit reached, and local ollama {LOCAL} failed too (HTTP 404)"
+
+
+def test_a_local_model_that_fails_too_is_never_reported_as_judging(
+ mock: respx.MockRouter,
+) -> None:
+ routes(mock, local=httpx.Response(404))
+ client = fallback()
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ assert client.switched == BOTH_FAILED
+ assert "judging with" not in client.switched
+
+
+def test_a_local_model_that_answers_again_is_reported_as_judging_again(
+ mock: respx.MockRouter,
+) -> None:
+ # A 400 fails one request, not the model (fix round 2), so the next can succeed.
+ _, _, _, local = routes(mock)
+ local.mock(side_effect=[httpx.Response(400), _ok(OPINION)])
+ client = fallback()
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+ assert client.switched is not None and "failed too" in client.switched
+ client.complete(MESSAGES)
+ assert client.switched == LIMIT
+
+
+def test_a_run_whose_local_model_fails_too_says_both_failed_in_the_report(
+ mock: respx.MockRouter,
+) -> None:
+ routes(mock, local=httpx.Response(404))
+ built = searching(StubSearcher())
+ built.judge = judge.Judge(judge.build_client(GROQ, KEY))
+ report = verify_mod.verify(CLAIM, built)
+ assert report.judge_notice == BOTH_FAILED
+ assert f"- judge fallback: {BOTH_FAILED}" in render_markdown(report)
+
+
+# --- one limit classification (final review, minor) ------------------------------------
+
+
+def test_judge_queries_keeps_the_status_of_the_failure() -> None:
+ class Down:
+ provider = "groq"
+ model = "m"
+ cost = judge.JudgeCost()
+
+ def complete(self, *_: object, **__: object) -> judge.Completion:
+ raise judge.JudgeUnavailable("HTTP 413 from https://api.test", status=413)
+
+ def close(self) -> None:
+ pass
+
+ built = judge.Judge(Down()) # type: ignore[arg-type]
+ assert built.queries(["a claim"]) == {}
+ assert built.unavailable and built.status == 413
diff --git a/tests/test_providers.py b/tests/test_providers.py
index 6b327b2..b83dedd 100644
--- a/tests/test_providers.py
+++ b/tests/test_providers.py
@@ -101,6 +101,7 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched:
self.calls.append(url)
return self.page if self.page is not None else unreachable(url)
diff --git a/tests/test_report.py b/tests/test_report.py
index f4fc0df..9d6d398 100644
--- a/tests/test_report.py
+++ b/tests/test_report.py
@@ -455,12 +455,21 @@ def test_exit_code_is_zero_for_a_clean_run() -> None:
assert report_with().exit_code() == 0
-@pytest.mark.parametrize("kind", list(Kind))
+@pytest.mark.parametrize("kind", [kind for kind in Kind if kind is not Kind.EVIDENCE_FOUND])
def test_exit_code_is_one_for_every_state(kind: Kind) -> None:
# Spec section 13.3: every UNVERIFIED and LOW CONFIDENCE state counts, not just errors.
assert report_with(finding(kind, line=1)).exit_code() == 1
+def test_a_supported_found_page_alone_exits_zero_like_a_supported_cited_claim() -> None:
+ # Final review: the spec is silent on search, and SUPPORTED (found by proofpath)
+ # is the searched run's clean answer, shown where a cited one is silent.
+ assert report_with(finding(Kind.EVIDENCE_FOUND, line=1)).exit_code() == 0
+ mixed = report_with(finding(Kind.EVIDENCE_FOUND, line=1), finding(Kind.NO_EVIDENCE, line=2))
+ assert mixed.exit_code() == 1
+ assert report_with(finding(Kind.EVIDENCE_FOUND, line=1), cancelled=True).exit_code() == 1
+
+
def test_exit_code_is_one_when_cancelled() -> None:
assert report_with(cancelled=True).exit_code() == 1
@@ -1651,7 +1660,9 @@ def test_language_unsupported_is_worded_as_the_unverified_family() -> None:
def test_search_lines_state_what_was_searched_and_what_was_not() -> None:
lines = search_lines(dataclasses.replace(report_with(), search=SUMMARY))
- assert lines[0] == search_experimental()
+ # The banner opens the report instead (spec 2026-09-28 section 5; final review).
+ assert search_experimental() not in lines
+ assert lines[0] == "evidence search by tavily, queries by sentence"
assert "evidence search by tavily, queries by sentence" in lines
assert "claims searched 3 of 4 check-worthy (7 sentences)" in lines
assert "claims not searched 4" in lines
@@ -1670,3 +1681,13 @@ def test_the_footer_and_the_markdown_carry_the_search_block() -> None:
markdown = render_markdown(report)
assert "## Evidence search" in markdown
assert "- claims not searched 4" in markdown
+
+
+def test_the_banner_opens_a_searched_markdown_report_and_nothing_else_carries_it() -> None:
+ report = dataclasses.replace(report_with(), search=SUMMARY)
+ lines = render_markdown(report).splitlines()
+ assert lines[2] == f"> **{search_experimental()}**"
+ assert sum(search_experimental() in line for line in lines) == 1
+ assert render_footer(report).banner == search_experimental()
+ assert render_footer(report_with()).banner is None
+ assert search_experimental() not in render_markdown(report_with())
diff --git a/tests/test_resolve.py b/tests/test_resolve.py
index a02fbf7..6c5a907 100644
--- a/tests/test_resolve.py
+++ b/tests/test_resolve.py
@@ -1184,3 +1184,109 @@ def test_a_non_json_retraction_body_leaves_the_check_unmade() -> None:
)
with pytest.raises(rs.ProviderError):
rs.Resolver(retries=0).retraction(NUMPY_DOI)
+
+
+# --- an entry that is nothing but an identifier (final review, Critical 1) ---------
+
+# The three addresses the final review reproduced a false ghost with. As entries they
+# carry no title, no author and no year: the identifier is the whole reference.
+ACS_DOI = "10.1021/acs.est.0c01234"
+ACS_PUBLISHER_URL = f"https://pubs.acs.org/doi/{ACS_DOI}"
+ACS_DOI_URL = f"https://doi.org/{ACS_DOI}"
+ARXIV_ABS_URL = "https://arxiv.org/abs/2106.09685"
+
+
+@pytest.mark.parametrize(
+ "raw",
+ [
+ ACS_PUBLISHER_URL,
+ ACS_DOI_URL,
+ ARXIV_ABS_URL,
+ f"[4] {ACS_DOI_URL}",
+ f"doi:{ACS_DOI}",
+ ACS_DOI,
+ "arXiv:2106.09685",
+ "https://arxiv.org/pdf/2106.09685v2",
+ # Round 2: a labelled address, and an address in angle brackets.
+ f"DOI: {ACS_DOI_URL}",
+ f"doi: {ACS_DOI_URL}.",
+ f"<{ACS_DOI_URL}>",
+ f"<{ARXIV_ABS_URL}>.",
+ f"[4] <{ACS_PUBLISHER_URL}>",
+ ],
+)
+def test_an_entry_that_is_only_an_identifier_is_recognised(raw: str) -> None:
+ assert rs.identifier_only(raw)
+
+
+@pytest.mark.parametrize(
+ "raw",
+ [
+ ALPHAFOLD,
+ ALPHAFOLD + " https://doi.org/10.1038/s41586-021-03819-2",
+ "https://www.nature.com/articles/s41586-021-03819-2",
+ "https://who.int/report.html",
+ f"See {ACS_DOI_URL} for the data",
+ "doi.org/10.1021/acs.est.0c01234 and a title",
+ f"<{ACS_DOI_URL}> Smith J. A title. 2020.",
+ ],
+)
+def test_an_entry_with_anything_beside_its_identifier_is_not_identifier_only(raw: str) -> None:
+ assert not rs.identifier_only(raw)
+
+
+def _crossref_work(doi: str, fixture_name: str = "crossref_work_numpy.json") -> respx.Route:
+ return respx.get(f"https://api.crossref.org/works/{doi}").mock(
+ return_value=httpx.Response(200, json=fixture(fixture_name))
+ )
+
+
+@respx.mock
+@pytest.mark.parametrize(
+ "raw",
+ [
+ ACS_PUBLISHER_URL,
+ ACS_DOI_URL,
+ f"[4] {ACS_DOI_URL}",
+ f"DOI: {ACS_DOI_URL}",
+ f"<{ACS_DOI_URL}>",
+ ],
+)
+def test_a_bare_doi_address_that_resolves_is_that_work_never_a_ghost(raw: str) -> None:
+ # The record's title is nowhere in the entry, because the entry is only the
+ # address. Resolving to *some* work is not evidence of fabrication (rule 3).
+ _crossref_work(ACS_DOI)
+ _mock()
+ result = rs.Resolver().resolve(raw)
+ assert result.state is rs.State.RESOLVED_LOW
+ assert result.best is not None and result.best.provider == "doi"
+ assert any("only an identifier" in note for note in result.notes)
+
+
+@respx.mock
+def test_a_bare_arxiv_address_that_resolves_is_that_work_never_a_ghost() -> None:
+ _mock(arxiv="arxiv_id_roberta.xml")
+ result = rs.Resolver().resolve(ARXIV_ABS_URL)
+ assert result.state is rs.State.RESOLVED_LOW
+ assert result.best is not None and result.best.provider == "arxiv"
+
+
+@respx.mock
+@pytest.mark.parametrize("raw", [ACS_PUBLISHER_URL, ACS_DOI_URL, ARXIV_ABS_URL])
+def test_a_bare_identifier_that_does_not_resolve_is_ambiguous_never_a_ghost(raw: str) -> None:
+ # Crossref only knows Crossref DOIs: a DataCite DOI (Zenodo, arXiv's own) is a 404
+ # there. With nothing but the identifier, there is nothing else to search by, and
+ # the matcher's answer to a URL string is noise (spec section 8).
+ respx.get(f"https://api.crossref.org/works/{ACS_DOI}").mock(return_value=httpx.Response(404))
+ routes = _mock()
+ result = rs.Resolver().resolve(raw)
+ assert result.state is rs.State.AMBIGUOUS
+ assert not routes["crossref"].called # the bibliographic matcher is never fed a URL
+
+
+@respx.mock
+def test_a_bare_identifier_whose_lookup_is_down_is_unavailable() -> None:
+ respx.get(f"https://api.crossref.org/works/{ACS_DOI}").mock(return_value=httpx.Response(503))
+ _mock()
+ result = rs.Resolver(retries=0).resolve(ACS_DOI_URL)
+ assert result.state is rs.State.UNAVAILABLE
diff --git a/tests/test_sarif.py b/tests/test_sarif.py
index 84d07fb..28eb6dc 100644
--- a/tests/test_sarif.py
+++ b/tests/test_sarif.py
@@ -423,3 +423,5 @@ def test_a_search_run_states_its_search_in_the_run_properties() -> None:
props = to_sarif(report, artifact="draft.md")["runs"][0]["properties"]
assert props["evidenceSearch"]["pagesRead"] == 1
assert props["evidenceSearch"]["experimental"] is True
+ assert next(iter(props["evidenceSearch"])) == "banner"
+ assert props["evidenceSearch"]["banner"].startswith("evidence search is experimental")
diff --git a/tests/test_search.py b/tests/test_search.py
index d443d6b..809943a 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -13,7 +13,7 @@
from proofpath import polite as pl
from proofpath.config import SearchConfig
from proofpath.polite import ProviderError
-from proofpath.search import SearchHit, canonical, keep_hits
+from proofpath.search import SearchHit, SearchKeyError, canonical, keep_hits, search_claim
from proofpath.search.providers import (
TAVILY_QUERY_LIMIT,
TAVILY_URL,
@@ -110,7 +110,7 @@ def test_keep_hits_dedupes_and_drops_posts_and_the_documents_own_host() -> None:
SearchHit("https://wiki.test/A", "", 5),
SearchHit("https://other.test/B", "", 6),
]
- kept = keep_hits(hits, own_host="blog.test", limit=2)
+ kept = keep_hits(hits, exclude=("blog.test",), limit=2)
assert [hit.url for hit in kept] == ["https://news.test/a?utm_source=x", "https://wiki.test/A"]
@@ -142,3 +142,100 @@ def test_build_searcher_needs_a_searxng_address() -> None:
)
built = build_searcher(SearchConfig(provider="searxng", base_url=SEARXNG), client())
assert built.searcher is not None and built.searcher.name == "searxng"
+
+
+class _Answers:
+ """A searcher that answers every query with the same hits, and records what it was
+ asked for."""
+
+ name = "stub"
+
+ def __init__(self, urls: list[str]) -> None:
+ self.urls = urls
+ self.asked: list[tuple[str, int]] = []
+
+ def search(self, query: str, max_results: int) -> list[SearchHit]:
+ self.asked.append((query, max_results))
+ return [SearchHit(url, "", rank) for rank, url in enumerate(self.urls, start=1)]
+
+
+def test_search_claim_asks_every_query_for_exactly_the_results_it_keeps() -> None:
+ """Final review, Important 3: one helper for the product and the eval, so the eval
+ measures the product's search and not an over-fetching copy of it."""
+ searcher = _Answers(["https://news.test/a", "https://wiki.test/b", "https://c.test/c"])
+ kept = search_claim(("first query", "second query"), searcher, 2)
+ assert searcher.asked == [("first query", 2), ("second query", 2)]
+ assert [hit.url for hit in kept] == ["https://news.test/a", "https://wiki.test/b"]
+
+
+def test_search_claim_excludes_a_site_across_its_subdomains_both_ways() -> None:
+ searcher = _Answers(
+ [
+ "https://www.afp.com/en/story",
+ "https://factcheck.afp.com/other",
+ "https://news.afp.com.evil.test/a",
+ "https://notafp.com/b",
+ ]
+ )
+ kept = search_claim(("q",), searcher, 3, exclude=("factcheck.afp.com",))
+ assert [hit.url for hit in kept] == ["https://news.afp.com.evil.test/a", "https://notafp.com/b"]
+
+
+# --- no contact email reaches a search provider (final review, minor) ----------------
+
+EMAIL = "someone@example.org"
+
+
+def _emailing_client() -> pl.PoliteClient:
+ """The run's client, as ``Engine.default`` builds it with a contact address: the
+ address is for Crossref and OpenAlex's polite pools, not for a search provider."""
+ return pl.PoliteClient(contact_email=EMAIL, throttle=pl.HostThrottle(), retries=0)
+
+
+def _leaks(request: httpx.Request) -> bool:
+ headers = " ".join(f"{name}: {value}" for name, value in request.headers.items())
+ return "example.org" in f"{request.url} {headers} {request.content.decode('utf-8')}"
+
+
+@respx.mock
+def test_tavily_never_sends_the_contact_email() -> None:
+ route = respx.post(TAVILY_URL).mock(return_value=httpx.Response(200, json={"results": []}))
+ TavilySearcher(_emailing_client(), ApiKey("k", source="test")).search("q", 3)
+ request = route.calls.last.request
+ assert not _leaks(request)
+ assert "proofpath" in request.headers["user-agent"]
+
+
+@respx.mock
+def test_searxng_never_sends_the_contact_email() -> None:
+ route = respx.get(f"{SEARXNG}/search").mock(
+ return_value=httpx.Response(200, json={"results": []})
+ )
+ SearxngSearcher(_emailing_client(), SEARXNG).search("q", 3)
+ request = route.calls.last.request
+ assert not _leaks(request)
+ assert "proofpath" in request.headers["user-agent"]
+
+
+# --- a rejected Tavily key is a credential problem (final review, minor) ---------------
+
+
+@respx.mock
+@pytest.mark.parametrize("status", [401, 403])
+def test_a_rejected_tavily_key_is_a_rejected_key_not_an_outage(status: int) -> None:
+ respx.post(TAVILY_URL).mock(return_value=httpx.Response(status, json={"detail": "tvly-x"}))
+ with pytest.raises(SearchKeyError) as caught:
+ TavilySearcher(client(), ApiKey("tvly-x", source="test")).search("q", 3)
+ text = str(caught.value)
+ assert "TAVILY_API_KEY" in text and f"HTTP {status}" in text
+ assert "tvly-x" not in text and "tvly-x" not in repr(caught.value)
+ assert isinstance(caught.value, ProviderError) # a caller that only knows outages
+
+
+@respx.mock
+def test_a_searxng_403_stays_an_outage() -> None:
+ """SearXNG answers 403 when ``format=json`` is off: no key is involved."""
+ respx.get(f"{SEARXNG}/search").mock(return_value=httpx.Response(403))
+ with pytest.raises(ProviderError) as caught:
+ SearxngSearcher(client(), SEARXNG).search("q", 3)
+ assert not isinstance(caught.value, SearchKeyError)
diff --git a/tests/test_search_queries.py b/tests/test_search_queries.py
index af2bd17..3a6cf74 100644
--- a/tests/test_search_queries.py
+++ b/tests/test_search_queries.py
@@ -83,7 +83,7 @@ def test_the_judge_writes_decontextualised_queries() -> None:
def test_a_rate_limited_judge_falls_back_to_the_sentence_and_says_so() -> None:
plan = plan_queries(
["ChatGPT was shut down in 2025."],
- judge(JudgeUnavailable("HTTP 429 from https://api.test/v1/chat/completions")),
+ judge(JudgeUnavailable("HTTP 429 from https://api.test/v1/chat/completions", status=429)),
)
assert plan.queries == (("ChatGPT was shut down in 2025.",),)
assert plan.by == SENTENCE_BY
@@ -119,9 +119,10 @@ def test_judge_queries_never_raises_on_an_unreadable_answer() -> None:
("OpenAI shut down ChatGPT in 2025.", True),
("NASA confirms water on Mars", True),
("OpenAI battı.", False), # noqa: RUF001 - real Turkish letters
- ("OpenAI batti ve kapandi.", False), # ASCII Turkish, "ve"
- ("El gobierno cerró la empresa.", False),
- ("Die Firma ist pleite.", False), # "ist"
+ # ASCII cases carry two function words: one is no longer evidence (final review).
+ ("OpenAI batti ve bu kapandi.", False), # ASCII Turkish, "ve" and "bu"
+ ("El gobierno cerró la empresa por una deuda.", False), # "por", "una"
+ ("Die Firma ist nicht pleite.", False), # "ist", "nicht"
("André said the deal is done.", True),
("Beyoncé released a new album.", True),
("Müller won the race in Berlin.", True),
@@ -155,7 +156,7 @@ def test_the_judge_translates_a_turkish_claim_for_the_check_and_the_queries() ->
def test_a_rate_limited_judge_leaves_a_turkish_claim_untranslated() -> None:
plan = plan_queries(
["OpenAI battı."], # noqa: RUF001 - real Turkish letters
- judge(JudgeUnavailable("HTTP 429 from https://api.test")),
+ judge(JudgeUnavailable("HTTP 429 from https://api.test", status=429)),
)
assert plan.hypotheses == (None,)
assert plan.notice is not None and plan.notice.startswith(LLM_LIMIT)
@@ -188,7 +189,7 @@ def test_an_english_that_is_not_english_is_no_translation(english: str) -> None:
def test_an_unchanged_echo_of_a_non_english_claim_is_no_translation() -> None:
"""Any language, not only Turkish: a French claim echoed back is still French."""
- claim = "Le chat est noir."
+ claim = "Le chat est dans une maison." # "dans", "une": two function words
assert not looks_english(claim)
answer = json.dumps({"items": [{"id": 0, "english": claim, "queries": ["black cat"]}]})
assert plan_queries([claim], judge(answer)).hypotheses == (None,)
@@ -213,3 +214,149 @@ def test_the_queries_prompt_carries_worked_examples_that_cannot_be_answered() ->
' "queries": ["company went bankrupt"]}]}'
)
assert _queries_from(example, 1) == {}
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ # Final review, Important 2: short foreign function words that English text
+ # and names also print. Every one of these used to be called "not English".
+ "MIT researchers found a new battery design.",
+ "Smith et al. reported the effect in 2020.",
+ "Los Angeles has the largest port in the country.",
+ "El Niño raised ocean temperatures in 2023.",
+ "The AMA endorsed the policy last year.",
+ "Y2K cost companies billions of dollars.",
+ "The launch is at 5pm EST today.",
+ "Le Monde reported the story first.",
+ "Guillermo del Toro and Leonardo da Vinci were both named.",
+ "Uğur Şahin founded BioNTech in 2008.", # a name keeps its Turkish letters
+ "The μ-opioid receptor binds morphine.", # a Greek symbol is not Greek text
+ ],
+)
+def test_english_text_with_names_acronyms_and_borrowed_words_is_english(text: str) -> None:
+ assert looks_english(text)
+
+
+def test_one_foreign_function_word_is_not_evidence_but_two_are() -> None:
+ assert looks_english("The deal with Pfizer is que sera.")
+ assert not looks_english("La empresa que cerró es grande.") # "que", "es"
+
+
+def test_an_english_echo_of_an_english_claim_is_accepted() -> None:
+ from proofpath.search.queries import _translation
+
+ claim = "OpenAI shut down ChatGPT in 2025."
+ assert _translation(claim, claim) == claim
+
+
+def test_an_echo_of_a_non_english_claim_is_still_refused() -> None:
+ from proofpath.search.queries import _translation
+
+ claim = "OpenAI battı." # noqa: RUF001 - real Turkish letters
+ assert _translation(claim, claim) is None
+
+
+@pytest.mark.parametrize(
+ "english",
+ ["The company went bankrupt.", "The Eiffel Tower is in Paris.", "the eiffel tower is in paris"],
+)
+def test_a_copied_prompt_example_is_no_translation(english: str) -> None:
+ """Ledger T11: an example answer copied into the answer would be checked as the
+ claim. The example's own claim still translates to it."""
+ from proofpath.search.queries import _translation
+
+ assert _translation("OpenAI iflas etti ve bu kapandı.", english) is None # noqa: RUF001
+ assert _translation("Şirket battı.", "The company went bankrupt.") == ( # noqa: RUF001
+ "The company went bankrupt."
+ )
+
+
+def test_the_refused_examples_are_the_ones_the_prompt_prints() -> None:
+ from proofpath.judge import load_prompt
+ from proofpath.search.queries import PROMPT_EXAMPLES
+
+ prompt = load_prompt("queries").template
+ for english, claim in PROMPT_EXAMPLES.items():
+ assert english in prompt and claim in prompt
+
+
+@pytest.mark.parametrize(
+ ("error", "head"),
+ [
+ (JudgeUnavailable("HTTP 429 from https://api.test", status=429), LLM_LIMIT),
+ # Groq answers 413 when a request busts its per-minute token budget.
+ (JudgeUnavailable("HTTP 413 from https://api.test", status=413), LLM_LIMIT),
+ # A detail that merely mentions 429 is not a limit: the status decides.
+ (
+ JudgeUnavailable(
+ "HTTP 429 from https://api.test; local ollama q: HTTP 404", status=404
+ ),
+ LLM_UNAVAILABLE,
+ ),
+ (JudgeUnavailable("request failed: ConnectError"), LLM_UNAVAILABLE),
+ ],
+)
+def test_the_limit_notice_is_decided_by_the_status_alone(
+ error: JudgeUnavailable, head: str
+) -> None:
+ plan = plan_queries(["ChatGPT was shut down in 2025."], judge(error))
+ assert plan.notice is not None and plan.notice.startswith(head)
+
+
+# --- round 2: short foreign sentences, and the judge's translation ---------------------
+
+FOREIGN_ROUND_2 = [
+ "El gobierno subió los impuestos.", # "los" + an accented lowercase word
+ "Le président a démissionné hier.", # accented lowercase words + "hier"
+ "Die Regierung hat das Gesetz beschlossen.", # "hat", "das"
+ "Il governo ha approvato la legge.", # "ha", "la"
+ "Los precios de la gasolina subieron un 20%.", # "la", "un"
+]
+# Read as English without a judge: every signal these carry is either capitalised
+# ("Las" opens the sentence, exactly as "Los" opens "Los Angeles has…") or a word the
+# rule leaves out on purpose ("de", which English prints in "de facto" and names).
+# No rule over these signals can call them foreign and keep the English cases English;
+# with a judge, round 2 (a) checks the judge's translation instead.
+ENGLISH_WITHOUT_A_JUDGE = [
+ "Las vacunas causan autismo.",
+ "De regering heeft de wet aangenomen.",
+ "Enflasyon yüzde 80 oldu.", # one signal: a single accented lowercase word
+]
+
+
+@pytest.mark.parametrize("text", FOREIGN_ROUND_2)
+def test_short_foreign_sentences_are_not_english(text: str) -> None:
+ assert not looks_english(text)
+
+
+@pytest.mark.parametrize("text", ENGLISH_WITHOUT_A_JUDGE)
+def test_sentences_with_one_signal_read_as_english_without_a_judge(text: str) -> None:
+ assert looks_english(text)
+
+
+def test_borrowed_accented_words_are_one_signal_not_many() -> None:
+ assert looks_english("The café serves crème brûlée every night.")
+
+
+@pytest.mark.parametrize("text", ENGLISH_WITHOUT_A_JUDGE)
+def test_with_a_judge_a_sentence_that_is_not_certainly_english_is_checked_translated(
+ text: str,
+) -> None:
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": "The claim, in English.", "queries": ["q"]}]}
+ )
+ plan = plan_queries([text], judge(answer))
+ assert plan.hypotheses == ("The claim, in English.",)
+
+
+def test_a_certainly_english_claim_is_checked_as_written_whatever_the_judge_says() -> None:
+ claim = "OpenAI shut down ChatGPT in 2025."
+ answer = json.dumps({"items": [{"id": 0, "english": "OpenAI closed ChatGPT.", "queries": []}]})
+ assert plan_queries([claim], judge(answer)).hypotheses == (claim,)
+
+
+def test_an_echo_of_a_not_certainly_english_claim_keeps_the_claim() -> None:
+ claim = "Los Angeles has the largest port in the country."
+ answer = json.dumps({"items": [{"id": 0, "english": claim, "queries": []}]})
+ assert plan_queries([claim], judge(answer)).hypotheses == (claim,)
diff --git a/tests/test_verify.py b/tests/test_verify.py
index 9f8d765..3634904 100644
--- a/tests/test_verify.py
+++ b/tests/test_verify.py
@@ -161,6 +161,8 @@ def __init__(
self.asked: list[tuple[str, str]] = [] # (url, text_kind) as the ladder saw it
# (url, counts_as_source, use_cache): how the target page was asked for.
self.options: list[tuple[str, bool, bool]] = []
+ # The URLs fetched without the contact address (``Fetcher.fetch(anonymous=)``).
+ self.anonymous: list[str] = []
self.on_call: Callable[[], None] | None = None
def fetch(
@@ -170,7 +172,10 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched:
+ if anonymous:
+ self.anonymous.append(url)
self.calls.append(url)
self.asked.append((url, text_kind))
self.options.append((url, counts_as_source, use_cache))
diff --git a/tests/test_verify_golden.py b/tests/test_verify_golden.py
index 5ea821e..f61563c 100644
--- a/tests/test_verify_golden.py
+++ b/tests/test_verify_golden.py
@@ -122,6 +122,7 @@ def fetch(
text_kind: str = "fulltext",
counts_as_source: bool = True,
use_cache: bool = True,
+ anonymous: bool = False,
) -> Fetched:
return self.pages[url]
diff --git a/tests/test_verify_search.py b/tests/test_verify_search.py
index 3c5506f..c911675 100644
--- a/tests/test_verify_search.py
+++ b/tests/test_verify_search.py
@@ -23,6 +23,8 @@
SEARCH_UNAVAILABLE,
Kind,
)
+from proofpath.resolve import ResolveResult, State
+from proofpath.sarif import to_sarif
from proofpath.search import SearchHit
from proofpath.search.queries import LLM_LIMIT, sentence_query
from proofpath.secrets import CREDENTIALS_MISSING
@@ -44,6 +46,7 @@
SUPPORTED_ROW,
FakeJudgeClient,
StubFetcher,
+ StubResolver,
_opinion,
engine,
fetched,
@@ -338,7 +341,8 @@ def test_the_cap_limits_the_search_and_the_rest_is_counted() -> None:
def test_a_rate_limited_judge_falls_back_to_the_sentence_and_the_report_says_so() -> None:
searcher = StubSearcher()
built = searching(searcher)
- built.judge = Judge(FakeJudgeClient([JudgeUnavailable("HTTP 429 from https://api.test")])) # type: ignore[arg-type]
+ limited = JudgeUnavailable("HTTP 429 from https://api.test", status=429)
+ built.judge = Judge(FakeJudgeClient([limited])) # type: ignore[arg-type]
report = verify(CLAIM, built)
assert searcher.queries == [sentence_query(CLAIM)]
assert report.search is not None
@@ -475,3 +479,240 @@ def test_a_translated_claims_judgement_is_cached_under_the_english_hash(tmp_path
# Never under the reader's own sentence: that would reuse this opinion for a
# differently translated run of the same Turkish claim later on.
assert db.get_judgement(claim_hash(TURKISH), not_supported.source_id, judge.name) is None
+
+
+# --- found pages are read as pages (final review, Critical 1) ------------------------
+
+# The three addresses the final review reproduced a false ghost with: each carries a
+# DOI or an arXiv id, which used to send the found page to the bibliographic indexes.
+IDENTIFIER_PAGES = (
+ "https://pubs.acs.org/doi/10.1021/acs.est.0c01234",
+ "https://doi.org/10.1021/acs.est.0c01234",
+ "https://arxiv.org/abs/2106.09685",
+)
+
+
+def _found_identifier_pages(cache: Cache | None = None) -> tuple[Engine, StubResolver]:
+ resolver = StubResolver() # answers GHOST to anything it is asked
+ fetcher = StubFetcher({url: fetched(url, f"{BACKING} {FILLER}") for url in IDENTIFIER_PAGES})
+ built = engine(
+ resolver=resolver,
+ fetcher=fetcher,
+ cache=cache,
+ config=Config(search=SearchConfig(results_per_claim=3)),
+ embedder=WordEmbedder(),
+ scorer=TableScorer({BACKING: SUPPORTED_ROW}),
+ )
+ built.searcher = StubSearcher({"ChatGPT": list(IDENTIFIER_PAGES)})
+ return built, resolver
+
+
+def test_a_found_page_with_a_doi_or_arxiv_id_is_read_as_a_page_never_a_ghost() -> None:
+ built, resolver = _found_identifier_pages()
+ report = verify(CLAIM, built)
+
+ assert resolver.resolved == [] # the indexes are never asked about a found page
+ assert Kind.GHOST not in report.counts()
+ assert [source.source_id for source in report.sources] == [
+ f"url:{url}" for url in IDENTIFIER_PAGES
+ ]
+ assert [source.reference.origin for source in report.sources] == ["search"] * 3
+ assert report.counts() == {Kind.EVIDENCE_FOUND: 3}
+
+
+def test_a_found_page_never_reads_a_cached_resolution_of_the_same_address(
+ tmp_path: Path,
+) -> None:
+ # A bibliography that printed the bare address resolved it through the indexes;
+ # that record is about an entry, not about the page the search found.
+ with Cache(tmp_path / "c.sqlite3") as db:
+ for url in IDENTIFIER_PAGES:
+ db.put_resolution(url, ResolveResult(State.GHOST, None, []))
+ built, resolver = _found_identifier_pages(cache=db)
+ # ``prepare`` only: the resolving stage is the one under test, and the model
+ # half would want the ``sources`` rows a real fetch writes.
+ ready = prepare(CLAIM, built)
+ assert resolver.resolved == []
+ assert [status.state for status in ready.sources.values()].count("GHOST REFERENCE") == 0
+ assert [status.source_id for status in ready.sources.values()] == [
+ f"url:{url}" for url in IDENTIFIER_PAGES
+ ]
+
+
+def test_a_found_page_leaves_no_resolution_behind_for_a_bibliography_entry(
+ tmp_path: Path,
+) -> None:
+ with Cache(tmp_path / "c.sqlite3") as db:
+ built, _ = _found_identifier_pages(cache=db)
+ prepare(CLAIM, built)
+ assert all(db.get_resolution(url) is None for url in IDENTIFIER_PAGES)
+
+
+# --- findings name their claim, and the hint is true in every case (final review) -----
+
+
+def test_a_search_unavailable_finding_carries_the_claim_it_was_about() -> None:
+ report = verify(CLAIM, searching(StubSearcher(error=ProviderError("HTTP 503", 503, None))))
+ [item] = report.findings
+ assert item.state == SEARCH_UNAVAILABLE
+ assert item.claim is not None and item.claim.text == CLAIM
+ [result] = to_sarif(report, artifact="claim.txt")["runs"][0]["results"]
+ assert result["properties"]["claim"] == CLAIM
+
+
+def test_a_language_unsupported_finding_carries_the_claim_it_was_about() -> None:
+ report = verify(TURKISH, searching(StubSearcher()))
+ [item] = report.findings
+ assert item.state == LANGUAGE_UNSUPPORTED
+ assert item.claim is not None and item.claim.text == TURKISH
+
+
+def test_the_language_hint_is_true_when_a_judge_was_on_and_did_not_translate() -> None:
+ """``--judge`` was on here, so "--judge translates a claim" would be false: the
+ hint says that no judge translated *this* claim, and how to have one do it."""
+ answer = json.dumps(
+ {"items": [{"id": 0, "english": TURKISH, "queries": ["OpenAI bankruptcy"]}]}
+ )
+ built = searching(StubSearcher())
+ built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type]
+ [item] = verify(TURKISH, built).findings
+ [hint] = item.detail
+ assert "no judge translated this claim" in hint
+ assert "--judge" in hint
+ assert "/config" not in hint # a TUI setting that would not help a TUI run
+
+
+# --- the banner at the top, and the exit code (final review, minors) -----------------
+
+
+def _cli_run(
+ built: Engine, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *args: str
+) -> tuple[int, str, str]:
+ from typing import Any
+
+ from typer.testing import CliRunner
+
+ from proofpath import verify as verify_mod
+ from proofpath.cli import app
+
+ monkeypatch.setenv("PROOFPATH_CONFIG_DIR", str(tmp_path / "conf"))
+
+ def fake_default(config: Config, **kwargs: Any) -> Engine:
+ return built
+
+ monkeypatch.setattr(verify_mod.Engine, "default", staticmethod(fake_default))
+ target = tmp_path / "claim.txt"
+ target.write_text(CLAIM, encoding="utf-8")
+ result = CliRunner().invoke(app, ["check", str(target), "--out", str(tmp_path / "r.md"), *args])
+ return result.exit_code, result.stdout, (tmp_path / "r.md").read_text(encoding="utf-8")
+
+
+def _supported_engine() -> Engine:
+ return searching(StubSearcher({"ChatGPT": [PAGE_A]}), pages={PAGE_A: f"{BACKING} {FILLER}"})
+
+
+def test_the_experimental_banner_opens_the_terminal_and_markdown_report(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from proofpath.report import search_experimental
+
+ _, stdout, markdown = _cli_run(_supported_engine(), tmp_path, monkeypatch)
+ banner = search_experimental()
+ lines = stdout.splitlines()
+ at = next(i for i, line in enumerate(lines) if banner in line)
+ finding = next(i for i, line in enumerate(lines) if "[evidence-found]" in line)
+ assert at < finding # before the first finding, not in the footer after it
+ assert stdout.count(banner) == 1
+ body = markdown.splitlines()
+ assert banner in body[2] # straight under the title
+ assert markdown.count(banner) == 1
+
+
+def test_a_run_whose_only_findings_are_supported_found_pages_exits_zero(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Spec section 13.3 says 0 is clean and is silent on search; a claim a found page
+ supports is the searched run's clean answer, like a supported cited claim."""
+ code, _, _ = _cli_run(_supported_engine(), tmp_path, monkeypatch)
+ assert code == 0
+
+
+@pytest.mark.parametrize(
+ ("searcher", "pages", "table"),
+ [
+ (StubSearcher({"ChatGPT": [PAGE_A]}), {PAGE_A: f"{BACKING} {FILLER}"}, REFUTED_ROW),
+ (StubSearcher(), {}, SUPPORTED_ROW), # NO EVIDENCE FOUND
+ (StubSearcher({"ChatGPT": [PAGE_A]}), {}, SUPPORTED_ROW), # unread page
+ (StubSearcher(error=ProviderError("HTTP 503", 503, None)), {}, SUPPORTED_ROW),
+ ],
+)
+def test_every_other_searched_outcome_still_exits_one(
+ searcher: StubSearcher,
+ pages: dict[str, str],
+ table: tuple[float, float, float],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ built = searching(searcher, pages=pages, table={BACKING: table})
+ code, _, _ = _cli_run(built, tmp_path, monkeypatch)
+ assert code == 1
+
+
+def test_a_rejected_search_key_is_reported_as_a_credential_not_an_outage() -> None:
+ from proofpath.search import SearchKeyError
+
+ rejected = SearchKeyError(
+ "tavily rejected the key (HTTP 401); check TAVILY_API_KEY in .env", 401
+ )
+ report = verify(CLAIM, searching(StubSearcher(error=rejected)))
+ [item] = report.findings
+ assert item.kind is Kind.UNVERIFIED
+ assert item.state == CREDENTIALS_MISSING
+ assert item.state != SEARCH_UNAVAILABLE
+ assert any("TAVILY_API_KEY" in line for line in item.detail)
+ assert item.claim is not None and item.claim.text == CLAIM
+ assert report.search is not None and report.search.searched == 0
+
+
+# --- round 2: a capped search is not a clean run (rule 6) ------------------------------
+
+SEVEN_CLAIMS = " ".join(f"OpenAI shut down ChatGPT in March 202{n} after a vote." for n in range(7))
+
+
+def test_a_capped_search_whose_searched_claims_are_all_supported_still_exits_one(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Seven check-worthy claims, five searched (the cap), every one of them backed:
+ the two never searched are unchecked, and exit 0 would call the text clean."""
+ backings = {
+ f"OpenAI shut down ChatGPT in March 202{n} after a vote.": SUPPORTED_ROW for n in range(7)
+ }
+ pages = {f"https://news.test/{n}": f"{text} {FILLER}" for n, text in enumerate(backings)}
+ searcher = StubSearcher({f"202{n}": [f"https://news.test/{n}"] for n in range(7)})
+ built = searching(searcher, pages=pages, table=backings)
+ report = verify(SEVEN_CLAIMS, built)
+ assert report.search is not None
+ assert (report.search.eligible, report.search.searched) == (7, 5)
+ assert set(report.counts()) == {Kind.EVIDENCE_FOUND}
+ assert report.exit_code() == 1
+
+
+def test_a_found_page_is_fetched_without_the_contact_address_and_a_cited_one_with_it() -> None:
+ """Round 2, privacy: the user chose the pages their text cites, not the ones the
+ search found, so only the latter are fetched anonymously."""
+ built = searching(StubSearcher({"ChatGPT": [PAGE_A]}), pages={PAGE_A: f"{BACKING} {FILLER}"})
+ verify(CLAIM, built)
+ assert built.fetcher.anonymous == [PAGE_A] # type: ignore[attr-defined]
+
+ cited = "https://cited.test/report"
+ text = f"{CLAIM} [1]\n\nReferences\n\n[1] {cited}\n"
+ built = engine(
+ # The indexes do not cover it, so it is read from the address it prints.
+ resolver=StubResolver({"cited.test": ResolveResult(State.NOT_INDEXED, None, [])}),
+ fetcher=StubFetcher({cited: fetched(cited, f"{BACKING} {FILLER}")}),
+ embedder=WordEmbedder(),
+ scorer=TableScorer({BACKING: SUPPORTED_ROW}),
+ )
+ verify(text, built)
+ assert cited in built.fetcher.calls # type: ignore[attr-defined]
+ assert built.fetcher.anonymous == [] # type: ignore[attr-defined]
From 998048e2e08a8d70e92e4ea88e41d135f0b993bf Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Tue, 29 Sep 2026 11:46:23 +0300
Subject: [PATCH 4/7] =?UTF-8?q?docs(eval):=20measure=20evidence=20search?=
=?UTF-8?q?=20on=20AVeriTeC=20=E2=80=94=200.404?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The product's own search path reads the pages it finds for 100 AVeriTeC
dev claims and scores 0.404 three-way, against a 0.708 majority
baseline. That is better than the 0.270 the gold source URLs gave, but
the gate of spec §17.1 is not met, so the feature stays experimental.
The run from 2026-09-28 used a path the final review replaced and is
dropped.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
README.md | 2 +-
docs/eval/2026-09-28-averitec-search.md | 37 -------------------
docs/eval/2026-09-29-averitec-search.md | 47 +++++++++++++++++++++++++
src/proofpath/report.py | 2 +-
4 files changed, 49 insertions(+), 39 deletions(-)
delete mode 100644 docs/eval/2026-09-28-averitec-search.md
create mode 100644 docs/eval/2026-09-29-averitec-search.md
diff --git a/README.md b/README.md
index a387bb9..d565a24 100644
--- a/README.md
+++ b/README.md
@@ -336,7 +336,7 @@ fallback below takes over; only if that fails too is the plain sentence searched
the report says so.
**This is experimental.** On AVeriTeC's search scenario, proofpath scores
-`<>` against a 0.708 majority baseline, and every report from a
+[**0.404**](docs/eval/2026-09-29-averitec-search.md) against a 0.708 majority baseline, and every report from a
run that searched prints that line at the top.
## Optional LLM judge (v0.3)
diff --git a/docs/eval/2026-09-28-averitec-search.md b/docs/eval/2026-09-28-averitec-search.md
deleted file mode 100644
index e4f178c..0000000
--- a/docs/eval/2026-09-28-averitec-search.md
+++ /dev/null
@@ -1,37 +0,0 @@
-# AVeriTeC dev — 2026-09-28
-
-- claims: 100 (limit=100)
-- dataset: `https://raw.githubusercontent.com/MichSchli/AVeriTeC/main/data/dev.json` (sha256 499793726b4a…)
-- one run of the whole product: real claims, real source pages, real fetch ladder.
-
-## Headline
-
-3-way accuracy **0.326** vs majority baseline 0.708, over 89 of 100 claims.
-
-The 3-way number excludes the Conflicting Evidence/Cherrypicking rows, which proofpath has no verdict for; counting them as wrong gives a 4-way accuracy of 0.290.
-
-## Per label
-
-| label | n | correct | accuracy |
-|---|---|---|---|
-| Supported | 19 | 9 | 0.474 |
-| Refuted | 63 | 18 | 0.286 |
-| Not Enough Evidence | 7 | 2 | 0.286 |
-| Conflicting Evidence/Cherrypicking | 11 | 0 | 0.000 |
-
-## Source coverage
-
-| state | count |
-|---|---|
-| ok | 178 |
-| UNVERIFIED (blocked, browser not permitted) | 80 |
-| UNVERIFIED (blocked, robots.txt) | 36 |
-| UNVERIFIED (unreachable) | 2 |
-| UNVERIFIED (provider unavailable) | 1 |
-
-0 of 297 source URLs are web.archive.org snapshots (0.0 %).
-
-## Notes
-
-
-
diff --git a/docs/eval/2026-09-29-averitec-search.md b/docs/eval/2026-09-29-averitec-search.md
new file mode 100644
index 0000000..42fe159
--- /dev/null
+++ b/docs/eval/2026-09-29-averitec-search.md
@@ -0,0 +1,47 @@
+# AVeriTeC dev — 2026-09-29
+
+- claims: 100 (limit=100)
+- measured path: sentence queries, no judge
+- dataset: `https://raw.githubusercontent.com/MichSchli/AVeriTeC/main/data/dev.json` (sha256 499793726b4a…)
+- one run of the whole product: real claims, real source pages, real fetch ladder.
+
+## Headline
+
+3-way accuracy **0.404** vs majority baseline 0.708, over 89 of 100 claims.
+
+The 3-way number excludes the Conflicting Evidence/Cherrypicking rows, which proofpath has no verdict for; counting them as wrong gives a 4-way accuracy of 0.360.
+
+## Per label
+
+| label | n | correct | accuracy |
+|---|---|---|---|
+| Supported | 19 | 9 | 0.474 |
+| Refuted | 63 | 25 | 0.397 |
+| Not Enough Evidence | 7 | 2 | 0.286 |
+| Conflicting Evidence/Cherrypicking | 11 | 0 | 0.000 |
+
+## Source coverage
+
+| state | count |
+|---|---|
+| ok | 224 |
+| UNVERIFIED (blocked, browser not permitted) | 74 |
+| UNVERIFIED (blocked, robots.txt) | 20 |
+| UNVERIFIED (unreachable) | 3 |
+| UNVERIFIED (provider unavailable) | 2 |
+| UNVERIFIED (reached, no text extracted) | 1 |
+
+0 of 324 source URLs are web.archive.org snapshots (0.0 %).
+
+## Notes
+
+- This is the evidence-search gate of spec §17.1. At 0.404 it is below the 0.708
+ majority baseline, so the feature stays marked experimental.
+- For comparison: the same 100 claims read from their **gold** source URLs scored 0.270
+ (`2026-09-16-averitec.md`). An earlier search run on 2026-09-28 scored 0.326. That run
+ took the old path, which over-fetched results and skipped the language gate; the
+ final review replaced that path with this one.
+- 74 pages needed the browser step, which this run did not use (`--no-browser`). A run
+ with the browser allowed would read more pages.
+- Measured with sentence queries and no judge. A judge writes better queries; that
+ path has not been measured yet.
diff --git a/src/proofpath/report.py b/src/proofpath/report.py
index 8dd72ce..d6dc798 100644
--- a/src/proofpath/report.py
+++ b/src/proofpath/report.py
@@ -59,7 +59,7 @@
LANGUAGE_UNSUPPORTED = f"{UNVERIFIED_PREFIX} (language not supported)"
# The §17.1 gate: printed on every run that searched until the eval beats the
# majority baseline. Updated by hand from docs/eval/*-averitec-search.md.
-AVERITEC_SEARCH_SCORE = "not measured yet"
+AVERITEC_SEARCH_SCORE = "0.404"
AVERITEC_BASELINE = "0.708"
From 964f3ec4a63d86d54a727366cb30964d212ce681 Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Tue, 29 Sep 2026 11:47:02 +0300
Subject: [PATCH 5/7] docs(spec): list the evidence-search states in section 15
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The PR checklist asks for every new reported state to be in spec §15;
the four from the evidence-search spec were only there.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
docs/superpowers/specs/2026-09-10-proofpath-design.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/docs/superpowers/specs/2026-09-10-proofpath-design.md b/docs/superpowers/specs/2026-09-10-proofpath-design.md
index cb63679..f84f5c6 100644
--- a/docs/superpowers/specs/2026-09-10-proofpath-design.md
+++ b/docs/superpowers/specs/2026-09-10-proofpath-design.md
@@ -940,6 +940,10 @@ presented as evidence of absence.
| `PARAGRAPH-SCOPED` | the citation supports a paragraph, not one sentence (§9); every sentence is verified separately and grouped |
| `UNSUPPORTED CITATION STYLE` | a citation style proofpath cannot pair yet — footnote-only and superscript-letter (`ᵃ`) styles; the claim is listed but not judged. Author-year has been paired since v0.2 and is no longer reported here |
| `UNRESOLVED MARKER` | a citation marker naming no bibliography entry: a number outside what the list prints, or an author-year item with no matching entry — or with several and no `2020a`/`2020b` to say which. Reported, never guessed at (§9 step 2) |
+| `SUPPORTED` / `REFUTED (found by proofpath)` | v0.4.8, evidence search: a page proofpath found itself for a claim with no source, with its passage. Tagged `FOUND BY PROOFPATH (not cited by the author)`. See `2026-09-28-evidence-search-design.md` §4 |
+| `NO EVIDENCE FOUND (searched)` | v0.4.8: the search ran and no page confirmed or contradicted the claim. Never `REFUTED` (rule 2) |
+| `UNVERIFIED (search unavailable)` | v0.4.8: the search provider failed. A rejected key is `UNVERIFIED (credentials missing)` |
+| `UNVERIFIED (language not supported)` | v0.4.8: the claim is not in English and no judge translated it. The NLI models read English only |
Unparseable pages fail loudly with the page number and processing continues.
From d79d32318557f0f59d6d930a587a071e15b95485 Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Tue, 29 Sep 2026 11:52:41 +0300
Subject: [PATCH 6/7] docs: show the evidence-search states and the judge
fallback on the page
The README's state table and the landing page's state list, X row and
judge paragraph still described the tool before this branch: no search
states, and a judge that went quiet when its provider stopped.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
README.md | 4 ++++
site/src/components/Judge.astro | 4 +++-
site/src/data/posts.ts | 2 +-
site/src/data/states.ts | 13 +++++++++++++
4 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index d565a24..86f11dc 100644
--- a/README.md
+++ b/README.md
@@ -188,6 +188,10 @@ distinct, printed state (spec §15), never collapsed into a verdict:
| `AMBIGUOUS` | several plausible records, all listed |
| `NEI` | the source was read and neither supports nor contradicts |
| `PARAGRAPH-SCOPED` | the citation covers a paragraph; each sentence is judged separately |
+| `FOUND BY PROOFPATH (not cited by the author)` | evidence search (experimental): proofpath found this page itself, for a text that cited nothing; the verdict still quotes its passage |
+| `NO EVIDENCE FOUND (searched)` | the search ran and no page it found confirmed or contradicted the claim; never `REFUTED` |
+| `UNVERIFIED (search unavailable)` | the search provider did not answer |
+| `UNVERIFIED (language not supported)` | the claim is not in English and no judge translated it; the models read English only |
| `UNSUPPORTED CITATION STYLE` | reserved; no detected style produces it in v0.2 |
An earlier build sometimes aborted with `134` after printing a complete report (ONNX
diff --git a/site/src/components/Judge.astro b/site/src/components/Judge.astro
index 017059b..8e091c1 100644
--- a/site/src/components/Judge.astro
+++ b/site/src/components/Judge.astro
@@ -16,7 +16,9 @@ import SketchArrow from './sketch/SketchArrow.astro';
--judge asks it about the low-confidence verdicts alone, and its answer is
printed beside the local verdict, never in place of it.{' '}
--summarize is one call over the finished report, labelled model-written.
- If the provider does not answer, the report says so; the SARIF log never carries either.
+ If the provider hits its limit or stops answering, the run switches at once to a local{' '}
+ qwen3.5:9b, if it is installed, and says so; the next run tries the
+ provider first again. The SARIF log never carries either.
diff --git a/site/src/data/posts.ts b/site/src/data/posts.ts
index a606681..bc75e3b 100644
--- a/site/src/data/posts.ts
+++ b/site/src/data/posts.ts
@@ -62,7 +62,7 @@ export const platforms: readonly Platform[] = [
},
{
name: 'X',
- how: 'Cannot be read at all. The run says so and asks you to paste the text; the links inside it are then verified as usual.',
+ how: 'Cannot be read at all. The run says so and asks you to paste the text; the links inside it are then verified as usual, and with evidence search set up a text with no link is searched (experimental).',
state: { label: 'paste the text', tone: 'ink' },
},
];
diff --git a/site/src/data/states.ts b/site/src/data/states.ts
index 9830791..48fb8be 100644
--- a/site/src/data/states.ts
+++ b/site/src/data/states.ts
@@ -41,4 +41,17 @@ export const states: readonly State[] = [
name: 'PARAGRAPH-SCOPED',
cause: 'the citation covers a paragraph; each sentence is judged separately',
},
+ {
+ name: 'FOUND BY PROOFPATH (not cited by the author)',
+ cause: 'evidence search (experimental): proofpath found this page itself, for a text that cited nothing; the verdict still quotes its passage',
+ },
+ {
+ name: 'NO EVIDENCE FOUND (searched)',
+ cause: 'the search ran and no page it found confirmed or contradicted the claim; never REFUTED',
+ },
+ { name: 'UNVERIFIED (search unavailable)', cause: 'the search provider did not answer' },
+ {
+ name: 'UNVERIFIED (language not supported)',
+ cause: 'the claim is not in English and no judge translated it; the models read English only',
+ },
];
From 0eb4e9ed4dfeececfc5ab608b68ee071550bc380 Mon Sep 17 00:00:00 2001
From: Yigtwxx
Date: Tue, 29 Sep 2026 12:07:08 +0300
Subject: [PATCH 7/7] feat(judge): say how to add the local model when there is
none
When the judge's API gives out and no local model can take over, the
run now says how to add one: install Ollama and pull qwen3.5:9b, or
only pull the model when Ollama is already there. It goes out once per
run on the switch notice's channels, and never when the user turned the
fallback off. Nothing is installed; it is only a line.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_01BS4saoFyA3Ga9HgfRUUVux
---
README.md | 3 +-
src/proofpath/judge.py | 82 +++++++++++++++++++++-----
src/proofpath/verify.py | 7 ++-
tests/test_judge_fallback.py | 111 +++++++++++++++++++++++++++++++++++
tests/test_tui_app.py | 22 +++++++
5 files changed, 207 insertions(+), 18 deletions(-)
diff --git a/README.md b/README.md
index 86f11dc..1027c7f 100644
--- a/README.md
+++ b/README.md
@@ -396,7 +396,8 @@ in the environment or a `.env` file, never from config, and is never printed.
otherwise stops answering, the run switches at once to `qwen3.5:9b` in your local
Ollama, if it is installed (`ollama pull qwen3.5:9b`; proofpath never pulls it). It says so on stderr, in the TUI
above the prompt, and in the report. The next run tries the configured API again first.
-`proofpath config set judge.fallback off` turns this off.
+`proofpath config set judge.fallback off` turns this off. Without a local model to
+switch to, the run says so and tells you how to add one, on the same channels.
## Measured
diff --git a/src/proofpath/judge.py b/src/proofpath/judge.py
index 71ab70a..7758668 100644
--- a/src/proofpath/judge.py
+++ b/src/proofpath/judge.py
@@ -556,6 +556,20 @@ def _record(self, response: httpx.Response, messages: list[dict[str, str]]) -> C
#: What ``switched`` says once the local model has failed too: the primary's reason,
#: then the local one. "Judging with local ollama" would then be false (final review).
JUDGE_FALLBACK_FAILED = "{head}, and local ollama {model} failed too ({cause})"
+#: The install/pull hint: offered once, beside the original error, when there is no
+#: fallback for a reason the user can fix by adding the local model -- never when
+#: they chose ``judge.fallback = off`` themselves, and never a notice of a switch
+#: that did not happen (``JudgeClient.hint`` is its own attribute, apart from
+#: ``switched``, so the two states are never conflated). ``{model}`` is the fallback
+#: as configured (``judge.fallback``, ``qwen3.5:9b`` unless the user named another);
+#: ``{provider}`` the primary's display name, same as the switch notices above.
+JUDGE_FALLBACK_HINT_UNREACHABLE = (
+ "install Ollama (https://ollama.com) and run: ollama pull {model} — "
+ "the judge then keeps going locally when {provider} runs out"
+)
+JUDGE_FALLBACK_HINT_MISSING = (
+ "run: ollama pull {model} — the judge then keeps going locally when {provider} runs out"
+)
#: The statuses that mean "over the provider's limit", read wherever a limit is named
#: (the switch notice, the search's fallback notice), so both classify it one way.
LIMIT_STATUSES = frozenset({429, 413}) # Groq answers 413 when a request busts TPM
@@ -668,6 +682,11 @@ class _Plan:
model: str | None
why: str = ""
+ #: The hint template for ``why``, or ``None`` when there is nothing to suggest
+ #: (a fallback, or the user's own ``off``). Formatted with the model and provider
+ #: only once the primary has actually failed (``_switch``): asking Ollama is
+ #: already paid for here, but the hint is only worth showing beside a real error.
+ hint: str | None = None
class FallbackClient:
@@ -706,6 +725,11 @@ def __init__(
self.on_switch = on_switch
#: The notice, once the switch has happened; ``None`` before.
self.switched: str | None = None
+ #: The install/pull hint, once raised for this run; ``None`` before, and
+ #: ``None`` for good when there was nothing to suggest. Apart from
+ #: ``switched``: a hint is offered instead of a switch, never alongside one,
+ #: and the two must never be read as the same thing.
+ self.hint: str | None = None
# Worker threads share one client (the TUI's summary, a run's stages): the
# plan and the switch go through this lock so that exactly one thread
# switches and every other one sees the result.
@@ -775,6 +799,7 @@ def reset(self) -> None:
self._plan = None
self._dead = None
self.switched = None
+ self.hint = None
self._primary_detail = ""
self._primary_failure = None
self._notice = None
@@ -901,37 +926,58 @@ def _choose(self) -> _Plan:
return _Plan(None, _WHY_OFF)
names = _tags(self._ollama_url, timeout=TAGS_TIMEOUT)
if names is None:
- return _Plan(None, f"ollama is not reachable at {_ollama_root(self._ollama_url)}")
+ return _Plan(
+ None,
+ f"ollama is not reachable at {_ollama_root(self._ollama_url)}",
+ hint=JUDGE_FALLBACK_HINT_UNREACHABLE,
+ )
if not _installed(setting, names):
- return _Plan(None, f"{setting} is not installed")
+ return _Plan(None, f"{setting} is not installed", hint=JUDGE_FALLBACK_HINT_MISSING)
return _Plan(setting) # as written: the user's own spelling of the name
def _switch(self, failure: JudgeUnavailable) -> tuple[JudgeClient, str | None]:
"""Switch once: the local client, and the notice when *this* call switched.
- Raises the primary's error, with the reason, when there is no fallback. The
- notice is returned rather than sent from here: the listener runs after the
- lock is released, so a slow or blocking listener can never hold another
- thread's switch (fix round 1, C).
+ Raises the primary's error, with the reason, when there is no fallback. When
+ that reason is one the user can fix -- Ollama unreachable, the model not
+ installed, never ``judge.fallback = off`` -- the hint goes out through the
+ same listener the switch notice uses, once per run (the ``self.hint is None``
+ guard: this branch runs again on every later call, since without a local
+ client nothing ever leaves ``_route`` early). The detail the error carries is
+ unchanged; the hint travels beside it, never inside it.
+
+ The notice (or hint) is announced after the lock is released, same as a
+ switch: the listener runs after the lock is released, so a slow or blocking
+ listener can never hold another thread's switch (fix round 1, C).
"""
+ hint: str | None = None
with self._lock:
if self._local is not None:
return self._local, None # another thread switched while we waited
plan = self._plan
if plan is None or plan.model is None:
why = _WHY_OFF if plan is None else plan.why
- raise JudgeUnavailable(
+ if plan is not None and plan.hint is not None and self.hint is None:
+ hint = plan.hint.format(
+ model=self._fallback, provider=_display(self._primary.provider)
+ )
+ self.hint = hint
+ error = JudgeUnavailable(
f"{failure}; no local fallback: {why}",
status=failure.status,
cause=failure.cause,
- ) from None
- notice = _notice(self._primary.provider, plan.model, failure)
- self._primary_detail = str(failure)
- self._primary_failure = failure
- self._notice = notice
- self._local = self._local_factory(plan.model, self._ollama_url)
- self.switched = notice
- return self._local, notice
+ )
+ else:
+ notice = _notice(self._primary.provider, plan.model, failure)
+ self._primary_detail = str(failure)
+ self._primary_failure = failure
+ self._notice = notice
+ self._local = self._local_factory(plan.model, self._ollama_url)
+ self.switched = notice
+ return self._local, notice
+ if hint is not None:
+ self._announce(hint)
+ raise error from None
def _head(provider: str, failure: JudgeUnavailable) -> str:
@@ -1167,6 +1213,12 @@ def switched(self) -> str | None:
"""The local-fallback notice once this run has switched, else ``None``."""
return self._client.switched if isinstance(self._client, FallbackClient) else None
+ @property
+ def hint(self) -> str | None:
+ """The install/pull hint this run has raised, once there is one, else
+ ``None``. Distinct from ``switched``: a hint means nothing switched."""
+ return self._client.hint if isinstance(self._client, FallbackClient) else None
+
def listen(self, on_switch: Callable[[str], None] | None) -> None:
"""Point the switch notice at whoever is watching this run.
diff --git a/src/proofpath/verify.py b/src/proofpath/verify.py
index ca3f320..2172e2f 100644
--- a/src/proofpath/verify.py
+++ b/src/proofpath/verify.py
@@ -1372,7 +1372,10 @@ def check() -> None:
tier_note=pipeline.tier_note(engine.thresholds),
cancelled=cancelled,
search=prepared.search,
- judge_notice=None if engine.judge is None else engine.judge.switched,
+ # The hint (no fallback, but one is worth adding) rides the same one-line
+ # footer field as the switch notice: never both, since a hint means nothing
+ # switched (spec section 11 addendum).
+ judge_notice=None if engine.judge is None else (engine.judge.switched or engine.judge.hint),
)
if cancelled:
raise Cancelled("run cancelled", report=report)
@@ -1467,7 +1470,7 @@ def _summarise(
# The extra call is part of what the run took; a footer that left it out
# would price the summary at nothing.
elapsed=time.monotonic() - started,
- judge_notice=judge.switched or report.judge_notice,
+ judge_notice=judge.switched or judge.hint or report.judge_notice,
)
diff --git a/tests/test_judge_fallback.py b/tests/test_judge_fallback.py
index 248dcd0..f7050be 100644
--- a/tests/test_judge_fallback.py
+++ b/tests/test_judge_fallback.py
@@ -545,6 +545,117 @@ def test_fallback_off_never_asks_ollama_and_keeps_the_retries(mock: respx.MockRo
assert local.call_count == 0
+# --- the install hint ----------------------------------------------------------------
+#
+# When there is no local model to switch to, the run could still get one. The hint is
+# delivered through the same channel as the switch notice (``on_switch``) so cli, tui
+# and the report footer need nothing new to show it -- but it is never a notice
+# (``switched``): nothing switched, and false-ghost-style overclaiming is exactly
+# what spec section 8 forbids one state pretending to be another.
+
+HINT_UNREACHABLE = (
+ f"install Ollama (https://ollama.com) and run: ollama pull {LOCAL} — "
+ "the judge then keeps going locally when Groq runs out"
+)
+HINT_MISSING = f"run: ollama pull {LOCAL} — the judge then keeps going locally when Groq runs out"
+
+
+def test_an_unreachable_ollama_offers_to_install_it(mock: respx.MockRouter) -> None:
+ routes(mock, tags=httpx.ConnectError("refused"))
+ seen: list[str] = []
+ client = fallback(on_switch=seen.append)
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+
+ assert seen == [HINT_UNREACHABLE]
+ assert client.hint == HINT_UNREACHABLE
+ assert client.switched is None # a hint is not a notice: nothing switched
+
+
+def test_a_missing_model_offers_to_pull_it(mock: respx.MockRouter) -> None:
+ """Ollama answers, but not with the configured model: the shorter hint, since
+ Ollama itself does not need installing."""
+ routes(mock, tags=_tags("gemma4:12b"))
+ seen: list[str] = []
+ client = fallback(on_switch=seen.append)
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+
+ assert seen == [HINT_MISSING]
+ assert client.hint == HINT_MISSING
+
+
+def test_the_hint_is_offered_only_once_per_run(mock: respx.MockRouter) -> None:
+ routes(mock, tags=httpx.ConnectError("refused"))
+ seen: list[str] = []
+ client = fallback(on_switch=seen.append)
+ for _ in range(3):
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+
+ assert seen == [HINT_UNREACHABLE] # once, not once per failed call
+
+
+def test_fallback_off_gets_no_hint(mock: respx.MockRouter) -> None:
+ """The user turned the fallback off themselves; there is nothing to suggest."""
+ routes(mock)
+ seen: list[str] = []
+ client = fallback(setting="off", on_switch=seen.append)
+ with pytest.raises(judge.JudgeUnavailable):
+ client.complete(MESSAGES)
+
+ assert seen == []
+ assert client.hint is None
+
+
+def test_an_ollama_primary_gets_no_hint() -> None:
+ """``judge.provider ollama`` has no fallback behind it to suggest (spec 11)."""
+ client = judge.build_client(judge.provider_defaults("ollama"), None)
+ assert not isinstance(client, judge.FallbackClient)
+ assert judge.Judge(client).hint is None
+
+
+def test_a_run_with_no_fallback_offers_the_hint_once_and_the_footer_carries_it(
+ mock: respx.MockRouter,
+) -> None:
+ routes(mock, tags=httpx.ConnectError("refused"))
+ built = searching(StubSearcher())
+ built.judge = judge.Judge(judge.build_client(GROQ, KEY))
+ events: list[Event] = []
+ report = verify_mod.verify(CLAIM, built, on_event=events.append)
+
+ notices = [event for event in events if isinstance(event, Note) and event.notice]
+ assert [note.text for note in notices] == [HINT_UNREACHABLE]
+ assert report.judge_notice == HINT_UNREACHABLE
+ footer = render_footer(report)
+ assert footer.judge_notice == HINT_UNREACHABLE
+ markdown = render_markdown(report)
+ assert f"- judge fallback: {HINT_UNREACHABLE}" in markdown
+
+
+def test_the_cli_prints_the_install_hint_on_stderr(
+ mock: respx.MockRouter, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ routes(mock, tags=httpx.ConnectError("refused"))
+ monkeypatch.setenv("GROQ_API_KEY", KEY.value)
+ monkeypatch.setenv("PROOFPATH_CONFIG_DIR", str(tmp_path / "conf"))
+ built = searching(StubSearcher())
+
+ def fake_default(config: Config, **kwargs: Any) -> verify_mod.Engine:
+ built.judge = kwargs["judge"]
+ built.escalate = kwargs.get("escalate", True)
+ return built
+
+ monkeypatch.setattr(verify_mod.Engine, "default", staticmethod(fake_default))
+ target = tmp_path / "claim.txt"
+ target.write_text(CLAIM, encoding="utf-8")
+ result = runner.invoke(app, ["check", str(target), "--judge", "--format", "json"])
+
+ assert HINT_UNREACHABLE in result.stderr
+ assert json.loads(result.stdout)["judge_notice"] == HINT_UNREACHABLE
+ assert KEY.value not in result.output
+
+
def test_a_users_own_override_is_honoured_as_written(mock: respx.MockRouter) -> None:
"""``judge.fallback`` names a model: it is checked against the tags and used as
written. The name here is a stand-in, not a model proofpath knows about."""
diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py
index 6e50c53..ae51c11 100644
--- a/tests/test_tui_app.py
+++ b/tests/test_tui_app.py
@@ -3516,6 +3516,28 @@ async def test_the_judge_notice_sits_right_aligned_directly_above_the_prompt() -
assert footer.region.bottom == frame.region.y
+INSTALL_HINT = (
+ "install Ollama (https://ollama.com) and run: ollama pull qwen3.5:9b — "
+ "the judge then keeps going locally when Groq runs out"
+)
+
+
+async def test_the_judge_notice_shows_the_install_hint_when_there_is_no_fallback() -> None:
+ """No local model to switch to: the hint rides the same row the switch notice
+ uses (spec section 11 addendum) -- it is a ``Note(notice=True)`` like any other."""
+ app, schedulers = build_app()
+ async with app.run_test(size=SIZE) as pilot:
+ notice = app.query_one(JudgeNotice)
+ scheduler = schedulers[0]
+ await submit(pilot, "/check draft.md")
+ run = scheduler.runs[0]
+ scheduler.push(run, Note(INSTALL_HINT, notice=True))
+ await pilot.pause()
+
+ assert notice.display
+ assert notice.text == INSTALL_HINT
+
+
async def test_an_ordinary_note_never_shows_the_judge_notice() -> None:
app, schedulers = build_app()
async with app.run_test(size=SIZE) as pilot: