From 540d6becf8330b45c5c22e100d290702e62269f7 Mon Sep 17 00:00:00 2001 From: Yigtwxx Date: Wed, 30 Sep 2026 11:11:58 +0300 Subject: [PATCH 1/2] fix: read Wikipedia pages, mark abstract verdicts, paste the Tavily key Every Wikipedia page was reported UNVERIFIED (blocked) at all four ladder steps: the bot-wall check matched "captcha" inside MediaWiki's inline config script. Challenge phrases are now matched outside script/style contents and only on small pages; cf-chl still counts anywhere. A verdict judged against an abstract (or a short page) did not say so on its own row, so read alone it looked like a full-text verdict (rule 6). It now carries a line saying so, SARIF included. /config gains a "Tavily API:" row: paste the key and press Enter. It is saved to the config directory's .env, never shown, and a paste on the row can no longer leak into the prompt bar. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019tqixmvHDK9mB8Rg2X5qCs --- CHANGELOG.md | 20 ++ README.md | 6 +- docs/superpowers/OPEN-ITEMS.md | 18 ++ src/proofpath/fetch.py | 43 +++- src/proofpath/report.py | 5 + src/proofpath/sarif.py | 16 +- src/proofpath/secrets.py | 129 +++++++++- src/proofpath/tui/app.py | 7 + src/proofpath/tui/widgets/config_panel.py | 222 ++++++++++++++++- src/proofpath/verify.py | 5 + tests/data/verify-report-golden.json | 4 +- tests/test_fetch.py | 110 +++++++++ tests/test_sarif.py | 20 ++ tests/test_secrets.py | 191 +++++++++++++++ tests/test_tui_app.py | 282 +++++++++++++++++++++- tests/test_verify.py | 33 +++ tests/test_verify_search.py | 19 ++ 17 files changed, 1107 insertions(+), 23 deletions(-) create mode 100644 tests/test_secrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fb536..8cd29bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,26 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +### Added +- **`Tavily API:` in `/config`.** The search section of the settings panel has a key + row: press Enter, paste the key, Enter again. It is saved to the `.env` in your + config directory (mode 0600 on macOS/Linux), never to `config.toml` or the project's + `.env`, and it is never shown: the row says only whether a key is saved and where. + Backspace removes it. + +### Fixed +- **Wikipedia pages no longer read as `UNVERIFIED (blocked)`.** The bot-wall check + matched `captcha` inside MediaWiki's inline configuration script, so every Wikipedia + page was reported blocked at all four ladder steps despite answering 200. Challenge + phrases are now matched outside `' + b"

Istanbul

" + ARTICLE_PROSE.encode() + b"

" +) +# A long real article whose visible text legitimately mentions a CAPTCHA. +CAPTCHA_ARTICLE = ( + b"

GPT-4 and the TaskRabbit worker

" + b"

During testing the model asked a human worker to solve a CAPTCHA for it.

" + + ARTICLE_PROSE.encode() + + b"

" +) +TITLE_ONLY_CHALLENGE = b"Just a moment..." +CF_CHL_IN_SCRIPT = ( + b"" + b"

Checking your browser before accessing the site.

" +) +# Walls whose only marker trace is a script's src attribute (DataDome, reCAPTCHA): +# the opening tag must survive the script-content removal. +DATADOME_WALL = ( + b'' + b"

Please enable JS and disable any ad blocker

" +) +RECAPTCHA_SRC_WALL = ( + b'' + b"

Please verify you are a human

" +) +# A custom element whose name starts with "style" is not a

Hello there

", "ok"), (503, "html", CHALLENGE, "blocked"), (503, "html", b"

maintenance

", "retryable"), (401, "html", b"", "blocked"), @@ -287,6 +336,46 @@ def test_classify(status: int | None, kind: fx.Kind, body: bytes, expected: str) assert fx.classify(status, kind, body) == expected +def test_article_prose_is_past_the_challenge_word_limit() -> None: + assert fx._html_words(WIKIPEDIA_PAGE) >= fx.CHALLENGE_MAX_WORDS + assert fx._html_words(CAPTCHA_ARTICLE) >= fx.CHALLENGE_MAX_WORDS + + +def test_script_cut_by_the_scan_boundary_does_not_leak_a_marker() -> None: + """A " + assert body.index(b"captcha") < fx.CHALLENGE_SCAN_BYTES < body.index(b"") + assert not fx._has_challenge(body) + assert fx.classify(200, "html", body) == "ok" + + +def test_unterminated_script_at_the_end_is_ignored() -> None: + body = b"

Short page.

" b"

Checking your browser before accessing the site.

" ) +# Cloudflare's underscore forms of the same token: the challenge options object in +# an inline script, and the challenge token in a form action. +CF_CHL_OPT_SCRIPT = b"" +CF_CHL_FORM_TOKEN = ( + b'
' + b"

One more step

" +) +# Real articles that still carry a Cloudflare URL token after the challenge was +# passed: the clearance redirect echoed in og:url, and an outbound link copied +# from a challenged URL. Long pages, so the token must not read as a wall. +CF_CHL_RT_TOKEN_ARTICLE = ( + b'' + b"

The strait

" + ARTICLE_PROSE.encode() + b"

" + b"" +) +CF_CHL_JSCHL_LINK_ARTICLE = ( + b"

The strait

" + ARTICLE_PROSE.encode() + b"

" + b'

See the source.

' + b"
" +) # Walls whose only marker trace is a script's src attribute (DataDome, reCAPTCHA): # the opening tag must survive the script-content removal. DATADOME_WALL = ( @@ -309,6 +330,10 @@ def test_corrupt_pdf_is_noted_not_crashed(client: httpx.Client) -> None: (200, "html", TITLE_ONLY_CHALLENGE, "blocked"), # the phrase lives in (200, "html", b"<noscript>Enable JavaScript and cookies</noscript>", "blocked"), (200, "html", CF_CHL_IN_SCRIPT, "blocked"), # structural marker, scripts included + (200, "html", CF_CHL_OPT_SCRIPT, "blocked"), + (200, "html", CF_CHL_FORM_TOKEN, "blocked"), # visible words: the marker decides + (200, "html", CF_CHL_RT_TOKEN_ARTICLE, "ok"), # URL token on a long page + (200, "html", CF_CHL_JSCHL_LINK_ARTICLE, "ok"), (200, "html", DATADOME_WALL, "blocked"), # marker in a script's src attribute (200, "html", RECAPTCHA_SRC_WALL, "blocked"), (200, "html", CUSTOM_ELEMENT_WALL, "blocked"), # <style-guide> is not <style> @@ -339,6 +364,8 @@ def test_classify(status: int | None, kind: fx.Kind, body: bytes, expected: str) def test_article_prose_is_past_the_challenge_word_limit() -> None: assert fx._html_words(WIKIPEDIA_PAGE) >= fx.CHALLENGE_MAX_WORDS assert fx._html_words(CAPTCHA_ARTICLE) >= fx.CHALLENGE_MAX_WORDS + assert fx._html_words(CF_CHL_RT_TOKEN_ARTICLE) >= fx.CHALLENGE_MAX_WORDS + assert fx._html_words(CF_CHL_JSCHL_LINK_ARTICLE) >= fx.CHALLENGE_MAX_WORDS def test_script_cut_by_the_scan_boundary_does_not_leak_a_marker() -> None: @@ -368,6 +395,11 @@ def test_unterminated_script_at_the_end_is_ignored() -> None: (RECAPTCHA_SRC_WALL, True), (CUSTOM_ELEMENT_WALL, True), (b"<script-x></script-x><p>Access Denied</p>", True), + (CF_CHL_IN_SCRIPT, True), + (CF_CHL_OPT_SCRIPT, True), # underscore form, inside a script + (CF_CHL_FORM_TOKEN, True), # underscore form, in a form action + (CF_CHL_RT_TOKEN_ARTICLE, False), + (CF_CHL_JSCHL_LINK_ARTICLE, False), (WIKIPEDIA_PAGE, False), (CAPTCHA_ARTICLE, False), ], diff --git a/tests/test_sarif.py b/tests/test_sarif.py index 57bff55..2aca670 100644 --- a/tests/test_sarif.py +++ b/tests/test_sarif.py @@ -28,6 +28,7 @@ FOUND_BY_PROOFPATH, LEVELS, STATE_WORDS, + UNDERSPECIFIED, Coverage, Finding, Kind, @@ -250,9 +251,24 @@ def test_a_verdict_judged_against_an_abstract_says_so_in_its_message() -> None: assert FOUND_BY_PROOFPATH not in text +def test_an_underspecified_claim_says_so_in_its_message_after_the_abstract_line() -> None: + # OPEN-ITEMS 19.1: a viewer shows only this line, so a verdict on a claim that + # states too little says so there, after the abstract line; provenance stays out. + item = dataclasses.replace( + finding(Kind.NOT_SUPPORTED, line=1), + detail=(ABSTRACT_BASIS, UNDERSPECIFIED, FOUND_BY_PROOFPATH), + ) + text = to_sarif(report_with(item), artifact="draft.md")["runs"][0]["results"][0]["message"][ + "text" + ] + assert text.endswith(f" — {ABSTRACT_BASIS} — {UNDERSPECIFIED}") + assert FOUND_BY_PROOFPATH not in text + + def test_a_full_text_verdict_message_has_no_abstract_marker() -> None: result = results_by_kind(to_sarif(full_report(), artifact="draft.md"))[Kind.NOT_SUPPORTED.value] assert ABSTRACT_BASIS not in result["message"]["text"] + assert UNDERSPECIFIED not in result["message"]["text"] # --- locations ---------------------------------------------------------------------- diff --git a/tests/test_tui_app.py b/tests/test_tui_app.py index eb1d3a9..c64bc5c 100644 --- a/tests/test_tui_app.py +++ b/tests/test_tui_app.py @@ -65,7 +65,12 @@ from proofpath.tui.banner import split_hint from proofpath.tui.runs import Run, State from proofpath.tui.theme import PLAIN, RICH -from proofpath.tui.widgets.config_panel import ConfigPanel, PanelLine +from proofpath.tui.widgets.config_panel import ( + KEY_NEEDED, + KEY_UNUSED, + ConfigPanel, + PanelLine, +) from proofpath.tui.widgets.prompt import HELD_SUMMARY from proofpath.verify import Engine @@ -3095,6 +3100,9 @@ async def test_the_rows_come_in_the_configs_order_and_the_selection_stays_inside TAVILY = "Tavily API:" PASTED = "tvly-secret-for-test" +# A key with search off is found but unused; the row says so (saving never enables +# search: setting search.provider is the user's consent to it). +UNUSED = f" {PLAIN.glyphs.sep} {KEY_UNUSED}" def _key_note(verb: str, name: str = "TAVILY_API_KEY") -> str: @@ -3149,7 +3157,8 @@ async def test_a_pasted_key_is_saved_to_the_config_dirs_env_and_never_shown( await until(pilot, lambda: _key_note("saved to") in _notes(app), "the saved note") assert not panel.editing and app.focused is panel row = _panel_row(panel, TAVILY) - assert row == f"> {TAVILY:<18}saved {PLAIN.glyphs.sep} {user_dotenv_path()}" + # Saved, and still unused: search.provider is off until the user sets it. + assert row == f"> {TAVILY:<18}saved {PLAIN.glyphs.sep} {user_dotenv_path()}{UNUSED}" shown = _panel_text(panel) # Enter again: the edit starts empty, not holding the key it just saved. again = await open_key_edit(pilot, panel) @@ -3193,7 +3202,7 @@ async def test_backspace_on_the_key_row_removes_the_saved_key() -> None: app, _ = build_app() async with app.run_test(size=SIZE) as pilot: panel = await open_panel(pilot) - assert _panel_row(panel, TAVILY).endswith(f"saved {PLAIN.glyphs.sep} {path}") + assert f"saved {PLAIN.glyphs.sep} {path} " in _panel_row(panel, TAVILY) await select_key_row(pilot, panel) await pilot.press("backspace") await until(pilot, lambda: _key_note("removed from") in _notes(app), "the removed note") @@ -3212,7 +3221,7 @@ async def test_a_key_in_the_environment_is_named_as_such_and_not_shown( panel = await open_panel(pilot) row = _panel_row(panel, TAVILY) text = _panel_text(panel) - assert row.endswith("set in the environment (it overrides the file)") + assert "set in the environment (it overrides the file)" in row assert PASTED not in text @@ -3334,7 +3343,7 @@ async def test_a_saved_key_shadowed_by_the_project_env_says_so( row = _panel_row(panel, TAVILY) text = _panel_text(panel) sep = PLAIN.glyphs.sep - assert row.endswith(f"saved {sep} {user} {sep} overridden by {tmp_path / '.env'}") + assert f"saved {sep} {user} {sep} overridden by {tmp_path / '.env'} " in row assert PASTED not in text and "project-key-value" not in text @@ -3350,12 +3359,81 @@ async def test_a_saved_key_shadowed_by_the_environment_says_so( panel = await open_panel(pilot) row = _panel_row(panel, TAVILY) text = _panel_text(panel) - assert row.endswith( - f"saved {PLAIN.glyphs.sep} {user} {PLAIN.glyphs.sep} overridden by the environment" - ) + sep = PLAIN.glyphs.sep + assert f"saved {sep} {user} {sep} overridden by the environment " in row assert PASTED not in text and "env-key-value" not in text +def _save_tavily_key() -> None: + user = user_dotenv_path() + user.parent.mkdir(parents=True, exist_ok=True) + user.write_text(f"TAVILY_API_KEY={PASTED}\n", encoding="utf-8") + + +def _set_search_provider(provider: str) -> None: + path = config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f'[search]\nprovider = "{provider}"\n', encoding="utf-8") + + +@pytest.mark.usefixtures("no_keys") +async def test_a_saved_key_with_search_off_says_the_provider_must_be_set() -> None: + _save_tavily_key() + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + row = _panel_row(panel, TAVILY) + text = _panel_text(panel) + sep = PLAIN.glyphs.sep + assert row == f" {TAVILY:<18}saved {sep} {user_dotenv_path()}{UNUSED}", row + assert PASTED not in text + assert load_config().search.provider == "off" # saving never turned search on + + +@pytest.mark.usefixtures("no_keys") +async def test_a_saved_key_with_tavily_selected_carries_no_notice() -> None: + _save_tavily_key() + _set_search_provider("tavily") + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + row = _panel_row(panel, TAVILY) + assert row == f" {TAVILY:<18}saved {PLAIN.glyphs.sep} {user_dotenv_path()}", row + + +@pytest.mark.usefixtures("no_keys") +async def test_tavily_selected_without_a_key_says_search_needs_one() -> None: + _set_search_provider("tavily") + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + row = _panel_row(panel, TAVILY) + sep = PLAIN.glyphs.sep + assert row == f" {TAVILY:<18}(not set) {sep} {KEY_NEEDED}", row + assert len(row) <= 80 - 6, "the line fits an 80-column panel without wrapping" + + +@pytest.mark.usefixtures("no_keys") +async def test_cycling_the_search_provider_redraws_the_key_row() -> None: + _save_tavily_key() + app, _ = build_app() + async with app.run_test(size=SIZE) as pilot: + panel = await open_panel(pilot) + assert _panel_row(panel, TAVILY).endswith(UNUSED) + index = next(i for i, row in enumerate(panel.rows) if row.key == "search.provider") + await pilot.press(*["down"] * index) + await pilot.press("right") # off -> tavily + await written(pilot, "search.provider", "tavily") + with_tavily = _panel_row(panel, TAVILY) + await pilot.press("right") # tavily -> searxng + await written(pilot, "search.provider", "searxng") + with_searxng = _panel_row(panel, TAVILY) + text = _panel_text(panel) + assert with_tavily.endswith(f"saved {PLAIN.glyphs.sep} {user_dotenv_path()}"), with_tavily + assert with_searxng.endswith(f"saved {PLAIN.glyphs.sep} {user_dotenv_path()}{UNUSED}") + assert PASTED not in text + + @pytest.mark.usefixtures("no_keys") async def test_a_choice_row_writes_at_once_and_wraps() -> None: app, _ = build_app() diff --git a/tests/test_tui_rich.py b/tests/test_tui_rich.py index 2847e84..4333c70 100644 --- a/tests/test_tui_rich.py +++ b/tests/test_tui_rich.py @@ -937,6 +937,7 @@ async def test_the_config_panel_marks_the_row_badges_the_value_and_wears_the_acc # and looks for a real key, so both are pointed at the test's own directory. monkeypatch.setenv("PROOFPATH_CONFIG_DIR", str(tmp_path / "conf")) monkeypatch.delenv("GROQ_API_KEY", raising=False) + monkeypatch.delenv("TAVILY_API_KEY", raising=False) monkeypatch.chdir(tmp_path) app, _ = rich_app() async with app.run_test(size=SIZE) as pilot: diff --git a/tests/test_verify_search.py b/tests/test_verify_search.py index 81663da..d215c2c 100644 --- a/tests/test_verify_search.py +++ b/tests/test_verify_search.py @@ -22,6 +22,7 @@ FOUND_BY_PROOFPATH, LANGUAGE_UNSUPPORTED, SEARCH_UNAVAILABLE, + UNDERSPECIFIED, Kind, ) from proofpath.resolve import ResolveResult, State @@ -268,6 +269,61 @@ def test_a_verdict_from_a_full_text_found_page_has_no_abstract_line() -> None: assert found.detail == (FOUND_BY_PROOFPATH,) +# --- OPEN-ITEMS 19.1: a claim that states too little ----------------------------- + +WON = "openai won" +AWARD = "OpenAI's recognition with a 2026 Global Recognition Award was announced." + + +def test_a_found_verdict_on_an_underspecified_claim_says_so() -> None: + """The live case: the passage entails "openai won", but so would any win. The + verdict and its tier stand; the row says how little the claim pinned down.""" + report = verify( + WON, + searching( + StubSearcher({"openai": [PAGE_A]}), + pages={PAGE_A: f"{AWARD} {FILLER}"}, + table={AWARD: SUPPORTED_ROW}, + ), + ) + found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND) + assert found.detail == (UNDERSPECIFIED, FOUND_BY_PROOFPATH) + assert found.verdict is not None and found.tier == found.verdict.tier + + +def test_a_specific_claim_carries_no_underspecified_line() -> None: + report = verify(CLAIM, _supported_engine()) + found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND) + assert UNDERSPECIFIED not in found.detail + + +def test_the_underspecified_line_sits_after_the_abstract_line_and_before_provenance() -> None: + report = verify( + WON, + searching( + StubSearcher({"openai": [PAGE_A]}), + pages={PAGE_A: AWARD}, + table={AWARD: REFUTED_ROW}, + ), + ) + refuted = next(f for f in report.findings if f.kind is Kind.NOT_SUPPORTED) + assert refuted.detail == (ABSTRACT_BASIS, UNDERSPECIFIED, FOUND_BY_PROOFPATH) + + +def test_a_translated_claim_is_judged_underspecified_on_its_english_form() -> None: + kazandi = "OpenAI kazandı." # noqa: RUF001 - a real Turkish letter, not a typo + answer = json.dumps({"items": [{"id": 0, "english": "OpenAI won.", "queries": ["OpenAI won"]}]}) + built = searching( + StubSearcher({"OpenAI won": [PAGE_A]}), + pages={PAGE_A: f"{AWARD} {FILLER}"}, + table={AWARD: SUPPORTED_ROW}, + ) + built.judge = Judge(FakeJudgeClient([answer])) # type: ignore[arg-type] + report = verify(kazandi, built) + found = next(f for f in report.findings if f.kind is Kind.EVIDENCE_FOUND) + assert found.detail == ("checked as: OpenAI won.", UNDERSPECIFIED, FOUND_BY_PROOFPATH) + + def own_page(url: str, text: str) -> Fetched: """A page fetched as the *target* itself, read straight from its text (no HTML parsing needed): its own host must never be counted as a source for it."""