From b3fa5b845ca46e1839f98085afbfa8802e833162 Mon Sep 17 00:00:00 2001 From: kunnalsinngh14 Date: Mon, 24 Aug 2026 19:07:24 +0530 Subject: [PATCH 1/3] fixed #1744 --- .../cli/commands/update_cmd/command.py | 17 +- .../cli/commands/update_cmd/deterministic.py | 51 ++++++ .../src/repowise/core/workspace/update.py | 12 ++ .../cli/test_update_stale_reconciliation.py | 147 ++++++++++++++++++ 4 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 tests/unit/cli/test_update_stale_reconciliation.py diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py index 67161b044..2a7303e7d 100644 --- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py +++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py @@ -819,12 +819,19 @@ def run_update( if not dry_run: _repair_module_attribution(repo_path) + # Stale structural pages (e.g. file_page rows marked stale or expired) in + # the DB must be reconciled even when HEAD has not moved. + from .deterministic import load_stale_structural_file_paths + + stale_db_paths = load_stale_structural_file_paths(repo_path) + if ( head and head == base_ref and not config_changed and not renderer_changed and not working_tree_diffs + and not stale_db_paths ): console.print("[green]Already up to date.[/green]") # D7: on a template (index-only) wiki, "up to date" is true of the code @@ -1198,9 +1205,13 @@ def run_update( # and no model will come along later to fix it. Appended rather than merged # so the cascade's own ordering is preserved. stale_renderer_paths = _stale_renderer_paths(repo_path, parsed_files) - if stale_renderer_paths: - affected.regenerate = list(dict.fromkeys([*affected.regenerate, *stale_renderer_paths])) - console.print(f"Pages from an older renderer: [cyan]{len(stale_renderer_paths)}[/cyan]") + stale_extra = list(dict.fromkeys([*stale_renderer_paths, *stale_db_paths])) + if stale_extra: + affected.regenerate = list(dict.fromkeys([*affected.regenerate, *stale_extra])) + if stale_renderer_paths: + console.print(f"Pages from an older renderer: [cyan]{len(stale_renderer_paths)}[/cyan]") + if stale_db_paths and not stale_renderer_paths: + console.print(f"Reconciling stale structural pages: [cyan]{len(stale_db_paths)}[/cyan]") console.print(f"Pages to regenerate: [cyan]{len(affected.regenerate)}[/cyan]") if affected.decay_only: diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py index 3c2d26dfb..179f0cf7a 100644 --- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py +++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py @@ -428,3 +428,54 @@ async def _load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]: return {} finally: await engine.dispose() + + +def load_stale_structural_file_paths(repo_path: Path) -> list[str]: + """Return file paths of structural pages currently marked stale or expired in DB. + + Covers file_page rows and other structural pages whose freshness_status is 'stale' + or 'expired'. Extracting their file paths allows `repowise update` to reconcile + lingering stale structural pages even when the repo is already at HEAD. + """ + return run_async(_load_stale_structural_file_paths(repo_path)) + + +async def _load_stale_structural_file_paths(repo_path: Path) -> list[str]: + from repowise.cli.helpers import get_db_url_for_repo + from repowise.core.persistence import create_engine, create_session_factory, get_session + + db_path = repo_path / ".repowise" / "wiki.db" + if not db_path.exists(): + return [] + + engine = create_engine(get_db_url_for_repo(repo_path)) + try: + from sqlalchemy import select as sa_select + + from repowise.core.cost_estimator import STRUCTURAL_PAGE_TYPES + from repowise.core.persistence.models import Page + + async with get_session(create_session_factory(engine)) as session: + rows = await session.execute( + sa_select(Page.id, Page.page_type, Page.target_path).where( + Page.freshness_status.in_(["stale", "expired"]) + ) + ) + stale_paths: list[str] = [] + for pid, page_type, target_path in rows: + if page_type in STRUCTURAL_PAGE_TYPES or page_type == "file_page": + if page_type == "file_page" and target_path: + stale_paths.append(target_path) + elif pid.startswith("file_page:"): + stale_paths.append(pid[len("file_page:") :]) + elif target_path: + file_path = target_path.split("::", 1)[0] + if file_path: + stale_paths.append(file_path) + return list(dict.fromkeys(stale_paths)) + except Exception as exc: + console.print(f"[yellow]Could not read stale structural pages: {exc}[/yellow]") + return [] + finally: + await engine.dispose() + diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py index 2320946ed..295cce831 100644 --- a/packages/core/src/repowise/core/workspace/update.py +++ b/packages/core/src/repowise/core/workspace/update.py @@ -267,6 +267,18 @@ def check_repo_staleness( return True, current_head, 0 if current_head == last_commit: + db_path = repo_path / ".repowise" / "wiki.db" + if db_path.exists(): + try: + from repowise.cli.commands.update_cmd.deterministic import ( + load_stale_structural_file_paths, + ) + + stale_db_paths = load_stale_structural_file_paths(repo_path) + if stale_db_paths: + return True, current_head, 0 + except Exception: + pass return False, current_head, 0 behind = count_commits_between(repo_path, last_commit, current_head) diff --git a/tests/unit/cli/test_update_stale_reconciliation.py b/tests/unit/cli/test_update_stale_reconciliation.py new file mode 100644 index 000000000..a2b6a6c98 --- /dev/null +++ b/tests/unit/cli/test_update_stale_reconciliation.py @@ -0,0 +1,147 @@ +"""Tests for stale structural page reconciliation during repowise update and workspace staleness checks.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from repowise.cli.commands.doctor_cmd import repo_checks +from repowise.cli.commands.update_cmd.deterministic import load_stale_structural_file_paths +from repowise.core.workspace.update import check_repo_staleness + + +async def _setup_repo_with_pages( + tmp_path: Path, stale_paths: list[str] +) -> tuple[Path, str]: + """Helper to initialize a git repo with a DB and pages (some of which are marked stale).""" + import git as gitpython + + from repowise.core.persistence import ( + create_engine, + create_session_factory, + get_session, + init_db, + upsert_page, + upsert_repository, + ) + + repo_path = (tmp_path / "test_repo").resolve() + repo_path.mkdir(parents=True, exist_ok=True) + git_repo = gitpython.Repo.init(repo_path) + + # Commit a dummy file + dummy = repo_path / "foo.py" + dummy.write_text("def foo(): pass\n") + git_repo.index.add(["foo.py"]) + commit = git_repo.index.commit("Initial commit") + head_sha = commit.hexsha + + repowise_dir = repo_path / ".repowise" + repowise_dir.mkdir(exist_ok=True) + + engine = create_engine(f"sqlite+aiosqlite:///{repowise_dir / 'wiki.db'}") + await init_db(engine) + sf = create_session_factory(engine) + + async with get_session(sf) as session: + repo = await upsert_repository( + session, name="test_repo", local_path=str(repo_path), url="https://example.test/repo" + ) + for path in ["foo.py", "bar.py"]: + page_id = f"file_page:{path}" + freshness = "stale" if path in stale_paths else "fresh" + await upsert_page( + session, + page_id=page_id, + repository_id=repo.id, + page_type="file_page", + title=f"File: {path}", + content="def code(): pass", + summary="Summary", + target_path=path, + source_hash="hash", + model_name="mock", + provider_name="mock", + freshness_status=freshness, + ) + # Also test symbol spotlight structural page with target_path using '::' + if "spotlight.py" in stale_paths: + await upsert_page( + session, + page_id="symbol_spotlight:spotlight.py::my_func", + repository_id=repo.id, + page_type="symbol_spotlight", + title="Spotlight: my_func", + content="def my_func(): pass", + summary="Summary", + target_path="spotlight.py::my_func", + source_hash="hash", + model_name="mock", + provider_name="mock", + freshness_status="stale", + ) + await session.commit() + + await engine.dispose() + return repo_path, head_sha + + +def test_load_stale_structural_file_paths(tmp_path: Path) -> None: + """load_stale_structural_file_paths returns file paths for file_pages marked stale or expired.""" + repo_path, _ = asyncio.run(_setup_repo_with_pages(tmp_path, ["bar.py"])) + + stale = load_stale_structural_file_paths(repo_path) + assert stale == ["bar.py"] + + +def test_load_stale_structural_file_paths_spotlight(tmp_path: Path) -> None: + """load_stale_structural_file_paths extracts file path from symbol_spotlight target_path.""" + repo_path, _ = asyncio.run(_setup_repo_with_pages(tmp_path, ["spotlight.py"])) + + stale = load_stale_structural_file_paths(repo_path) + assert stale == ["spotlight.py"] + + +def test_load_stale_structural_file_paths_empty_when_all_fresh(tmp_path: Path) -> None: + """load_stale_structural_file_paths returns empty list when all pages are fresh.""" + repo_path, _ = asyncio.run(_setup_repo_with_pages(tmp_path, [])) + + stale = load_stale_structural_file_paths(repo_path) + assert stale == [] + + +def test_check_repo_staleness_detects_stale_structural_pages(tmp_path: Path) -> None: + """check_repo_staleness returns is_stale=True when current_head == last_commit but DB has stale pages.""" + repo_path, head_sha = asyncio.run(_setup_repo_with_pages(tmp_path, ["bar.py"])) + + is_stale, current_head, behind = check_repo_staleness(repo_path, head_sha) + assert is_stale is True + assert current_head == head_sha + assert behind == 0 + + +def test_check_repo_staleness_returns_false_when_all_fresh(tmp_path: Path) -> None: + """check_repo_staleness returns is_stale=False when current_head == last_commit and no pages are stale.""" + repo_path, head_sha = asyncio.run(_setup_repo_with_pages(tmp_path, [])) + + is_stale, current_head, behind = check_repo_staleness(repo_path, head_sha) + assert is_stale is False + assert current_head == head_sha + assert behind == 0 + + +def test_doctor_detects_stale_pages_and_clears_after_reconciliation(tmp_path: Path) -> None: + """Doctor check flags stale pages, and load_stale_structural_file_paths identifies them.""" + repo_path, _ = asyncio.run(_setup_repo_with_pages(tmp_path, ["foo.py", "bar.py"])) + + # 1. Doctor initially reports stale pages + all_ok, checks = repo_checks._run_repo_checks(repo_path, repair=False, fmt="quiet") + stale_check = next(c for c in checks if c.name == "Stale pages") + assert stale_check.ok is False + assert stale_check.detail == "2 stale" + + # 2. Verify load_stale_structural_file_paths returns the 2 stale paths + stale_paths = load_stale_structural_file_paths(repo_path) + assert sorted(stale_paths) == ["bar.py", "foo.py"] From 9d3b938a9174249968d56599ca365420e42592a8 Mon Sep 17 00:00:00 2001 From: kunnalsinngh14 Date: Sat, 29 Aug 2026 18:38:16 +0530 Subject: [PATCH 2/3] fixed #1744 --- .../cli/commands/update_cmd/command.py | 16 ++-- .../cli/commands/update_cmd/deterministic.py | 52 +---------- .../src/repowise/core/persistence/__init__.py | 2 + .../repowise/core/persistence/crud/pages.py | 92 +++++++++++++++++++ .../src/repowise/core/workspace/update.py | 18 ++-- .../cli/test_update_stale_reconciliation.py | 78 +++++++++++++++- 6 files changed, 189 insertions(+), 69 deletions(-) diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py index 2a7303e7d..5ec01e494 100644 --- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py +++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py @@ -819,21 +819,21 @@ def run_update( if not dry_run: _repair_module_attribution(repo_path) - # Stale structural pages (e.g. file_page rows marked stale or expired) in - # the DB must be reconciled even when HEAD has not moved. - from .deterministic import load_stale_structural_file_paths - - stale_db_paths = load_stale_structural_file_paths(repo_path) - + stale_db_paths: list[str] = [] if ( head and head == base_ref and not config_changed and not renderer_changed and not working_tree_diffs - and not stale_db_paths ): - console.print("[green]Already up to date.[/green]") + # Stale structural pages (e.g. file_page rows marked stale or expired) in + # the DB must be reconciled even when HEAD has not moved. + from repowise.core.persistence import load_stale_structural_file_paths + + stale_db_paths = load_stale_structural_file_paths(repo_path) + if not stale_db_paths: + console.print("[green]Already up to date.[/green]") # D7: on a template (index-only) wiki, "up to date" is true of the code # but the pages are still unwritten. Point at the command that writes # them rather than leaving the user at a dead end, the way `update diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py index 179f0cf7a..db4be2cb6 100644 --- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py +++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py @@ -430,52 +430,6 @@ async def _load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]: await engine.dispose() -def load_stale_structural_file_paths(repo_path: Path) -> list[str]: - """Return file paths of structural pages currently marked stale or expired in DB. - - Covers file_page rows and other structural pages whose freshness_status is 'stale' - or 'expired'. Extracting their file paths allows `repowise update` to reconcile - lingering stale structural pages even when the repo is already at HEAD. - """ - return run_async(_load_stale_structural_file_paths(repo_path)) - - -async def _load_stale_structural_file_paths(repo_path: Path) -> list[str]: - from repowise.cli.helpers import get_db_url_for_repo - from repowise.core.persistence import create_engine, create_session_factory, get_session - - db_path = repo_path / ".repowise" / "wiki.db" - if not db_path.exists(): - return [] - - engine = create_engine(get_db_url_for_repo(repo_path)) - try: - from sqlalchemy import select as sa_select - - from repowise.core.cost_estimator import STRUCTURAL_PAGE_TYPES - from repowise.core.persistence.models import Page - - async with get_session(create_session_factory(engine)) as session: - rows = await session.execute( - sa_select(Page.id, Page.page_type, Page.target_path).where( - Page.freshness_status.in_(["stale", "expired"]) - ) - ) - stale_paths: list[str] = [] - for pid, page_type, target_path in rows: - if page_type in STRUCTURAL_PAGE_TYPES or page_type == "file_page": - if page_type == "file_page" and target_path: - stale_paths.append(target_path) - elif pid.startswith("file_page:"): - stale_paths.append(pid[len("file_page:") :]) - elif target_path: - file_path = target_path.split("::", 1)[0] - if file_path: - stale_paths.append(file_path) - return list(dict.fromkeys(stale_paths)) - except Exception as exc: - console.print(f"[yellow]Could not read stale structural pages: {exc}[/yellow]") - return [] - finally: - await engine.dispose() - +from repowise.core.persistence import ( + load_stale_structural_file_paths as load_stale_structural_file_paths, +) diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.py index 52e64d5e3..31f2131d1 100644 --- a/packages/core/src/repowise/core/persistence/__init__.py +++ b/packages/core/src/repowise/core/persistence/__init__.py @@ -61,6 +61,8 @@ get_scc_members, get_stale_decisions, get_stale_pages, + get_stale_structural_file_paths, + load_stale_structural_file_paths, get_top_entry_points, link_graph_nodes_to_external_systems, list_chat_messages, diff --git a/packages/core/src/repowise/core/persistence/crud/pages.py b/packages/core/src/repowise/core/persistence/crud/pages.py index 8a60a1e4c..eb1a18a86 100644 --- a/packages/core/src/repowise/core/persistence/crud/pages.py +++ b/packages/core/src/repowise/core/persistence/crud/pages.py @@ -659,3 +659,95 @@ async def get_stale_pages( ) ) return list(result.scalars().all()) + + +async def get_stale_structural_file_paths( + session: AsyncSession, + repository_id: str, +) -> list[str]: + """Return file paths of structural pages currently marked stale or expired in DB for *repository_id*. + + Covers file_page rows and other structural pages whose freshness_status is 'stale' + or 'expired'. Extracting their file paths allows update commands to reconcile + lingering stale structural pages even when the repo is already at HEAD. + """ + from repowise.core.cost_estimator import STRUCTURAL_PAGE_TYPES + + result = await session.execute( + select(Page.id, Page.page_type, Page.target_path).where( + Page.repository_id == repository_id, + Page.freshness_status.in_(["stale", "expired"]), + ) + ) + stale_paths: list[str] = [] + for pid, page_type, target_path in result: + if page_type in STRUCTURAL_PAGE_TYPES or page_type == "file_page": + if page_type == "file_page" and target_path: + stale_paths.append(target_path) + elif pid.startswith("file_page:"): + stale_paths.append(pid[len("file_page:") :]) + elif target_path: + file_path = target_path.split("::", 1)[0] + if file_path: + stale_paths.append(file_path) + return list(dict.fromkeys(stale_paths)) + + +def load_stale_structural_file_paths(repo_path: Any) -> list[str]: + """Sync wrapper around _load_stale_structural_file_paths_async.""" + import asyncio + import concurrent.futures + from pathlib import Path + + path_obj = Path(repo_path) + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit( + lambda: asyncio.run(_load_stale_structural_file_paths_async(path_obj)) + ).result() + return asyncio.run(_load_stale_structural_file_paths_async(path_obj)) + + +async def _load_stale_structural_file_paths_async(repo_path: Any) -> list[str]: + """Load stale structural file paths for *repo_path* from the database.""" + from pathlib import Path + + import structlog + + from ..database import ( + create_engine, + create_session_factory, + get_configured_db_url, + get_repo_db_path, + get_session, + resolve_db_url, + ) + from .repository import get_repository_by_path + + logger = structlog.get_logger(__name__) + path_obj = Path(repo_path) + + # Store reachability check: if no env DB URL is set and local wiki.db doesn't exist, return [] + if get_configured_db_url() is None and not get_repo_db_path(path_obj).exists(): + return [] + + url = resolve_db_url(path_obj) + engine = create_engine(url) + try: + sf = create_session_factory(engine) + async with get_session(sf) as session: + repo = await get_repository_by_path(session, str(path_obj)) + if repo is None: + return [] + return await get_stale_structural_file_paths(session, repo.id) + except Exception as exc: + logger.debug("load_stale_structural_file_paths_failed", error=str(exc)) + return [] + finally: + await engine.dispose() + diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py index 295cce831..03340466a 100644 --- a/packages/core/src/repowise/core/workspace/update.py +++ b/packages/core/src/repowise/core/workspace/update.py @@ -267,18 +267,14 @@ def check_repo_staleness( return True, current_head, 0 if current_head == last_commit: - db_path = repo_path / ".repowise" / "wiki.db" - if db_path.exists(): - try: - from repowise.cli.commands.update_cmd.deterministic import ( - load_stale_structural_file_paths, - ) + try: + from repowise.core.persistence import load_stale_structural_file_paths - stale_db_paths = load_stale_structural_file_paths(repo_path) - if stale_db_paths: - return True, current_head, 0 - except Exception: - pass + stale_db_paths = load_stale_structural_file_paths(repo_path) + if stale_db_paths: + return True, current_head, 0 + except Exception: + pass return False, current_head, 0 behind = count_commits_between(repo_path, last_commit, current_head) diff --git a/tests/unit/cli/test_update_stale_reconciliation.py b/tests/unit/cli/test_update_stale_reconciliation.py index a2b6a6c98..236c02ca7 100644 --- a/tests/unit/cli/test_update_stale_reconciliation.py +++ b/tests/unit/cli/test_update_stale_reconciliation.py @@ -8,7 +8,7 @@ import pytest from repowise.cli.commands.doctor_cmd import repo_checks -from repowise.cli.commands.update_cmd.deterministic import load_stale_structural_file_paths +from repowise.core.persistence import load_stale_structural_file_paths from repowise.core.workspace.update import check_repo_staleness @@ -145,3 +145,79 @@ def test_doctor_detects_stale_pages_and_clears_after_reconciliation(tmp_path: Pa # 2. Verify load_stale_structural_file_paths returns the 2 stale paths stale_paths = load_stale_structural_file_paths(repo_path) assert sorted(stale_paths) == ["bar.py", "foo.py"] + + +def test_stale_structural_paths_scoped_to_repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """load_stale_structural_file_paths filters by repository_id in a shared database.""" + import git as gitpython + + from repowise.core.persistence import ( + create_engine, + create_session_factory, + get_session, + init_db, + upsert_page, + upsert_repository, + ) + + db_file = tmp_path / "shared_wiki.db" + db_url = f"sqlite+aiosqlite:///{db_file}" + monkeypatch.setenv("REPOWISE_DB_URL", db_url) + + repo_a = (tmp_path / "repo_a").resolve() + repo_b = (tmp_path / "repo_b").resolve() + repo_a.mkdir() + repo_b.mkdir() + gitpython.Repo.init(repo_a) + gitpython.Repo.init(repo_b) + + async def _setup_shared_db() -> None: + engine = create_engine(db_url) + await init_db(engine) + sf = create_session_factory(engine) + async with get_session(sf) as session: + r_a = await upsert_repository(session, name="repo_a", local_path=str(repo_a)) + r_b = await upsert_repository(session, name="repo_b", local_path=str(repo_b)) + + # Add stale page for Repo A + await upsert_page( + session, + page_id="file_page:a.py", + repository_id=r_a.id, + page_type="file_page", + title="File: a.py", + content="code", + summary="", + target_path="a.py", + source_hash="", + model_name="mock", + provider_name="mock", + freshness_status="stale", + ) + # Add stale page for Repo B + await upsert_page( + session, + page_id="file_page:b.py", + repository_id=r_b.id, + page_type="file_page", + title="File: b.py", + content="code", + summary="", + target_path="b.py", + source_hash="", + model_name="mock", + provider_name="mock", + freshness_status="stale", + ) + await session.commit() + await engine.dispose() + + asyncio.run(_setup_shared_db()) + + # Query repo A: must only return a.py + stale_a = load_stale_structural_file_paths(repo_a) + assert stale_a == ["a.py"] + + # Query repo B: must only return b.py + stale_b = load_stale_structural_file_paths(repo_b) + assert stale_b == ["b.py"] From d47fe4fc2e83b041d6a13a3257d86a9465a748c6 Mon Sep 17 00:00:00 2001 From: kunnalsinngh14 Date: Sat, 29 Aug 2026 18:54:57 +0530 Subject: [PATCH 3/3] fixed #1744 --- packages/core/src/repowise/core/persistence/crud/pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/repowise/core/persistence/crud/pages.py b/packages/core/src/repowise/core/persistence/crud/pages.py index eb1a18a86..80539cebb 100644 --- a/packages/core/src/repowise/core/persistence/crud/pages.py +++ b/packages/core/src/repowise/core/persistence/crud/pages.py @@ -746,8 +746,8 @@ async def _load_stale_structural_file_paths_async(repo_path: Any) -> list[str]: return [] return await get_stale_structural_file_paths(session, repo.id) except Exception as exc: - logger.debug("load_stale_structural_file_paths_failed", error=str(exc)) - return [] + logger.warning("load_stale_structural_file_paths_failed", error=str(exc)) + raise finally: await engine.dispose()