diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..311059aac --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,42 @@ +name: Helix acceptance benchmark + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Require the pinned public embedded wheel + run: python tools/check_helix_wheel.py + - name: Install isolated benchmark dependencies + run: | + python -m pip install -e . --no-deps + python -m pip install -r requirements-runtime.txt -r benchmarks/requirements-networkx.txt + - name: Run isolated NetworkX behavioral parity + env: + PYTHONPATH: . + run: python benchmarks/parity_networkx.py > parity-result.json + - name: Run 5k/15k and 20k/60k comparison + env: + PYTHONPATH: . + run: python benchmarks/helix_vs_networkx.py --out benchmark-result.json --check-gates + - uses: actions/upload-artifact@v4 + with: + name: helix-vs-networkx-${{ matrix.os }}-${{ github.sha }} + path: | + benchmark-result.json + parity-result.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..b0ac58293 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: Helix native CI on: push: @@ -18,10 +18,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - # The audit-coverage, monolith-roundtrip, and always-on-roundtrip - # validators read blobs from origin/v8. A shallow checkout omits that - # ref, so fetch the full history here and the validators run for real - # (rather than skipping). The other jobs stay shallow. + # The heading audit reads a pinned pre-split v8 commit. A shallow + # checkout can omit it, so fetch full history for this job. fetch-depth: 0 - name: Install uv @@ -47,11 +45,23 @@ jobs: run: uv run --frozen python -m tools.skillgen --always-on-roundtrip test: - runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.10", "3.12"] - + include: + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.12" + - os: ubuntu-24.04-arm + python-version: "3.12" + - os: macos-latest + python-version: "3.12" + - os: windows-latest + python-version: "3.10" + - os: windows-latest + python-version: "3.12" + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 with: @@ -65,6 +75,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Require the pinned public embedded wheel + run: python tools/check_helix_wheel.py + # --frozen installs straight from the committed uv.lock without re-resolving # or rewriting it, so CI never churns the lock. - name: Install dependencies @@ -77,14 +90,38 @@ jobs: run: | uv run --frozen graphify --help uv run --frozen graphify install + uv build + + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.12" + + - run: uv sync --all-extras --frozen + + - name: Ruff + run: uv run --frozen ruff check graphify tests tools benchmarks + + - name: Pyright native integration surface + run: >- + uv run --frozen pyright graphify/helix graphify/operations.py graphify/cli.py + graphify/cluster.py graphify/analyze.py graphify/benchmark.py + graphify/global_graph.py graphify/impact.py graphify/affected.py + graphify/paths.py graphify/prs.py graphify/export.py graphify/watch.py + + - name: Compile and whitespace checks + run: | + uv run --frozen python -m compileall -q graphify + git diff --check security-scan: # The dev deps include bandit and pip-audit. Run them in CI so a new # HIGH-severity finding or vulnerable dependency is caught on the PR that # introduces it, rather than at the next manual audit. - # Non-blocking for now (continue-on-error) to avoid breaking CI on - # pre-existing findings; remove continue-on-error after the initial - # cleanup pass. runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -98,9 +135,10 @@ jobs: run: uv sync --frozen - name: bandit (static security analysis) - continue-on-error: true run: uv run --frozen bandit -r graphify -ll - name: pip-audit (dependency vulnerabilities) - continue-on-error: true - run: uv run --frozen pip-audit --strict + run: | + uv export --quiet --frozen --all-extras --no-dev --no-emit-project \ + --output-file audited-requirements.txt + uv run --frozen pip-audit --strict --requirement audited-requirements.txt diff --git a/.github/workflows/release-graph.yml b/.github/workflows/release-graph.yml index 246ec40d9..b9b19d185 100644 --- a/.github/workflows/release-graph.yml +++ b/.github/workflows/release-graph.yml @@ -37,12 +37,12 @@ jobs: run: uv run --frozen graphify cluster-only . --no-label - name: Generate HTML viewer - run: uv run --frozen graphify export html --graph graphify-out/graph.json + run: uv run --frozen graphify export html --graph graphify-out/graph.helix - name: Bundle release asset run: | mkdir -p dist-graph - cp graphify-out/graph.json dist-graph/ + cp -R graphify-out/graph.helix dist-graph/ cp graphify-out/graph.html dist-graph/ [ -f graphify-out/GRAPH_REPORT.md ] && cp graphify-out/GRAPH_REPORT.md dist-graph/ || true tar -czf graphify-self-graph.tar.gz -C dist-graph . diff --git a/.gitignore b/.gitignore index 0a6775b2a..8674115f1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ .ruff_cache/ *.so *.egg +.DS_Store .graphify/ graphify-out/ .graphify_*.json diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5672bf0df..4a1effe34 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -8,27 +8,53 @@ graphify is a Claude Code skill backed by a Python library. The skill orchestrat detect() → extract() → build_graph() → cluster() → analyze() → report() → export() ``` -Each stage is a single function in its own module. They communicate through plain Python dicts and NetworkX graphs - no shared state, no side effects outside `graphify-out/`. +Extraction produces a transient, record-only `GraphBuildData`; every durable +consumer receives a generation-safe `LoadedGraph` around Helix's native immutable +graph. Embedded, on-disk Helix is the only durable graph/index store; +NetworkX is neither imported nor installed in production. + +A build writes topology to an inactive generation, loads an immutable native +snapshot for clustering and analysis, then stages topology and all durable state +together. Counts and a SHA-256 checksum are verified before the active-generation +pointer changes. A failed or interrupted build leaves the previous generation and +its incremental hashes active. ## Module responsibilities | Module | Function | Input → Output | |--------|----------|----------------| -| `detect.py` | `collect_files(root)` | directory → `[Path]` filtered list | +| `detect.py` | `detect(root)` | directory → typed file lists and Helix-backed incremental state | | `extract.py` | `extract(path)` | file path → `{nodes, edges}` dict | -| `build.py` | `build_graph(extractions)` | list of extraction dicts → `nx.Graph` | -| `cluster.py` | `cluster(G)` | graph → graph with `community` attr on each node | +| `build.py` | `build_from_json(extraction)` | extraction dict → transient record-only `GraphBuildData` | +| `cluster.py` | `cluster(G)` | native snapshot → weighted Leiden membership | | `analyze.py` | `analyze(G)` | graph → analysis dict (god nodes, surprises, questions) | | `report.py` | `render_report(G, analysis)` | graph + analysis → GRAPH_REPORT.md string | -| `export.py` | `export(G, out_dir, ...)` | graph → Obsidian vault, graph.json, graph.html, graph.svg | +| `export.py` | `export(G, out_dir, ...)` | native snapshot → Obsidian vault, GraphML, Cypher, HTML, SVG | | `callflow_html.py` | `write_callflow_html(...)` | graphify-out files → Mermaid architecture/call-flow HTML | | `ingest.py` | `ingest(url, ...)` | URL → file saved to corpus dir | | `cache.py` | `check_semantic_cache / save_semantic_cache` | files → (cached, uncached) split | | `security.py` | validation helpers | URL / path / label → validated or raises | | `validate.py` | `validate_extraction(data)` | extraction dict → raises on schema errors | -| `serve.py` | `start_server(graph_path)` | graph file path → MCP stdio server | -| `watch.py` | `watch(root, flag_path)` | directory → writes flag file on change | -| `benchmark.py` | `run_benchmark(graph_path)` | graph file → corpus vs subgraph token comparison | +| `serve.py` | `serve(store_path)` | active Helix generation → reloadable MCP stdio server | +| `watch.py` | `watch(root)` | changes → atomic embedded Helix rebuild | +| `benchmark.py` | `run_benchmark(store_path)` | active Helix graph → corpus vs subgraph token comparison | + +## Durable generation schema + +Every active generation contains graph metadata, ordered typed nodes and edges, +communities and names, analysis/report inputs, content and semantic hashes, +extraction cache, extractor state, learning/provenance, and semantic-build +metadata. There are no production topology, cache, label, analysis, or learning +sidecars. + +Helix maintains unique equality indexes on `GraphifyNode(storage_key)` and +`GraphifyControl(control_key)`. `storage_key` is the generation plus Helix's +canonical typed external ID, so values such as `1`, `"1"`, and `true` cannot +collide. Semantic relations are native edge labels; they are not duplicated in +the edge attribute blob. + +Human-facing reports and explicit exports remain files. Existing Obsidian +application configuration remains untouched. ## Extraction output schema @@ -69,7 +95,7 @@ All external input passes through `graphify/security.py` before use: - URLs → `validate_url()` (http/https only) + `_NoFileRedirectHandler` (blocks file:// redirects) - Fetched content → `safe_fetch()` / `safe_fetch_text()` (size cap, timeout) -- Graph file paths → `validate_graph_path()` (must resolve inside `graphify-out/`) +- Helix store paths → `validate_store_path()` (must be an existing store directory) - Node labels → `sanitize_label()` (strips control chars, caps 256 chars, HTML-escapes) See `SECURITY.md` for the full threat model. @@ -82,4 +108,8 @@ One test file per module under `tests/`. Run with: pytest tests/ -q ``` -All tests are pure unit tests - no network calls, no file system side effects outside `tmp_path`. +The default suite includes unit, migration, corruption, atomicity, concurrency, +native algorithm, and embedded close/reopen tests. The native tests require the +pinned `helix-db-embedded` wheel. They are skipped in minimal source-only +environments and mandatory in CI. NetworkX parity/performance is isolated under +`benchmarks/` and is never a production dependency. diff --git a/Dockerfile b/Dockerfile index a313833c5..cb213dd73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # # Build: docker build -t graphify . # Run: docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ -# /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +# /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" # # Builds from source so the image includes the Streamable HTTP transport even -# before it lands on PyPI. The graph.json is mounted at runtime (-v), never +# before it lands on PyPI. The native Helix store is mounted at runtime (-v), never # baked into the image. FROM python:3.12-slim @@ -22,4 +22,4 @@ USER graphify EXPOSE 8080 ENTRYPOINT ["python", "-m", "graphify.serve"] -CMD ["/data/graph.json", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"] +CMD ["/data/graph.helix", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md index 0459f57cf..acd967856 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,13 @@ Then, in your AI assistant: /graphify . ``` -That's it. You get **three files**: +That's it. You get **three outputs**: ``` graphify-out/ ├── graph.html open in any browser — click nodes, filter, search ├── GRAPH_REPORT.md the highlights: key concepts, surprising connections, suggested questions -└── graph.json the full graph — query it anytime without re-reading your files +└── graph.helix/ the embedded native graph — query it anytime without re-reading your files ``` **Works in** Claude Code, Cursor, Codex, Gemini CLI, GitHub Copilot, and 15+ more — [pick your platform](#install). @@ -103,7 +103,7 @@ What you get out of the box: | **God nodes** | The most-connected concepts, so you see what everything flows through | | **Communities** | The graph split into subsystems (Leiden), with LLM-free labels | | **Cross-file links** | `calls` / `imports` / `inherits` / `mixes_in` resolved across ~40 languages via tree-sitter AST | -| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against `graph.json` | +| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against the active `graph.helix` generation | | **Rationale + doc refs** | `# NOTE:` / `# WHY:` comments and ADR/RFC citations become first-class nodes linked to the code | | **Beyond code** | Docs, PDFs, images, and video/audio all map into the same graph | | **Local-first** | Code is parsed locally with tree-sitter (no LLM, nothing leaves your machine); only the semantic pass over docs/media calls a backend, and only if you configure one | @@ -136,10 +136,7 @@ Every system ran on the same harness with the same model and budgets, scored by brew install python@3.12 uv ``` -**Windows quick install:** -```powershell -winget install astral-sh.uv -``` +**Windows (PowerShell):** install Python from python.org, then run `py -m pip install graphifyy` or install `uv` and run `uv tool install graphifyy`. Native x86_64 Windows is part of the supported test matrix; installation requires the matching public `helix-db-embedded` wheel. **Ubuntu/Debian:** ```bash @@ -194,7 +191,7 @@ for example `graphify claude install --project` or `graphify codex install --pro > **Running with `uvx` / `uv tool run` instead of installing?** Name the package, not the command: `uvx --from graphifyy graphify install`. Plain `uvx graphify …` fails (`No solution found … no versions of graphify`) because `uv tool run` reads the first word as a *package*, and the package is `graphifyy` — the `graphify` command lives inside it. -> **Avoid `pip install` on Mac/Windows** if possible. The skill resolves Python at runtime from `graphify-out/.graphify_python`; if that points to a different environment than where `pip` installed the package, you'll get `ModuleNotFoundError: No module named 'graphify'`. `uv tool install` and `pipx install` isolate the package in their own env and avoid this entirely. +> **Avoid `pip install` on Mac** if possible. The skill resolves Python at runtime from `graphify-out/.graphify_python`; if that points to a different environment than where `pip` installed the package, you'll get `ModuleNotFoundError: No module named 'graphify'`. `uv tool install` and `pipx install` isolate the package in their own env and avoid this entirely. > **Git hooks and uv tool / pipx:** `graphify hook install` embeds the current interpreter path directly into the hook scripts at install time, so the post-commit hook fires correctly even in GUI git clients and CI runners where `~/.local/bin` is not on PATH. If you reinstall or upgrade graphify, re-run `graphify hook install` to refresh the embedded path. @@ -206,7 +203,7 @@ for example `graphify claude install --project` or `graphify codex install --pro | Platform | Install command | |----------|----------------| | Claude Code (Linux/Mac) | `graphify install` | -| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows` | +| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows`; native graph builds are supported | | CodeBuddy | `graphify install --platform codebuddy` | | Codex | `graphify install --platform codex` | | OpenCode | `graphify install --platform opencode` | @@ -370,7 +367,7 @@ You can also set `GRAPHIFY_GOOGLE_WORKSPACE=1`. Graphify exports shortcuts into /graphify . --cluster-only # rerun clustering without re-extracting /graphify . --cluster-only --resolution 1.5 # more granular communities /graphify . --cluster-only --exclude-hubs 99 # suppress utility super-hubs from god-node rankings -/graphify . --no-viz # skip the HTML, just the report + JSON +/graphify . --no-viz # skip HTML; keep the report + native store /graphify . --wiki # build a markdown wiki from the graph graphify export callflow-html # Mermaid architecture/call-flow HTML (auto-regenerates on every git commit if hook is installed) @@ -382,7 +379,7 @@ graphify export callflow-html # Mermaid architecture/call-flow HTML (auto-r /graphify add # transcribe and add a video graphify hook install # auto-rebuild on git commit -graphify merge-graphs a.json b.json # combine two graphs +graphify global add ./project-a --as project-a # register a project in the native global graph graphify prs # PR dashboard: CI state, review status, worktree mapping graphify prs 42 # deep dive on PR #42 with graph impact @@ -418,20 +415,19 @@ dist/ ## Team setup -`graphify-out/` is meant to be committed to git so everyone on the team starts with a map. +`graphify-out/graph.helix` is a local embedded store. Share source and rebuild it per checkout; do not merge native store internals across branches. **Recommended `.gitignore` additions:** ``` -graphify-out/cost.json # local only -# graphify-out/cache/ # optional: commit for speed, skip to keep repo small +graphify-out/graph.helix/ # local embedded store ``` -> `manifest.json` is now portable — keys are stored as relative paths and re-anchored on load, so committing it is safe and avoids a full rebuild on first checkout. +Hashes, extraction cache, analysis, labels, learning state, and generation metadata all live inside the Helix store. **Workflow:** -1. One person runs `/graphify .` and commits `graphify-out/`. -2. Everyone pulls — their assistant reads the graph immediately. -3. Run `graphify hook install` to auto-rebuild after each commit (AST only, no API cost). This also sets up a git merge driver so `graph.json` is never left with conflict markers — two devs committing in parallel get their graphs union-merged automatically. +1. Each developer runs `/graphify .` after checkout to create the local native store. +2. Run `graphify hook install` to update the active generation after commits. +3. Helix serializes writers and keeps readers on immutable snapshots while a new generation activates. 4. When docs or papers change, run `/graphify --update` to refresh those nodes. --- @@ -441,18 +437,18 @@ graphify-out/cost.json # local only ```bash # query the graph from the terminal graphify query "show the auth flow" -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +graphify query "what connects DigestAuth to Response?" --store graphify-out/graph.helix # expose the graph as an MCP server (for repeated tool-call access) -python -m graphify.serve graphify-out/graph.json -python -m graphify.serve --graph graphify-out/graph.json # --graph flag also accepted +python -m graphify.serve graphify-out/graph.helix +python -m graphify.serve --graph graphify-out/graph.helix # --graph flag also accepted # register with Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix # or serve over HTTP so a whole team points at one URL (no local graphify needed): -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ``` The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. @@ -477,7 +473,7 @@ The default `127.0.0.1` bind is loopback-only. Set `--host 0.0.0.0` **and** `--a ```bash docker build -t graphify . docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ - /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" + /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ``` > **WSL / Linux note:** Ubuntu ships `python3`, not `python`. Use a venv to avoid conflicts: @@ -523,7 +519,6 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_QUERY_LOG` | Enable the query log and write it to this path instead of the default | optional — off unless this or `_ENABLE` is set | | `GRAPHIFY_QUERY_LOG_DISABLE` | Set to `1` to force the query log off (wins over the enable vars) | optional | | `GRAPHIFY_QUERY_LOG_RESPONSES` | When the log is enabled, also record full subgraph responses (off by default) | optional | -| `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | | `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | --- @@ -564,14 +559,14 @@ pip uninstall graphifyy # or remove the old s **`python -m graphify` works but `graphify` command doesn't** Your shell's `PATH` doesn't include the bin directory the command was installed to. Prefer `uv tool install` / `pipx install` over plain `pip`, then run `uv tool update-shell` / `pipx ensurepath` and open a new terminal (see the install notes above). -**`/graphify .` causes "path not recognized" in PowerShell** -PowerShell treats a leading `/` as a path separator. Use `graphify .` (no slash) on Windows. +**Graphify fails to install on Windows** +Windows is supported natively—WSL and downloaded-DLL workarounds are not required or accepted. If pip reports that no compatible `helix-db-embedded` distribution exists, the matching public `win_amd64` wheel has not reached PyPI yet; use the release whose matching public wheel is available. **Graph has fewer nodes after `--update` or rebuild** If a refactor deleted files, the old nodes linger. Pass `--force` (or set `GRAPHIFY_FORCE=1`) to overwrite even when the rebuild has fewer nodes. **`extract` exits with "extraction was incomplete ... refusing to overwrite"** -When an extraction pass crashes or a walk can't fully read the corpus, the run would be smaller than a complete one, so `graphify extract` refuses to overwrite a larger existing graph with the partial result (protecting your `graph.json`). Fix the underlying failure and re-run, or pass `--allow-partial` to overwrite anyway. +When extraction fails, Graphify does not activate the incomplete generation. Fix the underlying failure and rerun; existing readers remain pinned to the prior valid generation. **Graph has duplicate nodes for the same entity (ghost duplicates)** Ghost duplicates (same symbol appearing twice — once from AST extraction with a source location, once from semantic extraction without) are now automatically merged at build time. If you see this in a graph built before v0.8.33, run a full re-extract to clean up: @@ -594,14 +589,14 @@ graphify extract . --mode deep --token-budget 4000 # smaller inpu With a cloud gateway like OpenRouter, prefer `--backend openai` (set `OPENAI_BASE_URL`) over the Ollama shim — it's a cleaner OpenAI-compatible path. If the model has its own max-output ceiling, lowering `--token-budget` is the reliable lever. **Graph HTML is too large to open in a browser (>5000 nodes)** -Skip HTML generation and use the JSON directly: +Skip HTML generation and query the native store directly: ```bash graphify cluster-only ./my-project --no-viz graphify query "..." ``` -**`graph.json` has conflict markers after two devs commit at once** -Run `graphify hook install` — it sets up a git merge driver that union-merges `graph.json` automatically so conflicts never happen. +**A native store was copied or merged between branches** +Rebuild from source. Native stores are generation directories, not mergeable source artifacts. **Extraction returns empty nodes/edges for docs or PDFs** Docs, PDFs, and images require an LLM call — code-only corpora need no key. Check that your API key is set and the backend is correct: @@ -617,10 +612,9 @@ graphify install # overwrites the skill file ``` **Claude Code prompt cache invalidated after every `graphify extract`** -Graphify writes output files (`graph.json`, `graphify-out/`) into the workspace. If those paths aren't ignored, every write invalidates Claude Code's prompt cache, forcing a full re-upload at cache-write rates on the next turn. Add them to `.claudeignore`: +Graphify writes its embedded store and exports under `graphify-out/`. If that path is not ignored, every write can invalidate Claude Code's prompt cache. Add it to `.claudeignore`: ```text # .claudeignore -graph.json graphify-out/ ``` @@ -657,13 +651,17 @@ graphify extract ./raw --code-only # index code only — local AST, no API key ( /graphify query "..." --dfs --budget 1500 /graphify path "DigestAuth" "Response" /graphify explain "SwinTransformer" +graphify impact --base origin/main # PR impact from active Helix generation +graphify update . # reuse embedded extraction records, atomically replace +graphify cluster # rerun native weighted Leiden against active Helix +graphify analyze # refresh durable analysis in one generation graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome useful # record how a Q&A turned out (work memory; outcome ∈ useful|dead_end|corrected) graphify reflect # aggregate graphify-out/memory/ outcomes into reflections/LESSONS.md graphify reflect --if-stale # no-op when LESSONS.md is already newer than every input (cheap to run each session) graphify reflect --out docs/LESSONS.md # write the lessons doc somewhere else -graphify reflect --graph graphify-out/graph.json # group lessons by community + write the work-memory overlay (.graphify_learning.json) - # the overlay tags nodes preferred/tentative/contested (recency-weighted, with provenance); +graphify reflect --graph graphify-out/graph.helix # group lessons by community + commit work-memory state + # native state tags nodes preferred/tentative/contested (recency-weighted, with provenance); # graphify explain / query then show a "Lesson:" hint, flagged "code changed — re-verify" when the source moved on graphify uninstall # remove from all platforms in one shot @@ -736,7 +734,7 @@ graphify extract ./src --no-gitignore # include git-ignored source; sti graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) -graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) +graphify extract ./docs --force # force a full source rescan and activate a new native generation graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora @@ -746,7 +744,7 @@ graphify export callflow-html --max-sections 8 # cap generated architecture graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json --as myrepo # register a project graph into ~/.graphify/global-graph.json +graphify global add graphify-out/graph.helix --as myrepo # register a project store into ~/.graphify/global-graph.helix graphify global remove myrepo # remove a project from the global graph graphify global list # show all registered repos + node/edge counts graphify global path # print path to the global graph file @@ -761,7 +759,6 @@ graphify prs --repo owner/repo # run against a different GitHub repo GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # use a specific backend for triage graphify clone https://github.com/karpathy/nanoGPT -graphify merge-graphs a.json b.json --out merged.json graphify --version # print installed version graphify watch ./src graphify check-update ./src @@ -769,7 +766,7 @@ graphify update ./src graphify update ./src --no-cluster # skip reclustering, write raw AST graph only graphify update ./src --force # overwrite even if new graph has fewer nodes graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location +graphify cluster-only ./my-project --store path/to/graph.helix # custom Helix store directory graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # parallel community labeling (large graphs) graphify cluster-only ./my-project --resolution 1.5 # more, smaller communities graphify cluster-only ./my-project --exclude-hubs 99 # exclude p99 degree nodes from partitioning diff --git a/SECURITY.md b/SECURITY.md index 795454b21..b962c6448 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,7 +38,11 @@ graphify is a **local development tool**. It runs as a Claude Code skill and opt | YAML frontmatter injection | `_yaml_str()` escapes backslashes, double quotes, and newlines before embedding user-controlled strings (webpage titles, query questions) in YAML frontmatter. | | Encoding crashes on source files | All tree-sitter byte slices decoded with `errors="replace"` - non-UTF-8 source files degrade gracefully instead of crashing extraction. | | Symlink traversal | `os.walk(..., followlinks=False)` is explicit throughout `detect.py`. | -| Corrupted graph.json | `_load_graph()` in `serve.py` wraps `json.JSONDecodeError` and prints a clear recovery message instead of crashing. | +| Corrupted or mixed-generation store | Active Helix generations carry counts and SHA-256 checksums; readers reject mismatches before exposing the graph. | +| Partial/interrupted build | Topology and durable state remain inactive until complete verification and one active-generation pointer update. | +| Competing embedded writers | A process lock permits one writer; read-only handles reject every write path. | +| Public or mismatched Helix build | Startup validates the exact pinned SDK revision, version, and native graph API payload and fails closed. | +| MCP HTTP exposure | Streamable HTTP binds only to loopback addresses; non-loopback hosts are rejected. Use a trusted authenticated proxy if remote access is required. | ### What graphify does NOT do @@ -46,6 +50,7 @@ graphify is a **local development tool**. It runs as a Claude Code skill and opt - Does not execute code from source files (tree-sitter parses ASTs - no eval/exec) - Does not use `shell=True` in any subprocess call - Does not store credentials or API keys +- Does not persist Graphify graph/index state in JSON sidecars ### Optional network calls diff --git a/benchmarks/QUALIFICATION.md b/benchmarks/QUALIFICATION.md new file mode 100644 index 000000000..9e7f88ff6 --- /dev/null +++ b/benchmarks/QUALIFICATION.md @@ -0,0 +1,93 @@ +# Release qualification: public Helix `0.2.0b3` (draft) + +Graphify is pinned to the matching public PyPI releases +`helix-db==0.2.0b3` and `helix-db-embedded==0.2.0b3`. It uses ordinary public +`helixdb` imports and rejects incompatible SDK versions, missing embedded +payloads, malformed store paths, and unavailable required public APIs. No +Helix source checkout, Git dependency, private wheel, locally compiled +runtime, direct UniFFI import, dynamic native loader, or SDK monkeypatch is +used. + +Native Windows x86_64 is a supported Graphify target. Qualification still +requires a normal public `win_amd64` embedded wheel for the exact pinned +version. Public b3 currently publishes macOS universal2 and Linux +x86_64/aarch64 artifacts, but no Windows artifact, so the PR remains draft. + +## Graphify acceptance + +- Complete local suite: **2,925 passed, 3 skipped**. The Python 3.12 clean + environment accounted for the same behavior surface after installing the + declared development group, including wheel-packaging tests. +- Complete Python 3.10 suite: **2,912 passed, 16 skipped**. The additional + skips are the expected Python-version-gated optional video coverage; no + Helix or retained core behavior was skipped. +- CI matrix: Linux x86_64 on Python 3.10 and 3.12, Linux aarch64 on Python + 3.12, macOS universal2 on Python 3.12, and native Windows x86_64 on Python + 3.10 and 3.12. +- All four graph kinds, typed IDs, keyed multiedges, self-loops, hyperedges, + atomic activation, retained rollback, corruption, writer exclusion, + concurrent readers, generation hot reload, and read-only enforcement: passed. +- Incremental update and deletion, extraction cache, analysis, learning state, + clustering, affected/PR analysis, global aggregation, hooks, watch, exports, + MCP stdio, and MCP Streamable HTTP: passed. +- Generated skill check (134 artifacts), host coverage audit, schema singleton, + monolith contract, and always-on contract: passed. +- Ruff, scoped Pyright for the native integration surface, Bandit medium/high, + locked-runtime pip-audit, source compilation, package build, and + `git diff --check`: passed locally. +- Wheel installation in a clean Python 3.12 environment: passed. The installed + production dependency set contains neither NetworkX nor graspologic. +- Installed-wheel CLI flow: build native store, query, and shortest path: passed. + +Production source contains no NetworkX/graspologic imports or compatibility +graph. Legacy JSON graph files are never migrated, read, modified, or deleted; +the watch path emits one obsolete-format warning and requires a source rebuild. +Obsidian's `.obsidian/graph.json` remains solely because that filename belongs +to Obsidian's presentation configuration, not Graphify storage. + +## Benchmark qualification + +The isolated benchmark comparator is the only environment that installs +NetworkX. The deterministic parity corpus and full raw measurements are in +[`parity-networkx.json`](parity-networkx.json) and +[`helix-vs-networkx-2026-07-19.json`](helix-vs-networkx-2026-07-19.json); reproduction commands and +interpreted results are in [`RESULTS.md`](RESULTS.md). + +The public b3 candidate **does not pass the release gates**: + +| Graph | Helix ingest | Helix cold open | Peak RSS (informational) | Active store | Slowest absolute hot op | +|---|---:|---:|---:|---:|---:| +| 5k / 15k | 8.17s | 1.07s | 232.3 MiB | 35.12 MiB | 20.57ms | +| 20k / 60k | 32.55s | 4.88s | 647.1 MiB | 143.45 MiB | 8.93ms | + +At 20k/60k, weighted Leiden, sampled node centrality, and sampled edge +centrality are respectively **9.76x**, **9.93x**, and **4.13x** faster than the +current v8 NetworkX/JSON comparator, so the three native analytics gates pass. +Absolute hot-operation limits also pass. Build, 1% update, cold-open, active +and post-update storage, and relative warm-operation gates fail. + +Default retention deletes the inactive generation through public Helix +operations, but b3 does not physically reclaim enough embedded-store space; +the 20k/60k store grows from 143.45 MiB to 298.80 MiB after the 1% update. +Graphify deliberately does not manipulate SST/WAL files or call private +maintenance APIs. Peak RSS is reported but is non-blocking, as required. + +The public ERPNext corpus was independently reproduced at commit +`ea5c648ab04a2b30c5c238f6cb299c4237ff1c1e`. Helix now matches the loaded v8 +topology exactly at 25,443 nodes and 59,142 edges, but fresh build is 99.15s +versus 41.30s, cold open is 7.95s versus 0.33s, active storage is 10.34x v8, +and a representative steady query is 3.52s versus 0.05s. Raw measurements are +in [`report-erpnext-2026-07-19.json`](report-erpnext-2026-07-19.json). + +The report's other four corpora, exact gold queries, harness scripts, and raw +results were not distributed or attached. The five-corpus gold-recall table +therefore remains outstanding; the public ERPNext and synthetic results are +not a substitute for those missing inputs. + +## Conclusion + +The integration is correct enough for a draft PR, but is not production-ready. +It must remain draft until the exact public package pair provides a passing +Windows wheel, all CI platforms pass without core skips, the engineers' real +corpora and gold queries pass, and the failed public-package performance and +disk-reclamation gates are resolved without hidden workarounds. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 000000000..bd57a3c23 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,74 @@ +# Helix vs NetworkX benchmark + +Run on Python 3.12.13 and macOS arm64 with the matching public +`helix-db==0.2.0b3` / `helix-db-embedded==0.2.0b3` pair and NetworkX 3.6.1. +NetworkX was installed only in the isolated benchmark environment. Both +backends use the same deterministic topology and sampled betweenness uses 100 +sources with seed 42. Peak RSS is reported but is not a release gate. + +Lower is better. Exact samples and every acceptance check are retained in +[`helix-vs-networkx-2026-07-19.json`](helix-vs-networkx-2026-07-19.json). + +| Graph | Backend | Ingest | 1% update | Cold reopen | Hot open | 20 neighbors | BFS d=4 | 5 paths | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| 5k / 15k | NetworkX | 0.076s | 0.045s | 0.026s | n/a | 0.004ms | 1.39ms | 0.11ms | +| 5k / 15k | Helix | 8.167s | 10.730s | 1.073s | 5.48ms | 1.15ms | 20.57ms | 1.11ms | +| 20k / 60k | NetworkX | 0.304s | 0.192s | 0.124s | n/a | 0.003ms | 0.93ms | 0.11ms | +| 20k / 60k | Helix | 32.554s | 49.525s | 4.875s | 7.15ms | 1.10ms | 8.93ms | 3.25ms | + +| Graph | Backend | Community | Node BTW | Edge BTW | GraphML export | Peak ingest RSS | +|---|---|---:|---:|---:|---:|---:| +| 5k / 15k | NetworkX | 0.996s Louvain | 0.665s | 0.826s | 0.512s | 24.3 MiB | +| 5k / 15k | Helix | 0.095s Leiden | 0.093s | 0.301s | 0.582s | 232.3 MiB | +| 20k / 60k | NetworkX | 9.915s Louvain | 4.096s | 5.731s | 0.539s | 97.4 MiB | +| 20k / 60k | Helix | 1.015s Leiden | 0.412s | 1.386s | 2.553s | 647.1 MiB | + +| Graph | NetworkX JSON | Active Helix store | Helix after default-retention update | Eight concurrent cold reopens | +|---|---:|---:|---:|---:| +| 5k / 15k | 1.37 MiB | 35.12 MiB | 73.24 MiB | 4.03s | +| 20k / 60k | 5.58 MiB | 143.45 MiB | 298.80 MiB | 16.80s | + +## Gate result + +This candidate **does not pass the release gates** and the PR must remain +draft. Absolute hot-operation limits pass, and at 20k/60k native weighted +Leiden, sampled node centrality, and sampled edge centrality are respectively +9.76x, 9.93x, and 4.13x faster than the NetworkX comparator. Build, 1% update, +cold-open, storage, and relative warm-operation gates fail. + +Public b3 substantially improves the earlier public b1 engine result, but it +still does not reach the required 2x build/update or 3s cold-open limits. +Increasing Graphify's public write-batch size from 1,000 to 5,000 records did +not improve the remaining ingest time. + +Default retention logically deletes the inactive generation through public +Helix operations, but the embedded store does not reclaim its physical space; +the post-update store more than doubles. Graphify does not compact SST/WAL +files or call private maintenance APIs, so disk reclamation remains a public +package limitation. + +The behavioral parity result from the current candidate +passes directed path, BFS, DFS, exact node and edge betweenness, Louvain on a +golden graph, layout, relabeling, subgraph, and conversion checks. + +## Public ERPNext report reproduction + +The report-style public ERPNext run now has exact loaded-topology parity at +25,443 nodes and 59,142 edges. It still fails release qualification: fresh +build is 99.15s versus 41.30s, median cold open is 7.95s versus 0.33s, active +storage is 10.34x v8, and a representative steady query is 3.52s versus 0.05s. +The immediate unchanged-topology update passes at 46.65s versus 40.13s. See +[`report-erpnext-2026-07-19.json`](report-erpnext-2026-07-19.json) for commands, +versions, samples, and unavailable report inputs. + +Reproduce without Homebrew: + +```bash +python3.12 -m venv /tmp/graphify-networkx-benchmark +/tmp/graphify-networkx-benchmark/bin/python -m pip install -e . \ + -r benchmarks/requirements-networkx.txt +PYTHONPATH=. /tmp/graphify-networkx-benchmark/bin/python benchmarks/parity_networkx.py +PYTHONPATH=. /tmp/graphify-networkx-benchmark/bin/python \ + benchmarks/helix_vs_networkx.py \ + --out benchmarks/helix-vs-networkx-2026-07-19.json --check-gates +``` diff --git a/benchmarks/helix-vs-networkx-2026-07-19.json b/benchmarks/helix-vs-networkx-2026-07-19.json new file mode 100644 index 000000000..360cbc9c5 --- /dev/null +++ b/benchmarks/helix-vs-networkx-2026-07-19.json @@ -0,0 +1,460 @@ +{ + "helix_revision": "0.2.0b3", + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "pid": 55199, + "results": [ + { + "nodes": 5000, + "edges": 15000, + "networkx": { + "ingest_seconds": 0.07641220799996518, + "ingest_runs": 1, + "reopen_seconds": 0.025735499999427702, + "reopen_samples_seconds": [ + 0.025735499999427702, + 0.024884999998903368, + 0.03188929199677659 + ], + "neighbor_20_queries_seconds": 3.5420016502030194e-06, + "neighbor_samples_seconds": [ + 2.454200148349628e-05, + 4.417001036927104e-06, + 3.5420016502030194e-06, + 3.3750038710422814e-06, + 3.291999746579677e-06 + ], + "bfs_seconds": 0.001392665995808784, + "bfs_samples_seconds": [ + 0.0024065000034170225, + 0.001651625003432855, + 0.0013815839993185364, + 0.001379041001200676, + 0.001392665995808784 + ], + "shortest_path_5_queries_seconds": 0.00010879200272029266, + "shortest_path_samples_seconds": [ + 0.0003638340058387257, + 0.00010900000052060932, + 0.00010879200272029266, + 0.00010595899948384613, + 0.00010445800580782816 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 0.9960345830040751, + "community_samples_seconds": [ + 0.9838285829973756, + 1.0004757499991683, + 0.9960345830040751 + ], + "node_betweenness_seconds": 0.665316584003449, + "node_betweenness_samples_seconds": [ + 0.665316584003449 + ], + "edge_betweenness_seconds": 0.8258883750022505, + "edge_betweenness_samples_seconds": [ + 0.8258883750022505 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.0450237080003717, + "graphml_export_seconds": 0.5118335419974755, + "peak_rss_delta_bytes": 25477120, + "disk_bytes": 1436547 + }, + "helix": { + "ingest_seconds": 8.167331750002631, + "ingest_runs": 1, + "reopen_seconds": 1.0732905830009258, + "reopen_samples_seconds": [ + 1.0922817909959122, + 1.0656928750031511, + 1.0732905830009258 + ], + "hot_open_seconds": 0.005478249993757345, + "hot_open_samples_seconds": [ + 0.005679333000443876, + 0.005313916997693013, + 0.005478249993757345, + 0.0054750839990447275, + 0.005643750002491288 + ], + "neighbor_20_queries_seconds": 0.0011476250001578592, + "neighbor_samples_seconds": [ + 0.0012960420062881894, + 0.0011476250001578592, + 0.0011508749958011322, + 0.0011415420012781397, + 0.0011369999992894009 + ], + "bfs_seconds": 0.020567291998304427, + "bfs_samples_seconds": [ + 0.020894415996735916, + 0.020567291998304427, + 0.020452249998925254, + 0.020468916998652276, + 0.020609833001799416 + ], + "shortest_path_5_queries_seconds": 0.0011083749996032566, + "shortest_path_samples_seconds": [ + 0.0013168330042390153, + 0.001116250001359731, + 0.0011083749996032566, + 0.0010899580011027865, + 0.0010933750018011779 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 0.0947054170028423, + "community_samples_seconds": [ + 0.09471074999601115, + 0.0947054170028423, + 0.09400404199550394 + ], + "node_betweenness_seconds": 0.09330433300056029, + "node_betweenness_samples_seconds": [ + 0.09330433300056029 + ], + "edge_betweenness_seconds": 0.3006899579995661, + "edge_betweenness_samples_seconds": [ + 0.3006899579995661 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 10.729899457997817, + "graphml_export_seconds": 0.5823157080012606, + "eight_concurrent_reopens_seconds": 4.033307792000414, + "peak_rss_delta_bytes": 243630080, + "disk_after_ingest_bytes": 36827950, + "disk_after_update_bytes": 76794602 + } + }, + { + "nodes": 20000, + "edges": 60000, + "networkx": { + "ingest_seconds": 0.3044721250043949, + "ingest_runs": 1, + "reopen_seconds": 0.12366070800635498, + "reopen_samples_seconds": [ + 0.12366070800635498, + 0.10231733399996301, + 0.13137324999843258 + ], + "neighbor_20_queries_seconds": 3.4579934435896575e-06, + "neighbor_samples_seconds": [ + 2.2000000171829015e-05, + 4.082998202648014e-06, + 3.4579934435896575e-06, + 3.25000291923061e-06, + 3.416003892198205e-06 + ], + "bfs_seconds": 0.0009334999995189719, + "bfs_samples_seconds": [ + 0.0011756669991882518, + 0.0010096670011989772, + 0.0007989160003489815, + 0.0007129590012482367, + 0.0009334999995189719 + ], + "shortest_path_5_queries_seconds": 0.0001099579967558384, + "shortest_path_samples_seconds": [ + 0.00017779199697542936, + 0.00011137499677715823, + 0.0001099579967558384, + 0.00010837500303750858, + 0.00010750000365078449 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 9.914627540994843, + "community_samples_seconds": [ + 9.753872833003697, + 9.914627540994843, + 10.012469834000512 + ], + "node_betweenness_seconds": 4.096053250003024, + "node_betweenness_samples_seconds": [ + 4.096053250003024 + ], + "edge_betweenness_seconds": 5.731195250002202, + "edge_betweenness_samples_seconds": [ + 5.731195250002202 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.1919678330013994, + "graphml_export_seconds": 0.5387877500033937, + "peak_rss_delta_bytes": 102105088, + "disk_bytes": 5852869 + }, + "helix": { + "ingest_seconds": 32.553641709004296, + "ingest_runs": 1, + "reopen_seconds": 4.8753892079985235, + "reopen_samples_seconds": [ + 4.974821375006286, + 4.8753892079985235, + 4.873887665999064 + ], + "hot_open_seconds": 0.007152040998334996, + "hot_open_samples_seconds": [ + 0.007963042000483256, + 0.0071519999983138405, + 0.006855957995867357, + 0.007557749995612539, + 0.007152040998334996 + ], + "neighbor_20_queries_seconds": 0.0011049160020775162, + "neighbor_samples_seconds": [ + 0.0012251250009285286, + 0.001101458001357969, + 0.0011022909966413863, + 0.0011049160020775162, + 0.0011372919980203733 + ], + "bfs_seconds": 0.008930916999815963, + "bfs_samples_seconds": [ + 0.009267082998121623, + 0.008957958001701627, + 0.008930916999815963, + 0.008895042003132403, + 0.008882750000339001 + ], + "shortest_path_5_queries_seconds": 0.0032470840014866553, + "shortest_path_samples_seconds": [ + 0.004328625000198372, + 0.00329987499571871, + 0.0032433340020361356, + 0.0032470840014866553, + 0.003222624996851664 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 1.0154825419958797, + "community_samples_seconds": [ + 1.0154825419958797, + 1.0692213329966762, + 1.0087809999968158 + ], + "node_betweenness_seconds": 0.41240491600183304, + "node_betweenness_samples_seconds": [ + 0.41240491600183304 + ], + "edge_betweenness_seconds": 1.3862965000007534, + "edge_betweenness_samples_seconds": [ + 1.3862965000007534 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 49.52538345799985, + "graphml_export_seconds": 2.5530614579984103, + "eight_concurrent_reopens_seconds": 16.802997667000454, + "peak_rss_delta_bytes": 678559744, + "disk_after_ingest_bytes": 150420706, + "disk_after_update_bytes": 313312766 + } + } + ], + "acceptance": { + "passed": false, + "checks": [ + { + "name": "5000/15000 ingest vs v8", + "actual": 106.88516879405387, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 1% update vs v8", + "actual": 238.31665437038717, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open seconds", + "actual": 1.0732905830009258, + "limit": 0.5, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open vs v8", + "actual": 41.704671874445545, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 active store vs v8", + "actual": 25.63643932290416, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 post-update store vs v8", + "actual": 53.4577720046751, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 hot_open_seconds", + "actual": 0.005478249993757345, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds", + "actual": 0.0011476250001578592, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds vs v8", + "actual": 324.00464864043187, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 bfs_seconds", + "actual": 0.020567291998304427, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 bfs_seconds vs v8", + "actual": 14.768287629770176, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds", + "actual": 0.0011083749996032566, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds vs v8", + "actual": 10.188019081263908, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 ingest vs v8", + "actual": 106.91829903487684, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 1% update vs v8", + "actual": 257.98792789226735, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open seconds", + "actual": 4.8753892079985235, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open vs v8", + "actual": 39.42553205944749, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 active store vs v8", + "actual": 25.70033704837747, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 post-update store vs v8", + "actual": 53.53148447368291, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 clustering speedup", + "actual": 9.763464295021894, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 node centrality speedup", + "actual": 9.93211547940136, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 edge centrality speedup", + "actual": 4.134177104247963, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 hot_open_seconds", + "actual": 0.007152040998334996, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds", + "actual": 0.0011049160020775162, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds vs v8", + "actual": 319.5251871069282, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 bfs_seconds", + "actual": 0.008930916999815963, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 bfs_seconds vs v8", + "actual": 9.567131231299435, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds", + "actual": 0.0032470840014866553, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds vs v8", + "actual": 29.530221514464305, + "limit": 2.0, + "comparison": "<=", + "passed": false + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/helix-vs-networkx.json b/benchmarks/helix-vs-networkx.json new file mode 100644 index 000000000..4018784e6 --- /dev/null +++ b/benchmarks/helix-vs-networkx.json @@ -0,0 +1,460 @@ +{ + "helix_revision": "0.2.0b3", + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "pid": 41164, + "results": [ + { + "nodes": 5000, + "edges": 15000, + "networkx": { + "ingest_seconds": 0.07608495800377568, + "ingest_runs": 1, + "reopen_seconds": 0.027275708001980092, + "reopen_samples_seconds": [ + 0.02667249999649357, + 0.027275708001980092, + 0.03735450000385754 + ], + "neighbor_20_queries_seconds": 5.041001713834703e-06, + "neighbor_samples_seconds": [ + 2.2290994820650667e-05, + 8.75000114319846e-06, + 5.041001713834703e-06, + 4.624998837243766e-06, + 4.542001988738775e-06 + ], + "bfs_seconds": 0.001965709001524374, + "bfs_samples_seconds": [ + 0.003230207999877166, + 0.0022959170019021258, + 0.0019293750010547228, + 0.001913125001010485, + 0.001965709001524374 + ], + "shortest_path_5_queries_seconds": 0.0001400420005666092, + "shortest_path_samples_seconds": [ + 0.00046541699703084305, + 0.0001400420005666092, + 0.00014104200090514496, + 0.00013374999980442226, + 0.00013350000517675653 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 1.300195583004097, + "community_samples_seconds": [ + 1.2948922499999753, + 1.3108603340006084, + 1.300195583004097 + ], + "node_betweenness_seconds": 0.8536021670006448, + "node_betweenness_samples_seconds": [ + 0.8536021670006448 + ], + "edge_betweenness_seconds": 1.0998514579987386, + "edge_betweenness_samples_seconds": [ + 1.0998514579987386 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.058347334001155104, + "graphml_export_seconds": 0.22606754099979298, + "peak_rss_delta_bytes": 25526272, + "disk_bytes": 1436547 + }, + "helix": { + "ingest_seconds": 10.135198459000094, + "ingest_runs": 1, + "reopen_seconds": 1.4570273749995977, + "reopen_samples_seconds": [ + 1.4632811670016963, + 1.4570273749995977, + 1.414785832996131 + ], + "hot_open_seconds": 0.007826082997780759, + "hot_open_samples_seconds": [ + 0.007399832997180056, + 0.00796341599925654, + 0.007826082997780759, + 0.007802540996635798, + 0.007980541995493695 + ], + "neighbor_20_queries_seconds": 0.0016158340004039928, + "neighbor_samples_seconds": [ + 0.001879417002783157, + 0.0016267910032183863, + 0.0016158340004039928, + 0.0016101250002975576, + 0.0016047089957282878 + ], + "bfs_seconds": 0.027395249999244697, + "bfs_samples_seconds": [ + 0.027870541998709086, + 0.026020999997854233, + 0.02741887500451412, + 0.027395249999244697, + 0.026016083000286017 + ], + "shortest_path_5_queries_seconds": 0.001446291003958322, + "shortest_path_samples_seconds": [ + 0.001650833000894636, + 0.0014340410052682273, + 0.0014507079977192916, + 0.0014111669952399097, + 0.001446291003958322 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 0.12093970800196985, + "community_samples_seconds": [ + 0.12145224999403581, + 0.12093970800196985, + 0.12019025000336114 + ], + "node_betweenness_seconds": 0.12077770799805876, + "node_betweenness_samples_seconds": [ + 0.12077770799805876 + ], + "edge_betweenness_seconds": 0.394192749998183, + "edge_betweenness_samples_seconds": [ + 0.394192749998183 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 13.60693266599992, + "graphml_export_seconds": 0.7467261250058073, + "eight_concurrent_reopens_seconds": 7.142115583999839, + "peak_rss_delta_bytes": 359956480, + "disk_after_ingest_bytes": 37063727, + "disk_after_update_bytes": 77265897 + } + }, + { + "nodes": 20000, + "edges": 60000, + "networkx": { + "ingest_seconds": 0.37801691699860385, + "ingest_runs": 1, + "reopen_seconds": 0.1524028330022702, + "reopen_samples_seconds": [ + 0.1524028330022702, + 0.12498162500560284, + 0.15622158299811417 + ], + "neighbor_20_queries_seconds": 4.749999789055437e-06, + "neighbor_samples_seconds": [ + 2.7707996196113527e-05, + 5.875001079402864e-06, + 4.749999789055437e-06, + 4.499997885432094e-06, + 4.41699376096949e-06 + ], + "bfs_seconds": 0.0011056670045945793, + "bfs_samples_seconds": [ + 0.001745207999192644, + 0.001052082996466197, + 0.0011195409970241599, + 0.0010090420037158765, + 0.0011056670045945793 + ], + "shortest_path_5_queries_seconds": 0.0001388750024489127, + "shortest_path_samples_seconds": [ + 0.00020545800362015143, + 0.00014370799908647314, + 0.0001388750024489127, + 0.00013629200111608952, + 0.0001368750017718412 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 13.141373332997318, + "community_samples_seconds": [ + 13.224506249993283, + 13.122070040997642, + 13.141373332997318 + ], + "node_betweenness_seconds": 4.9877693750022445, + "node_betweenness_samples_seconds": [ + 4.9877693750022445 + ], + "edge_betweenness_seconds": 6.920005957996182, + "edge_betweenness_samples_seconds": [ + 6.920005957996182 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.253822416998446, + "graphml_export_seconds": 0.6834231249958975, + "peak_rss_delta_bytes": 102203392, + "disk_bytes": 5852869 + }, + "helix": { + "ingest_seconds": 40.671021500005736, + "ingest_runs": 1, + "reopen_seconds": 6.454431124999246, + "reopen_samples_seconds": [ + 6.424296000004688, + 6.456437791995995, + 6.454431124999246 + ], + "hot_open_seconds": 0.01001570800144691, + "hot_open_samples_seconds": [ + 0.009779542000615038, + 0.009466625000641216, + 0.01001570800144691, + 0.010156750002352055, + 0.010143042003619485 + ], + "neighbor_20_queries_seconds": 0.0016046660020947456, + "neighbor_samples_seconds": [ + 0.0018574580026324838, + 0.0016118330022436567, + 0.0016046660020947456, + 0.0016024159995140508, + 0.0015940000012051314 + ], + "bfs_seconds": 0.012172125003417023, + "bfs_samples_seconds": [ + 0.013258124999993015, + 0.012863666997873224, + 0.012022124996292405, + 0.012172125003417023, + 0.012125667002692353 + ], + "shortest_path_5_queries_seconds": 0.004266167001333088, + "shortest_path_samples_seconds": [ + 0.005248583001957741, + 0.004290082993975375, + 0.004266167001333088, + 0.004225707998557482, + 0.004209208003885578 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 1.413519665999047, + "community_samples_seconds": [ + 1.413519665999047, + 1.421890292003809, + 1.3064116250025108 + ], + "node_betweenness_seconds": 0.6120458329969551, + "node_betweenness_samples_seconds": [ + 0.6120458329969551 + ], + "edge_betweenness_seconds": 1.8529502079982194, + "edge_betweenness_samples_seconds": [ + 1.8529502079982194 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 63.57189174999803, + "graphml_export_seconds": 3.2455323329995736, + "eight_concurrent_reopens_seconds": 31.242825875000563, + "peak_rss_delta_bytes": 1041793024, + "disk_after_ingest_bytes": 150693438, + "disk_after_update_bytes": 313858260 + } + } + ], + "acceptance": { + "passed": false, + "checks": [ + { + "name": "5000/15000 ingest vs v8", + "actual": 133.20896435925138, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 1% update vs v8", + "actual": 233.20573080049456, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open seconds", + "actual": 1.4570273749995977, + "limit": 0.5, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open vs v8", + "actual": 53.4184987936454, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 active store vs v8", + "actual": 25.80056691497041, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 post-update store vs v8", + "actual": 53.785846895367854, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 hot_open_seconds", + "actual": 0.007826082997780759, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds", + "actual": 0.0016158340004039928, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds vs v8", + "actual": 320.5382763448465, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 bfs_seconds", + "actual": 0.027395249999244697, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 bfs_seconds vs v8", + "actual": 13.936574527562394, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds", + "actual": 0.001446291003958322, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds vs v8", + "actual": 10.327551720959685, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 ingest vs v8", + "actual": 107.59047987303687, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 1% update vs v8", + "actual": 250.45814511484716, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open seconds", + "actual": 6.454431124999246, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open vs v8", + "actual": 42.35112299325237, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 active store vs v8", + "actual": 25.746935050143783, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 post-update store vs v8", + "actual": 53.62468560290688, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 clustering speedup", + "actual": 9.296915811715476, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 node centrality speedup", + "actual": 8.149339650886992, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 edge centrality speedup", + "actual": 3.7345881870576587, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 hot_open_seconds", + "actual": 0.01001570800144691, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds", + "actual": 0.0016046660020947456, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds vs v8", + "actual": 337.824436496205, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 bfs_seconds", + "actual": 0.012172125003417023, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 bfs_seconds vs v8", + "actual": 11.008852532304912, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds", + "actual": 0.004266167001333088, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds vs v8", + "actual": 30.719473815328733, + "limit": 2.0, + "comparison": "<=", + "passed": false + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/helix_vs_networkx.py b/benchmarks/helix_vs_networkx.py new file mode 100644 index 000000000..c8a58fe97 --- /dev/null +++ b/benchmarks/helix_vs_networkx.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +"""Reproducible Helix comparison against the current v8 NetworkX/JSON baseline.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +from pathlib import Path +import platform +import random +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable + +try: + import resource +except ImportError: # Native Windows has no resource module; RSS is informational. + resource = None + +from graphify.helix.model import EdgeData, GraphBuildData, NodeData +from graphify.helix.persistence import HelixEmbeddedStore, HelixGraphReader +from graphify.helix.state import new_state + + +SIZES = ((5_000, 15_000), (20_000, 60_000)) + + +def _peak_rss_bytes() -> int: + if resource is None: + return 0 + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return value if platform.system() == "Darwin" else value * 1024 + + +def measure(call: Callable[[], Any]) -> tuple[Any, float, int]: + before = _peak_rss_bytes() + started = time.perf_counter() + result = call() + elapsed = time.perf_counter() - started + after = _peak_rss_bytes() + return result, elapsed, max(0, after - before) + + +def median_measure(call: Callable[[], Any], runs: int) -> tuple[Any, float, list[float]]: + samples: list[float] = [] + result: Any = None + for _ in range(runs): + result, elapsed, _ = measure(call) + samples.append(elapsed) + return result, statistics.median(samples), samples + + +def isolated_ingest_memory(backend: str, nodes: int, edges: int) -> int: + """Measure each backend in a fresh process so peak RSS cannot leak across runs.""" + output = subprocess.check_output( + [ + sys.executable, + str(Path(__file__).resolve()), + "--memory-backend", + backend, + "--memory-nodes", + str(nodes), + "--memory-edges", + str(edges), + ], + text=True, + env={**os.environ, "PYTHONPATH": os.environ.get("PYTHONPATH", ".")}, + ) + return int(output.strip()) + + +def memory_only(backend: str, nodes: int, edges: int) -> int: + root = Path(tempfile.mkdtemp(prefix="graphify-memory-benchmark-")) + try: + if backend == "networkx": + import networkx as networkx + + before = _peak_rss_bytes() + graph = topology(networkx.Graph, nodes, edges, 42) + payload = networkx.node_link_data(graph, edges="links") + (root / "graph.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + return max(0, _peak_rss_bytes() - before) + before = _peak_rss_bytes() + graph = helix_topology(nodes, edges, 42) + with HelixEmbeddedStore(root / "graph.helix") as store: + store.save_generation(graph, new_state(build={"benchmark": True})) + return max(0, _peak_rss_bytes() - before) + finally: + shutil.rmtree(root, ignore_errors=True) + + +def topology(graph_type: type, nodes: int, edges: int, seed: int): + graph = graph_type() + graph.add_nodes_from((index, {"label": f"node-{index}"}) for index in range(nodes)) + rng = random.Random(seed) + seen: set[tuple[int, int]] = set() + while len(seen) < edges: + source, target = rng.randrange(nodes), rng.randrange(nodes) + pair = (min(source, target), max(source, target)) + if source != target and pair not in seen: + seen.add(pair) + graph.add_edge(source, target, weight=1.0) + return graph + + +def helix_topology( + nodes: int, edges: int, seed: int, *, mixed_labels: bool = False +) -> GraphBuildData: + rng = random.Random(seed) + seen: set[tuple[int, int]] = set() + while len(seen) < edges: + source, target = rng.randrange(nodes), rng.randrange(nodes) + pair = (min(source, target), max(source, target)) + if source != target: + seen.add(pair) + return GraphBuildData( + nodes=[NodeData(index, {"label": f"node-{index}"}) for index in range(nodes)], + edges=[ + EdgeData( + source, + target, + { + "relation": ( + ("CALLS", "IMPORTS", "INHERITS")[index % 3] + if mixed_labels + else "RELATED_TO" + ), + **({"weight": 1.0} if index % 2 == 0 or not mixed_labels else {}), + **({"context": "code"} if mixed_labels and index % 5 == 0 else {}), + }, + ) + for index, (source, target) in enumerate(sorted(seen)) + ], + ) + + +def directory_size(path: Path) -> int: + return sum(child.stat().st_size for child in path.rglob("*") if child.is_file()) + + +def run_size(nodes: int, edges: int, root: Path) -> dict[str, Any]: + import networkx as networkx # benchmark-only dependency + + result: dict[str, Any] = {"nodes": nodes, "edges": edges} + nx_path = root / f"{nodes}-{edges}.networkx.json" + + def persist_networkx(graph: Any) -> None: + payload = networkx.node_link_data(graph, edges="links") + nx_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + def networkx_ingest(): + graph = topology(networkx.Graph, nodes, edges, 42) + persist_networkx(graph) + return graph + + nx_graph, nx_ingest, _ = measure(networkx_ingest) + _, nx_reopen, nx_reopen_samples = median_measure( + lambda: networkx.node_link_graph( + json.loads(nx_path.read_text(encoding="utf-8")), edges="links" + ), + 3, + ) + + def ingest_fixture(mixed_labels: bool, *, retain_last: bool): + samples: list[float] = [] + retained_graph = None + retained_path = None + for run in range(3): + sample_path = root / ( + f"{nodes}-{edges}.{'mixed' if mixed_labels else 'homogeneous'}-{run}.helix" + ) + + def ingest(): + graph = helix_topology(nodes, edges, 42, mixed_labels=mixed_labels) + with HelixEmbeddedStore(sample_path) as store: + store.save_generation(graph, new_state(build={"benchmark": True})) + return graph + + graph, elapsed, _ = measure(ingest) + samples.append(elapsed) + if retain_last and run == 2: + retained_graph, retained_path = graph, sample_path + else: + shutil.rmtree(sample_path, ignore_errors=True) + return retained_graph, retained_path, statistics.median(samples), samples + + helix_graph, store_path, helix_ingest, helix_ingest_samples = ingest_fixture( + False, retain_last=True + ) + _, _, helix_mixed_ingest, helix_mixed_ingest_samples = ingest_fixture(True, retain_last=False) + assert helix_graph is not None and store_path is not None + + def reopen(): + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load() + + loaded, helix_reopen, helix_reopen_samples = median_measure(reopen, 3) + durable = loaded.graph + if durable.node_count != nodes or durable.edge_count != edges: + raise RuntimeError("Helix reopen count verification failed") + helix_ingest_disk = directory_size(store_path) + + update_count = max(1, nodes // 100) + + def networkx_incremental(): + for index in range(update_count): + nx_graph.nodes[index]["label"] = f"updated-node-{index}" + persist_networkx(nx_graph) + + _, nx_incremental, _ = measure(networkx_incremental) + + helix_incremental_samples: list[float] = [] + with HelixEmbeddedStore(store_path) as store: + for run in range(3): + changed = GraphBuildData( + nodes=[ + NodeData( + node.id, + { + **node.attributes, + **( + {"label": f"updated-{run}-node-{node.id}"} + if isinstance(node.id, int) and node.id < update_count + else {} + ), + }, + ) + for node in helix_graph.nodes + ], + edges=list(helix_graph.edges), + ) + _, elapsed, _ = measure( + lambda: store.save_generation( + changed, + new_state(build={"benchmark": True, "incremental_percent": 1}), + ) + ) + helix_incremental_samples.append(elapsed) + helix_incremental = statistics.median(helix_incremental_samples) + loaded = reopen() + durable = loaded.graph + + reader = HelixGraphReader(store_path) + reader.get() + _, helix_hot_open, helix_hot_open_samples = median_measure(reader.get, 5) + probes = [index * (nodes // 20) for index in range(20)] + _, nx_neighbors, nx_neighbor_samples = median_measure( + lambda: [tuple(nx_graph.neighbors(node)) for node in probes], 5 + ) + _, helix_neighbors, helix_neighbor_samples = median_measure( + lambda: [durable.neighbors(node) for node in probes], 5 + ) + _, nx_bfs, nx_bfs_samples = median_measure( + lambda: list(networkx.bfs_tree(nx_graph, 0, depth_limit=4)), 5 + ) + _, helix_bfs, helix_bfs_samples = median_measure(lambda: _bounded_bfs(durable, 0, 4), 5) + _, nx_shortest, nx_shortest_samples = median_measure( + lambda: [networkx.shortest_path(nx_graph, 0, 4) for _ in range(5)], 5 + ) + _, helix_shortest, helix_shortest_samples = median_measure( + lambda: [durable.shortest_path(0, 4) for _ in range(5)], 5 + ) + _, nx_louvain, nx_louvain_samples = median_measure( + lambda: networkx.community.louvain_communities(nx_graph, seed=42), 3 + ) + _, helix_leiden, helix_leiden_samples = median_measure( + lambda: durable.to_undirected().leiden().communities, 3 + ) + _, nx_centrality, nx_centrality_samples = median_measure( + lambda: networkx.betweenness_centrality(nx_graph, k=min(100, nodes), seed=42), 1 + ) + _, helix_centrality, helix_centrality_samples = median_measure( + lambda: durable.betweenness_centrality( + __import__("helixdb").BetweennessOptions( + mode="sampled", sample_count=min(100, nodes), seed=42 + ) + ), + 1, + ) + _, nx_edge_centrality, nx_edge_centrality_samples = median_measure( + lambda: networkx.edge_betweenness_centrality(nx_graph, k=min(100, nodes), seed=42), 1 + ) + _, helix_edge_centrality, helix_edge_centrality_samples = median_measure( + lambda: durable.edge_betweenness_centrality( + __import__("helixdb").BetweennessOptions( + mode="sampled", sample_count=min(100, nodes), seed=42 + ) + ), + 1, + ) + + nx_export_path = root / f"{nodes}-{edges}.networkx.graphml" + helix_export_path = root / f"{nodes}-{edges}.helix.graphml" + _, nx_export, _ = measure(lambda: networkx.write_graphml(nx_graph, nx_export_path)) + + def helix_export_graphml(): + from graphify.export import to_graphml + + to_graphml(durable, {}, str(helix_export_path)) + + _, helix_export, _ = measure(helix_export_graphml) + + def concurrent_readers(): + with ThreadPoolExecutor(max_workers=4) as pool: + return list(pool.map(lambda _: reopen().graph.node_count, range(8))) + + _, helix_concurrency, _ = measure(concurrent_readers) + nx_memory_samples = [isolated_ingest_memory("networkx", nodes, edges) for _ in range(3)] + helix_memory_samples = [isolated_ingest_memory("helix", nodes, edges) for _ in range(3)] + nx_memory = int(statistics.median(nx_memory_samples)) + helix_memory = int(statistics.median(helix_memory_samples)) + result["networkx"] = { + "ingest_seconds": nx_ingest, + "ingest_runs": 1, + "reopen_seconds": nx_reopen, + "reopen_samples_seconds": nx_reopen_samples, + "neighbor_20_queries_seconds": nx_neighbors, + "neighbor_samples_seconds": nx_neighbor_samples, + "bfs_seconds": nx_bfs, + "bfs_samples_seconds": nx_bfs_samples, + "shortest_path_5_queries_seconds": nx_shortest, + "shortest_path_samples_seconds": nx_shortest_samples, + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": nx_louvain, + "community_samples_seconds": nx_louvain_samples, + "node_betweenness_seconds": nx_centrality, + "node_betweenness_samples_seconds": nx_centrality_samples, + "edge_betweenness_seconds": nx_edge_centrality, + "edge_betweenness_samples_seconds": nx_edge_centrality_samples, + "betweenness_mode": f"sampled ({min(100, nodes)} sources, seed 42)", + "incremental_1pct_seconds": nx_incremental, + "graphml_export_seconds": nx_export, + "peak_rss_delta_bytes": nx_memory, + "peak_rss_samples_bytes": nx_memory_samples, + "disk_bytes": nx_path.stat().st_size, + } + result["helix"] = { + "ingest_seconds": helix_ingest, + "ingest_runs": 3, + "ingest_samples_seconds": helix_ingest_samples, + "mixed_label_ingest_seconds": helix_mixed_ingest, + "mixed_label_ingest_samples_seconds": helix_mixed_ingest_samples, + "reopen_seconds": helix_reopen, + "reopen_samples_seconds": helix_reopen_samples, + "hot_open_seconds": helix_hot_open, + "hot_open_samples_seconds": helix_hot_open_samples, + "neighbor_20_queries_seconds": helix_neighbors, + "neighbor_samples_seconds": helix_neighbor_samples, + "bfs_seconds": helix_bfs, + "bfs_samples_seconds": helix_bfs_samples, + "shortest_path_5_queries_seconds": helix_shortest, + "shortest_path_samples_seconds": helix_shortest_samples, + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": helix_leiden, + "community_samples_seconds": helix_leiden_samples, + "node_betweenness_seconds": helix_centrality, + "node_betweenness_samples_seconds": helix_centrality_samples, + "edge_betweenness_seconds": helix_edge_centrality, + "edge_betweenness_samples_seconds": helix_edge_centrality_samples, + "betweenness_mode": f"sampled ({min(100, nodes)} sources, seed 42)", + "incremental_1pct_seconds": helix_incremental, + "incremental_1pct_samples_seconds": helix_incremental_samples, + "graphml_export_seconds": helix_export, + "eight_concurrent_reopens_seconds": helix_concurrency, + "peak_rss_delta_bytes": helix_memory, + "peak_rss_samples_bytes": helix_memory_samples, + "disk_after_ingest_bytes": helix_ingest_disk, + "disk_after_update_bytes": directory_size(store_path), + "post_delta_store_ratio": directory_size(store_path) / helix_ingest_disk, + } + result["comparisons"] = { + "ingest_vs_networkx": helix_ingest / nx_ingest, + "mixed_label_ingest_vs_networkx": helix_mixed_ingest / nx_ingest, + "incremental_1pct_vs_networkx": helix_incremental / nx_incremental, + "preferred_20k_ingest_seconds": 5.0 if nodes == 20_000 else None, + } + return result + + +def acceptance_gates(results: list[dict[str, Any]]) -> dict[str, Any]: + """Evaluate the release thresholds and retain every measured comparison.""" + checks: list[dict[str, Any]] = [] + + def check(name: str, actual: float, limit: float, comparison: str = "<=") -> None: + passed = actual <= limit if comparison == "<=" else actual >= limit + checks.append( + { + "name": name, + "actual": actual, + "limit": limit, + "comparison": comparison, + "passed": passed, + } + ) + + for row in results: + label = f"{row['nodes']}/{row['edges']}" + helix = row["helix"] + networkx = row["networkx"] + cold_limit = 0.5 if row["nodes"] == 5_000 else 3.0 + hot_limit = 0.100 if row["nodes"] == 5_000 else 0.500 + ingest_limit = 3.0 if row["nodes"] == 5_000 else 5.0 + check(f"{label} ingest seconds", helix["ingest_seconds"], ingest_limit) + check( + f"{label} mixed-label ingest seconds", + helix["mixed_label_ingest_seconds"], + ingest_limit, + ) + check( + f"{label} 1% production delta seconds", + helix["incremental_1pct_seconds"], + 2.0, + ) + check( + f"{label} post-delta store growth", + helix["post_delta_store_ratio"], + 1.3, + ) + if row["nodes"] == 20_000: + check( + f"{label} peak ingest RSS MiB", + helix["peak_rss_delta_bytes"] / (1024 * 1024), + 600.0, + ) + check(f"{label} cold open seconds", helix["reopen_seconds"], cold_limit) + check( + f"{label} cold open vs v8", + helix["reopen_seconds"] / networkx["reopen_seconds"], + 5.0, + ) + check( + f"{label} active store vs v8", + helix["disk_after_ingest_bytes"] / networkx["disk_bytes"], + 3.0, + ) + check( + f"{label} post-update store vs v8", + helix["disk_after_update_bytes"] / networkx["disk_bytes"], + 3.0, + ) + if row["nodes"] == 20_000: + check( + f"{label} clustering speedup", + networkx["community_seconds"] / helix["community_seconds"], + 3.0, + ">=", + ) + check( + f"{label} node centrality speedup", + networkx["node_betweenness_seconds"] / helix["node_betweenness_seconds"], + 3.0, + ">=", + ) + check( + f"{label} edge centrality speedup", + networkx["edge_betweenness_seconds"] / helix["edge_betweenness_seconds"], + 3.0, + ">=", + ) + check(f"{label} hot_open_seconds", helix["hot_open_seconds"], hot_limit) + for metric, baseline_metric in ( + ("neighbor_20_queries_seconds", "neighbor_20_queries_seconds"), + ("bfs_seconds", "bfs_seconds"), + ("shortest_path_5_queries_seconds", "shortest_path_5_queries_seconds"), + ): + check(f"{label} {metric}", helix[metric], hot_limit) + check( + f"{label} {metric} vs v8", + helix[metric] / networkx[baseline_metric], + 2.0, + ) + return {"passed": all(item["passed"] for item in checks), "checks": checks} + + +def _bounded_bfs(graph, start: Any, depth: int) -> list[Any]: + visited = {start} + frontier = {start} + for _ in range(depth): + frontier = {neighbor for node in frontier for neighbor in graph.neighbors(node)} - visited + visited.update(frontier) + return list(visited) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--out", type=Path, default=Path("benchmarks/helix-vs-networkx.json")) + parser.add_argument("--memory-backend", choices=("networkx", "helix")) + parser.add_argument("--memory-nodes", type=int) + parser.add_argument("--memory-edges", type=int) + parser.add_argument("--check-gates", action="store_true") + args = parser.parse_args() + if args.memory_backend: + if args.memory_nodes is None or args.memory_edges is None: + parser.error("--memory-backend requires --memory-nodes and --memory-edges") + print(memory_only(args.memory_backend, args.memory_nodes, args.memory_edges)) + return + root = Path(tempfile.mkdtemp(prefix="graphify-benchmark-")) + try: + results = [run_size(nodes, edges, root) for nodes, edges in SIZES] + report = { + "helix_revision": importlib.metadata.version("helix-db-embedded"), + "python": platform.python_version(), + "platform": platform.platform(), + "pid": os.getpid(), + "results": results, + "acceptance": acceptance_gates(results), + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + if args.check_gates and not report["acceptance"]["passed"]: + raise SystemExit(1) + finally: + shutil.rmtree(root, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/parity-networkx.json b/benchmarks/parity-networkx.json new file mode 100644 index 000000000..2a1684130 --- /dev/null +++ b/benchmarks/parity-networkx.json @@ -0,0 +1,15 @@ +{ + "helix_revision": "0.2.0b3", + "networkx_version": "3.6.1", + "checks": { + "directed_path": true, + "bfs": true, + "dfs": true, + "node_betweenness": true, + "edge_betweenness": true, + "louvain": true, + "layout": true, + "transformations": true + }, + "all_passed": true +} diff --git a/benchmarks/parity_networkx.py b/benchmarks/parity_networkx.py new file mode 100644 index 000000000..2a02d4072 --- /dev/null +++ b/benchmarks/parity_networkx.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Isolated behavioral parity checks; NetworkX is benchmark-only.""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile + +import networkx + +from graphify.helix.model import EdgeData, GraphBuildData, NodeData +from graphify.helix.native import native_backend_info +from graphify.helix.persistence import HelixEmbeddedStore +from graphify.helix.state import new_state + + +def load_native(build: GraphBuildData, root: Path): + with HelixEmbeddedStore(root / "graph.helix") as store: + store.save_generation(build, new_state()) + with HelixEmbeddedStore(root / "graph.helix", read_only=True) as store: + return store.load().graph + + +def close_scores(left: dict, right: dict, tolerance: float = 1e-9) -> bool: + return left.keys() == right.keys() and all( + abs(left[key] - right[key]) <= tolerance for key in left + ) + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="graphify-parity-") as temporary: + root = Path(temporary) + nodes = [NodeData("a"), NodeData(2), NodeData(b"c")] + edges = [ + EdgeData("a", 2, {"relation": "calls", "weight": 1.0}), + EdgeData(2, b"c", {"relation": "calls", "weight": 1.0}), + ] + native = load_native(GraphBuildData(kind="digraph", nodes=nodes, edges=edges), root) + reference = networkx.DiGraph() + reference.add_edge("a", 2, weight=1.0) + reference.add_edge(2, b"c", weight=1.0) + + from helixdb import TraversalOptions + + checks = { + "directed_path": native.shortest_path("a", b"c", direction="out").node_ids + == tuple(networkx.shortest_path(reference, "a", b"c")), + "bfs": {visit.node_id for visit in native.traverse(TraversalOptions( + seeds=("a",), max_depth=10, strategy="breadth_first", direction="out" + )).visits} == set(networkx.bfs_tree(reference, "a")), + "dfs": {visit.node_id for visit in native.traverse(TraversalOptions( + seeds=("a",), max_depth=10, strategy="depth_first", direction="out" + )).visits} == set(networkx.dfs_tree(reference, "a")), + } + native_node_scores = {row.node_id: row.score for row in native.betweenness_centrality()} + checks["node_betweenness"] = close_scores( + native_node_scores, networkx.betweenness_centrality(reference) + ) + native_edge_scores = { + (row.source, row.target): row.score + for row in native.edge_betweenness_centrality() + } + checks["edge_betweenness"] = close_scores( + native_edge_scores, networkx.edge_betweenness_centrality(reference) + ) + + community_nodes = [NodeData(name) for name in "abcdef"] + community_edges = [ + EdgeData(u, v, {"relation": "related", "weight": 1.0}) + for group in ("abc", "def") + for index, u in enumerate(group) + for v in group[index + 1 :] + ] + communities = load_native( + GraphBuildData(nodes=community_nodes, edges=community_edges), root / "communities" + ) + native_partition = {frozenset(row.node_ids) for row in communities.louvain_communities().communities} + nx_community = networkx.Graph() + nx_community.add_edges_from((edge.source, edge.target) for edge in community_edges) + nx_partition = { + frozenset(group) + for group in networkx.community.louvain_communities(nx_community, seed=42) + } + checks["louvain"] = native_partition == nx_partition + checks["layout"] = len(communities.spring_layout()) == 6 + checks["transformations"] = ( + communities.induced_subgraph(["a", "b"]).edge_count == 1 + and communities.relabel({"a": ("renamed", "a")}).contains_node(("renamed", "a")) + and communities.to_directed().directed + ) + + report = { + "helix_revision": native_backend_info().embedded_version, + "networkx_version": networkx.__version__, + "checks": checks, + "all_passed": all(checks.values()), + } + print(json.dumps(report, indent=2)) + if not report["all_passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/report-erpnext-2026-07-19.json b/benchmarks/report-erpnext-2026-07-19.json new file mode 100644 index 000000000..b0b7caf4e --- /dev/null +++ b/benchmarks/report-erpnext-2026-07-19.json @@ -0,0 +1,131 @@ +{ + "qualification_date": "2026-07-19", + "report_source": "helix_report.pdf (provided artifact)", + "method": { + "command": "graphify update --no-cluster", + "llm_environment_removed": [ + "ANTHROPIC_API_KEY", + "MOONSHOT_API_KEY", + "OLLAMA_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY", + "AZURE_OPENAI_API_KEY", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION" + ], + "timing": "/usr/bin/time -lp", + "runs_sequential": true, + "peak_rss_blocking": false + }, + "versions": { + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "helix_db": "0.2.0b3", + "helix_db_embedded": "0.2.0b3", + "graphify_v8_commit": "edec9eabeceeae6aa2375eddb3835efa1a32c0a3", + "candidate_base_commit": "17169c0ebcc2e64f54e2504ee2270e6a1cb1dc92" + }, + "corpus": { + "name": "erpnext", + "repository": "frappe/erpnext", + "commit": "ea5c648ab04a2b30c5c238f6cb299c4237ff1c1e" + }, + "topology": { + "v8_serialized_nodes": 24889, + "v8_serialized_edges": 59142, + "v8_loaded_nodes": 25443, + "v8_loaded_edges": 59142, + "helix_loaded_nodes": 25443, + "helix_loaded_edges": 59142, + "parity": true, + "note": "The v8 no-cluster payload relies on multigraph rehydration to create 554 external endpoint stubs. The candidate now preserves those stubs and every relation-distinct parallel edge natively." + }, + "fresh_build": { + "v8_seconds": 41.30, + "helix_seconds": 99.15, + "helix_over_v8": 2.4007263922518165, + "limit": 2.0, + "passed": false, + "v8_peak_rss_bytes": 1050017792, + "helix_peak_rss_bytes": 1630797824 + }, + "immediate_update": { + "v8_seconds": 40.13, + "helix_seconds": 46.65, + "helix_over_v8": 1.162471966110142, + "limit": 2.0, + "passed": true, + "helix_topology_unchanged": true, + "helix_peak_rss_bytes": 2554691584 + }, + "cold_open": { + "v8_samples_seconds": [0.317707, 0.328374, 0.343941], + "helix_samples_seconds": [7.904627, 7.948525, 8.007312], + "v8_median_seconds": 0.328374, + "helix_median_seconds": 7.948525, + "helix_over_v8": 24.205707516429438, + "absolute_limit_seconds": 3.0, + "relative_limit": 5.0, + "passed": false + }, + "active_store": { + "v8_bytes": 34901442, + "helix_bytes": 360832137, + "helix_over_v8": 10.338602542553973, + "limit": 3.0, + "passed": false, + "retain_rollback": false + }, + "representative_query": { + "question": "payment entry reconciliation", + "depth": 2, + "token_budget": 500, + "v8_warm_samples_seconds": [ + 0.900777, + 0.054018, + 0.050583, + 0.049234, + 0.048877 + ], + "helix_warm_samples_seconds": [ + 4.008484, + 3.529874, + 3.519547, + 3.522045, + 3.527935 + ], + "v8_steady_median_seconds": 0.0499085, + "helix_steady_median_seconds": 3.52499, + "absolute_limit_seconds": 0.5, + "relative_limit": 2.0, + "passed": false, + "note": "This is an operational latency probe, not a gold-recall substitute." + }, + "unavailable_report_inputs": { + "corpora": ["comfywerk", "agent", "backend", "passport"], + "gold_queries": "graphify-bench/queries/*.json", + "harnesses": ["bench_helix.py", "bench_inprocess.py"], + "raw_results": "helix_bench_results.json", + "searched_locations": [ + "workspace repositories", + "provided downloads", + "temporary work directories", + "GitHub repository/code search", + "PR #1993 comments and reviews", + "PDF link annotations" + ], + "effect": "The report's exact five-corpus gold-recall table cannot be reproduced or claimed passing without these non-redistributed inputs." + }, + "merge_ready": false, + "blocking_gates": [ + "fresh build exceeds 2x v8", + "cold open exceeds both absolute and relative limits", + "active store exceeds 3x v8", + "steady query exceeds both absolute and relative limits", + "exact report corpora, gold queries, harnesses, and raw results are unavailable", + "public helix-db-embedded 0.2.0b3 has no win_amd64 wheel" + ] +} diff --git a/benchmarks/requirements-networkx.txt b/benchmarks/requirements-networkx.txt new file mode 100644 index 000000000..f8efd2cab --- /dev/null +++ b/benchmarks/requirements-networkx.txt @@ -0,0 +1,2 @@ +# Isolated comparison environment only; never install in Graphify production. +networkx==3.6.1 diff --git a/docs/how-it-works.md b/docs/how-it-works.md index e0e6e5275..573c3cdb2 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -64,7 +64,7 @@ On a mixed corpus (Karpathy repos + 5 papers + 4 images, 52 files): **71.5x fewe Token reduction scales with corpus size. Six files already fits in a context window — the graph value there is structural clarity, not compression. At 52 files the savings compound quickly. -Each `worked/` folder in the repo has the raw input files and actual output (`GRAPH_REPORT.md`, `graph.json`) so you can run it yourself and verify. +Each `worked/` folder in the repo has raw input files and a report so you can run the current native build yourself and verify it. --- @@ -76,13 +76,13 @@ Code files are extracted in parallel using `ProcessPoolExecutor` — bypasses Py ## SHA256 cache -Every extracted file is fingerprinted by content hash. Re-runs skip unchanged files entirely — only new or modified files go through extraction again. The cache lives in `graphify-out/cache/`. +Every extracted file is fingerprinted by content hash. Re-runs skip unchanged files entirely — only new or modified files go through extraction again. Hashes and extraction cache entries live in the active Helix generation. --- ## The graph format -The output `graph.json` uses NetworkX's node-link format. Each node has: +The embedded `graphify-out/graph.helix` store is the sole runtime graph. Each native node has: - `id` — stable identifier - `label` — human-readable name - `file_type` — `code`, `document`, `paper`, `image`, `rationale` @@ -98,4 +98,4 @@ Each edge has: - `confidence_score` — float (INFERRED only) - `source_file` — where the relationship was found -Hyperedges (group relationships connecting 3+ nodes) live in `G.graph["hyperedges"]`. +Hyperedges (group relationships connecting 3+ nodes) are stored as generation-scoped native metadata. diff --git a/docs/node-summaries-rfc.md b/docs/node-summaries-rfc.md index af191f3f8..8ab2d37b9 100644 --- a/docs/node-summaries-rfc.md +++ b/docs/node-summaries-rfc.md @@ -5,7 +5,7 @@ summaries for AI coding agents. ## Problem -`graph.json` gives agents graph structure, source files, node labels, and +The native Helix graph gives agents graph structure, source files, node labels, and relationships. That helps avoid reading an entire repository, but agents still often need to inspect raw files just to answer a basic navigation question: @@ -84,7 +84,7 @@ or every symbol in the file. Those details already belong in the graph and can be fetched through `graphify explain`, `graphify query`, or direct file reads when needed. -## Option A: `summary` attribute in `graph.json` +## Option A: `summary` property on native nodes Add an optional `summary` field to file-level nodes: @@ -107,8 +107,8 @@ graphify explain "extract.py" Pros: -- Single artifact for graph consumers. -- Matches NetworkX node attributes and existing node metadata. +- One generation-safe store for graph consumers. +- Matches existing native node properties and metadata. - Easy for `explain`, `serve`, visualizers, and MCP tools to consume. - No sidecar freshness or node-ID join logic. @@ -116,12 +116,11 @@ Cons: - Adds text to the core graph artifact. - Expands the graph schema surface. -- Consumers that dump all of `graph.json` into an LLM context would pay for all - summaries at once. +- Consumers that request every node would pay for all summaries at once. -## Option B: sidecar `node-summaries.json` +## Option B: generation-scoped summary records -Write summaries to a separate artifact keyed by node ID: +Write summaries as generation-scoped native records keyed by node ID: ```json { @@ -146,15 +145,15 @@ graphify explain "extract.py" Pros: -- Keeps `graph.json` lean and topology-focused. +- Keeps topology records lean and focused. - Makes summaries clearly optional. - Can be regenerated independently. - Provides a natural place for future generator metadata. Cons: -- Adds a second artifact that consumers must discover and load. -- Introduces freshness and synchronization questions. +- Adds a second native record family that consumers must query. +- Requires transactional activation with the topology generation. - Every consumer that wants summaries must join by node ID. ## Suggested first implementation once storage is chosen @@ -177,8 +176,7 @@ Cons: ## Questions for maintainers and users -1. Should graphify prefer one artifact (`graph.json`) or keep generated text in a - sidecar? +1. Should graphify prefer node properties or generation-scoped summary records? 2. Should deterministic file-level summaries be generated during graph creation, or only through an explicit command such as `graphify summarize`? 3. Is `summary` the right term, or would `synopsis` better communicate a short, diff --git a/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md b/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md deleted file mode 100644 index 488591b8d..000000000 --- a/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md +++ /dev/null @@ -1,1143 +0,0 @@ -# Incremental Updates + Entity Deduplication Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add semantic cache + incremental graph updates to `graphify extract`, and add a new `graphify/dedup.py` module that runs MinHash/LSH + Jaro-Winkler entity deduplication before clustering on every run. - -**Architecture:** Two independent features wired into the same pipeline: (1) `__main__.py` extract block uses `check_semantic_cache`/`save_semantic_cache` + `detect_incremental` + `build_merge` for incremental runs; (2) new `graphify/dedup.py` implements the full dedup pipeline called from `build.py` after graph construction and before `cluster()`. - -**Tech Stack:** Python 3.10+, `datasketch` (MinHash/LSH), `rapidfuzz` (Jaro-Winkler), `networkx` (union-find via `nx.utils.UnionFind`), existing `graphify.cache`, `graphify.detect`, `graphify.build`. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `graphify/dedup.py` | **Create** | Full dedup pipeline: entropy gate → MinHash/LSH → Jaro-Winkler → community boost → union-find merge | -| `graphify/build.py` | **Modify** | Call `deduplicate_entities()` from `build()` and `build_merge()`; wire dormant `deduplicate_by_label` | -| `graphify/__main__.py` | **Modify** | Semantic cache wrapping, incremental mode auto-detection, `build_merge` swap, manifest save, `--dedup-llm` flag | -| `pyproject.toml` | **Modify** | Add `datasketch`, `rapidfuzz` to base dependencies | -| `tests/test_dedup.py` | **Create** | Unit + integration tests for dedup pipeline | -| `tests/test_incremental.py` | **Create** | Integration tests for incremental extract (cache hits, manifest, prune) | - ---- - -## Task 1: Add `datasketch` and `rapidfuzz` to dependencies - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add deps to pyproject.toml** - -Open `pyproject.toml`. The `dependencies` list ends around line 38. Add two entries: - -```toml -dependencies = [ - "networkx", - "datasketch", - "rapidfuzz", - "tree-sitter>=0.23.0", - # ... rest unchanged -] -``` - -- [ ] **Step 2: Install into venv** - -```bash -cd /home/safi/graphify -venv/bin/pip install datasketch rapidfuzz -q -``` - -Expected: both install without errors. - -- [ ] **Step 3: Verify import** - -```bash -venv/bin/python -c "from datasketch import MinHash, MinHashLSH; from rapidfuzz import fuzz; print('ok')" -``` - -Expected: `ok` - -- [ ] **Step 4: Commit** - -```bash -git add pyproject.toml -git commit -m "Add datasketch and rapidfuzz as base dependencies" -``` - ---- - -## Task 2: Create `graphify/dedup.py` — entropy gate + MinHash/LSH + Jaro-Winkler - -**Files:** -- Create: `graphify/dedup.py` -- Create: `tests/test_dedup.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_dedup.py`: - -```python -"""Tests for graphify/dedup.py entity deduplication pipeline.""" -from __future__ import annotations -import pytest -from graphify.dedup import deduplicate_entities, _entropy, _shingles - - -# ── entropy gate ───────────────────────────────────────────────────────────── - -def test_entropy_short_label_low(): - assert _entropy("AI") < 2.5 - -def test_entropy_normal_label_high(): - assert _entropy("AuthenticationManager") >= 2.5 - -def test_entropy_empty_string(): - assert _entropy("") == 0.0 - - -# ── shingles ───────────────────────────────────────────────────────────────── - -def test_shingles_produces_trigrams(): - s = _shingles("hello") - assert "hel" in s - assert "ell" in s - assert "llo" in s - -def test_shingles_short_string(): - # strings shorter than 3 chars return single shingle of the string itself - assert _shingles("ab") == {"ab"} - - -# ── full pipeline ───────────────────────────────────────────────────────────── - -def _make_nodes(*labels): - return [{"id": label.lower().replace(" ", "_"), "label": label, "source_file": "test.md"} for label in labels] - -def _make_edges(src, tgt, relation="relates_to"): - return [{"source": src, "target": tgt, "relation": relation}] - - -def test_exact_duplicates_merged(): - nodes = _make_nodes("UserService", "userservice", "User Service") - edges = [] - result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) - labels = {n["label"] for n in result_nodes} - # All three are the same concept — only one survives - assert len(result_nodes) == 1 - - -def test_typo_merged(): - # "GraphExtractor" vs "Graph Extractor" — Jaro-Winkler >= 0.92 - nodes = _make_nodes("GraphExtractor", "Graph Extractor") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 1 - - -def test_unrelated_not_merged(): - nodes = _make_nodes("UserService", "OrderService") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 - - -def test_short_low_entropy_not_merged(): - # "AI" and "ML" are low-entropy — entropy gate skips them - nodes = _make_nodes("AI", "ML") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 - - -def test_edges_rewired_after_merge(): - nodes = _make_nodes("GraphExtractor", "Graph Extractor", "Parser") - # edge from loser to Parser should be rewired to winner - edges = [{"source": "graph_extractor", "target": "parser", "relation": "uses"}] - result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 # merged + Parser - # edge should still exist (rewired to winner) - assert len(result_edges) == 1 - - -def test_self_loops_dropped_after_merge(): - # If both endpoints of an edge get merged into same node, drop the edge - nodes = _make_nodes("GraphExtractor", "Graph Extractor") - edges = [{"source": "graphextractor", "target": "graph_extractor", "relation": "same"}] - _, result_edges = deduplicate_entities(nodes, edges, communities={}) - assert result_edges == [] - - -def test_community_boost_aids_merge(): - # Two nodes in same community with score in 0.75-0.85 zone get boosted - # This is a structural test — use labels that would score ~0.80 without boost - # We verify that with communities set, they merge, without they don't - # Use labels that Jaro-Winkler scores ~0.88 (borderline) - nodes = _make_nodes("AuthManager", "Auth Manager") - edges = [] - # Same community → boost → merge - communities = {"authmanager": 1, "auth_manager": 1} - result_with, _ = deduplicate_entities(nodes, edges, communities=communities) - # Different community → no boost - communities_diff = {"authmanager": 1, "auth_manager": 2} - result_without, _ = deduplicate_entities(nodes, edges, communities=communities_diff) - assert len(result_with) <= len(result_without) - - -def test_empty_inputs(): - result_nodes, result_edges = deduplicate_entities([], [], communities={}) - assert result_nodes == [] - assert result_edges == [] - - -def test_single_node_no_crash(): - nodes = _make_nodes("UserService") - result_nodes, _ = deduplicate_entities(nodes, [], communities={}) - assert len(result_nodes) == 1 -``` - -- [ ] **Step 2: Run tests — verify they all fail** - -```bash -cd /home/safi/graphify -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all fail with `ModuleNotFoundError: No module named 'graphify.dedup'` - -- [ ] **Step 3: Create `graphify/dedup.py`** - -```python -"""Entity deduplication pipeline for graphify knowledge graphs. - -Pipeline: exact normalization → entropy gate → MinHash/LSH blocking → -Jaro-Winkler verification → same-community boost → union-find merge. -""" -from __future__ import annotations -import math -import re -from collections import defaultdict -from typing import Any - -from datasketch import MinHash, MinHashLSH -from rapidfuzz import fuzz - - -# ── helpers ─────────────────────────────────────────────────────────────────── - -def _norm(label: str) -> str: - """Lowercase + collapse non-alphanumeric runs to space.""" - return re.sub(r"[^a-z0-9]+", " ", label.lower()).strip() - - -def _entropy(label: str) -> float: - """Shannon entropy in bits/char of the normalised label.""" - s = _norm(label) - if not s: - return 0.0 - freq: dict[str, int] = defaultdict(int) - for ch in s: - freq[ch] += 1 - n = len(s) - return -sum((c / n) * math.log2(c / n) for c in freq.values()) - - -def _shingles(text: str, k: int = 3) -> set[str]: - """Return k-gram character shingles of text.""" - if len(text) < k: - return {text} - return {text[i : i + k] for i in range(len(text) - k + 1)} - - -def _make_minhash(text: str, num_perm: int = 128) -> MinHash: - m = MinHash(num_perm=num_perm) - for shingle in _shingles(text): - m.update(shingle.encode("utf-8")) - return m - - -# ── union-find ──────────────────────────────────────────────────────────────── - -class _UF: - def __init__(self) -> None: - self._parent: dict[str, str] = {} - - def find(self, x: str) -> str: - self._parent.setdefault(x, x) - while self._parent[x] != x: - self._parent[x] = self._parent[self._parent[x]] - x = self._parent[x] - return x - - def union(self, x: str, y: str) -> None: - self._parent.setdefault(x, x) - self._parent.setdefault(y, y) - rx, ry = self.find(x), self.find(y) - if rx != ry: - self._parent[ry] = rx - - def components(self) -> dict[str, list[str]]: - groups: dict[str, list[str]] = defaultdict(list) - for x in self._parent: - groups[self.find(x)].append(x) - return dict(groups) - - -# ── main entry point ────────────────────────────────────────────────────────── - -_ENTROPY_THRESHOLD = 2.5 -_LSH_THRESHOLD = 0.7 -_JW_THRESHOLD = 92.0 # rapidfuzz returns 0-100 -_COMMUNITY_BOOST = 5.0 # added to score when both nodes share community -_MERGE_THRESHOLD = 92.0 # final threshold after boost -_NUM_PERM = 128 -_CHUNK_SUFFIX = re.compile(r"_c\d+$") - - -def deduplicate_entities( - nodes: list[dict], - edges: list[dict], - *, - communities: dict[str, int], -) -> tuple[list[dict], list[dict]]: - """Deduplicate near-identical entities in a knowledge graph. - - Args: - nodes: list of node dicts with at minimum {"id": str, "label": str} - edges: list of edge dicts with {"source": str, "target": str, ...} - communities: mapping of node_id -> community_id (from cluster()) - - Returns: - (deduped_nodes, deduped_edges) with edges rewired to survivors - """ - if len(nodes) <= 1: - return nodes, edges - - # ── pass 1: exact normalization (always runs) ───────────────────────────── - norm_to_nodes: dict[str, list[dict]] = defaultdict(list) - for node in nodes: - key = _norm(node.get("label", node.get("id", ""))) - if key: - norm_to_nodes[key].append(node) - - uf = _UF() - for key, group in norm_to_nodes.items(): - if len(group) > 1: - winner = _pick_winner(group) - for node in group: - uf.union(winner["id"], node["id"]) - - exact_merges = sum(len(g) - 1 for g in norm_to_nodes.values() if len(g) > 1) - - # ── pass 2: MinHash/LSH + Jaro-Winkler (high-entropy nodes only) ───────── - # Build candidate set: one representative per exact-norm group - candidates: list[dict] = [] - seen_norms: set[str] = set() - for node in nodes: - key = _norm(node.get("label", node.get("id", ""))) - if key and key not in seen_norms: - seen_norms.add(key) - if _entropy(node.get("label", "")) >= _ENTROPY_THRESHOLD: - candidates.append(node) - - fuzzy_merges = 0 - if len(candidates) >= 2: - lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) - minhashes: dict[str, MinHash] = {} - - for node in candidates: - norm_label = _norm(node.get("label", node.get("id", ""))) - m = _make_minhash(norm_label) - minhashes[node["id"]] = m - try: - lsh.insert(node["id"], m) - except ValueError: - pass # duplicate key in LSH — already inserted - - for node in candidates: - node_id = node["id"] - norm_label = _norm(node.get("label", node.get("id", ""))) - neighbors = lsh.query(minhashes[node_id]) - - for neighbor_id in neighbors: - if neighbor_id == node_id: - continue - if uf.find(node_id) == uf.find(neighbor_id): - continue # already merged - - # Find the neighbour node - neighbor = next((n for n in candidates if n["id"] == neighbor_id), None) - if neighbor is None: - continue - - neighbor_norm = _norm(neighbor.get("label", neighbor.get("id", ""))) - score = fuzz.jaro_winkler_similarity(norm_label, neighbor_norm) * 100 - - # Same-community boost - c1 = communities.get(node_id) - c2 = communities.get(neighbor_id) - if c1 is not None and c2 is not None and c1 == c2: - score += _COMMUNITY_BOOST - - if score >= _MERGE_THRESHOLD: - all_nodes_in_group = norm_to_nodes.get(norm_label, [node]) + \ - norm_to_nodes.get(neighbor_norm, [neighbor]) - winner = _pick_winner(all_nodes_in_group) - uf.union(winner["id"], node_id) - uf.union(winner["id"], neighbor_id) - fuzzy_merges += 1 - - # ── build remap table from union-find components ────────────────────────── - components = uf.components() - remap: dict[str, str] = {} - surviving_ids: set[str] = set() - - for root, members in components.items(): - if len(members) == 1: - surviving_ids.add(root) - continue - group_nodes = [n for n in nodes if n["id"] in members] - winner = _pick_winner(group_nodes) if group_nodes else {"id": root} - winner_id = winner["id"] - surviving_ids.add(winner_id) - for member in members: - if member != winner_id: - remap[member] = winner_id - - # ── apply remap ─────────────────────────────────────────────────────────── - if not remap: - return nodes, edges - - total = len(remap) - msg = f"[graphify] Deduplicated {total} node(s)" - if exact_merges: - msg += f" ({exact_merges} exact" - if fuzzy_merges: - msg += f", {fuzzy_merges} fuzzy" - msg += ")" - print(msg + ".", flush=True) - - deduped_nodes = [n for n in nodes if n["id"] not in remap] - deduped_edges = [] - for edge in edges: - e = dict(edge) - e["source"] = remap.get(e["source"], e["source"]) - e["target"] = remap.get(e["target"], e["target"]) - if e["source"] != e["target"]: - deduped_edges.append(e) - - return deduped_nodes, deduped_edges - - -def _pick_winner(nodes: list[dict]) -> dict: - """Pick the canonical survivor: prefer no chunk suffix, then shorter ID.""" - if not nodes: - raise ValueError("Cannot pick winner from empty list") - def _score(n: dict) -> tuple[int, int]: - has_suffix = bool(_CHUNK_SUFFIX.search(n["id"])) - return (1 if has_suffix else 0, len(n["id"])) - return min(nodes, key=_score) -``` - -- [ ] **Step 4: Run tests — verify they pass** - -```bash -cd /home/safi/graphify -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests pass. - -- [ ] **Step 5: Run full suite — no regressions** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: same pass count as before (532 passed, 5 failed SQL). - -- [ ] **Step 6: Commit** - -```bash -git add graphify/dedup.py tests/test_dedup.py -git commit -m "Add graphify/dedup.py: entropy gate + MinHash/LSH + Jaro-Winkler entity deduplication" -``` - ---- - -## Task 3: Wire dedup into `build.py` - -**Files:** -- Modify: `graphify/build.py` (lines 119-137 `build()`, lines 191-244 `build_merge()`) - -- [ ] **Step 1: Write failing test** - -Add to `tests/test_dedup.py`: - -```python -def test_build_calls_dedup(): - """build() should deduplicate near-identical nodes across extractions.""" - from graphify.build import build - chunk1 = { - "nodes": [{"id": "graphextractor", "label": "GraphExtractor", "source_file": "a.py"}], - "edges": [], - } - chunk2 = { - "nodes": [{"id": "graph_extractor", "label": "Graph Extractor", "source_file": "b.py"}], - "edges": [], - } - G = build([chunk1, chunk2]) - # Should have merged to 1 node - assert G.number_of_nodes() == 1 -``` - -- [ ] **Step 2: Run — verify it fails** - -```bash -venv/bin/python -m pytest tests/test_dedup.py::test_build_calls_dedup -v --tb=short -``` - -Expected: FAIL — two separate nodes exist (no dedup wired yet). - -- [ ] **Step 3: Modify `build()` in `build.py`** - -Find `build()` at line 119. Current: - -```python -def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph: - ... - combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - for ext in extractions: - combined["nodes"].extend(ext.get("nodes", [])) - combined["edges"].extend(ext.get("edges", [])) - combined["hyperedges"].extend(ext.get("hyperedges", [])) - combined["input_tokens"] += ext.get("input_tokens", 0) - combined["output_tokens"] += ext.get("output_tokens", 0) - return build_from_json(combined, directed=directed) -``` - -Replace with: - -```python -def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True) -> nx.Graph: - """Merge multiple extraction results into one graph. - - directed=True produces a DiGraph. dedup=True (default) runs entity - deduplication before building the NetworkX graph. - """ - from graphify.dedup import deduplicate_entities - combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - for ext in extractions: - combined["nodes"].extend(ext.get("nodes", [])) - combined["edges"].extend(ext.get("edges", [])) - combined["hyperedges"].extend(ext.get("hyperedges", [])) - combined["input_tokens"] += ext.get("input_tokens", 0) - combined["output_tokens"] += ext.get("output_tokens", 0) - if dedup and combined["nodes"]: - combined["nodes"], combined["edges"] = deduplicate_entities( - combined["nodes"], combined["edges"], communities={} - ) - return build_from_json(combined, directed=directed) -``` - -- [ ] **Step 4: Modify `build_merge()` signature in `build.py`** - -Find `build_merge()` at line 191. Update signature and the internal `build()` call: - -```python -def build_merge( - new_chunks: list[dict], - graph_path: str | Path = "graphify-out/graph.json", - prune_sources: list[str] | None = None, - *, - directed: bool = False, - dedup: bool = True, -) -> nx.Graph: -``` - -Inside `build_merge`, find the line `G = build(all_chunks, directed=directed)` (around line 222) and replace with: - -```python - G = build(all_chunks, directed=directed, dedup=dedup) -``` - -Also update the safety-check block (around lines 235-242). When `dedup=True` or `prune_sources` is set, the graph can legitimately shrink — skip the shrink guard: - -```python - if graph_path.exists() and not dedup and not prune_sources: - existing_n = len(existing_nodes) - new_n = G.number_of_nodes() - if new_n < existing_n: - raise ValueError( - f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " - f"Pass prune_sources explicitly if you intend to remove nodes." - ) -``` - -- [ ] **Step 5: Run tests** - -```bash -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass including `test_build_calls_dedup`. - -- [ ] **Step 6: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532 passed, 5 failed (same pre-existing SQL failures). - -- [ ] **Step 7: Commit** - -```bash -git add graphify/build.py -git commit -m "Wire deduplicate_entities into build() and build_merge()" -``` - ---- - -## Task 4: Incremental updates — semantic cache + manifest in `__main__.py` - -**Files:** -- Modify: `graphify/__main__.py` (lines 1971–2117, the extract block) -- Create: `tests/test_incremental.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_incremental.py`: - -```python -"""Integration tests for incremental graphify extract behavior.""" -from __future__ import annotations -import json -import subprocess -import sys -from pathlib import Path - -import pytest - -PYTHON = sys.executable -FIXTURES = Path(__file__).parent / "fixtures" - - -def _run(args: list[str], cwd: Path) -> subprocess.CompletedProcess: - return subprocess.run( - [PYTHON, "-m", "graphify"] + args, - cwd=cwd, - capture_output=True, - text=True, - ) - - -def _make_docs_corpus(tmp_path: Path) -> Path: - """Create a minimal docs corpus with manifest + graph.json for incremental testing.""" - docs = tmp_path / "docs" - docs.mkdir() - (docs / "intro.md").write_text("# Introduction\nThis doc introduces the system.") - (docs / "api.md").write_text("# API Reference\nThe API has endpoints.") - return docs - - -def test_manifest_written_after_extract(tmp_path): - """After a full extract run, manifest.json must exist.""" - # We can't do a real LLM extract in CI, but we can test that the manifest - # path is resolved correctly by checking that a missing API key exits early - # before writing the manifest — and that the path would be correct. - docs = _make_docs_corpus(tmp_path) - r = _run(["extract", str(docs)], tmp_path) - # Should fail with no API key — but NOT with a path error - assert "no LLM API key" in r.stderr or r.returncode != 0 - # manifest should NOT exist (run failed before writing) - manifest = docs / "graphify-out" / "manifest.json" - assert not manifest.exists() - - -def test_incremental_mode_detected_via_manifest(tmp_path): - """If manifest.json + graph.json exist, incremental mode message is shown.""" - docs = _make_docs_corpus(tmp_path) - out = docs / "graphify-out" - out.mkdir() - # Fake a prior successful run - (out / "graph.json").write_text(json.dumps({"nodes": [], "links": []})) - (out / "manifest.json").write_text(json.dumps({"document": [str(docs / "intro.md")]})) - r = _run(["extract", str(docs)], tmp_path) - # Should show incremental scan message (even if it fails on API key) - assert "incremental" in r.stderr.lower() or "incremental" in r.stdout.lower() or r.returncode != 0 - - -def test_no_incremental_without_manifest(tmp_path): - """Without manifest.json, full scan message is shown.""" - docs = _make_docs_corpus(tmp_path) - r = _run(["extract", str(docs)], tmp_path) - # Full scan message (not incremental), then fails on API key - assert "incremental" not in r.stdout -``` - -- [ ] **Step 2: Run — verify tests fail or pass trivially** - -```bash -venv/bin/python -m pytest tests/test_incremental.py -v --tb=short 2>&1 | tail -20 -``` - -Note current behavior before changes. - -- [ ] **Step 3: Update the `elif cmd == "extract":` block in `__main__.py`** - -Find line 1971 (the `from graphify.detect import detect as _detect` line). Replace the detect + file-list block (lines 1971–1984) with: - -```python - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() - - if incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental(target, manifest_path=str(manifest_path)) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect(target) - - files_by_type = detection.get("files", {}) - if incremental_mode: - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - else: - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - if incremental_mode: - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) -``` - -- [ ] **Step 4: Wrap the semantic LLM call with semantic cache** - -Find the semantic extraction block (lines 1998–2024). Replace with: - -```python - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - sem_cache_hits = 0 - sem_cache_misses = 0 - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=target) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - backend=backend, - root=target, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print(f"[graphify extract] semantic extraction failed: {exc}", file=sys.stderr) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=target, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) -``` - -- [ ] **Step 5: Replace `build_from_json` with `build_merge` in incremental mode** - -Find the build block starting around line 2063. Replace: - -```python - from graphify.build import build_from_json as _build_from_json - ... - G = _build_from_json(merged) -``` - -With: - -```python - from graphify.build import ( - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - - if incremental_mode: - G = _build_merge( - [merged], - graph_path=graph_json_path, - prune_sources=deleted_files or None, - dedup=True, - ) - else: - G = _build_from_json(merged) -``` - -- [ ] **Step 6: Add manifest save after successful write** - -Find the `analysis_path.write_text(...)` line (around line 2101). After it, add: - -```python - try: - _save_manifest( - detection.get("files", {}), - manifest_path=str(manifest_path), - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) -``` - -Also add the same block in the `--no-cluster` path after `graph_json_path.write_text(...)` (around line 2043). - -- [ ] **Step 7: Add `--dedup-llm` flag parsing** - -In the args-parsing while loop (around lines 1913–1928), add: - -```python - elif a == "--dedup-llm": - dedup_llm = True; i += 1 -``` - -And initialize `dedup_llm = False` before the loop. - -- [ ] **Step 8: Update summary print at end** - -Replace the final summary lines (around 2104–2116) with: - -```python - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) -``` - -- [ ] **Step 9: Run incremental tests** - -```bash -venv/bin/python -m pytest tests/test_incremental.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass. - -- [ ] **Step 10: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed (pre-existing SQL). - -- [ ] **Step 11: Commit** - -```bash -git add graphify/__main__.py tests/test_incremental.py -git commit -m "Add incremental updates to graphify extract: semantic cache + build_merge + manifest" -``` - ---- - -## Task 5: Add `--dedup-llm` tiebreaker to `dedup.py` - -**Files:** -- Modify: `graphify/dedup.py` - -- [ ] **Step 1: Write failing test** - -Add to `tests/test_dedup.py`: - -```python -def test_dedup_llm_flag_accepted(): - """deduplicate_entities accepts dedup_llm_backend without crashing when no ambiguous pairs exist.""" - nodes = _make_nodes("UserService", "OrderService") - edges = [] - # Should not crash even with dedup_llm_backend set — just nothing to resolve - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}, dedup_llm_backend=None) - assert len(result_nodes) == 2 -``` - -- [ ] **Step 2: Run — verify it fails** - -```bash -venv/bin/python -m pytest tests/test_dedup.py::test_dedup_llm_flag_accepted -v --tb=short -``` - -Expected: FAIL — `deduplicate_entities` does not accept `dedup_llm_backend` kwarg. - -- [ ] **Step 3: Add `dedup_llm_backend` param to `deduplicate_entities`** - -Update the signature in `graphify/dedup.py`: - -```python -def deduplicate_entities( - nodes: list[dict], - edges: list[dict], - *, - communities: dict[str, int], - dedup_llm_backend: str | None = None, -) -> tuple[list[dict], list[dict]]: -``` - -After the fuzzy merge loop (before building the remap table), add the LLM tiebreaker block: - -```python - # ── pass 3: LLM tiebreaker for ambiguous pairs (opt-in) ────────────────── - if dedup_llm_backend is not None: - _llm_tiebreak(candidates, uf, communities, backend=dedup_llm_backend) -``` - -Add the helper at the bottom of `dedup.py`: - -```python -def _llm_tiebreak( - candidates: list[dict], - uf: _UF, - communities: dict[str, int], - *, - backend: str, - batch_size: int = 30, - low: float = 75.0, - high: float = 92.0, -) -> None: - """Batch-resolve ambiguous pairs (score in [low, high)) via LLM.""" - try: - from graphify.llm import extract_corpus_parallel as _llm # noqa: F401 - import os - from graphify.llm import BACKENDS - env_key = BACKENDS.get(backend, {}).get("env_key", "") - if not os.environ.get(env_key): - print(f"[graphify] --dedup-llm: {env_key} not set, skipping LLM tiebreaker.", flush=True) - return - except ImportError: - return - - # Collect ambiguous pairs - ambiguous: list[tuple[dict, dict, float]] = [] - for i, node in enumerate(candidates): - norm_i = _norm(node.get("label", node.get("id", ""))) - for j in range(i + 1, len(candidates)): - neighbor = candidates[j] - if uf.find(node["id"]) == uf.find(neighbor["id"]): - continue - norm_j = _norm(neighbor.get("label", neighbor.get("id", ""))) - score = fuzz.jaro_winkler_similarity(norm_i, norm_j) * 100 - c1 = communities.get(node["id"]) - c2 = communities.get(neighbor["id"]) - if c1 is not None and c2 is not None and c1 == c2: - score += _COMMUNITY_BOOST - if low <= score < high: - ambiguous.append((node, neighbor, score)) - - if not ambiguous: - return - - # Batch into groups of batch_size and call LLM - try: - from graphify.llm import _call_llm - except ImportError: - return - - for batch_start in range(0, len(ambiguous), batch_size): - batch = ambiguous[batch_start : batch_start + batch_size] - pairs_text = "\n".join( - f"{i+1}. \"{a['label']}\" vs \"{b['label']}\"" - for i, (a, b, _) in enumerate(batch) - ) - prompt = ( - "For each pair below, answer only 'yes' or 'no': are they the same real-world concept?\n\n" - f"{pairs_text}\n\n" - "Reply with one line per pair: '1. yes', '2. no', etc." - ) - try: - response = _call_llm(prompt, backend=backend, max_tokens=200) - lines = response.strip().splitlines() - for line in lines: - line = line.strip() - if not line: - continue - parts = line.split(".", 1) - if len(parts) != 2: - continue - try: - idx = int(parts[0].strip()) - 1 - except ValueError: - continue - if 0 <= idx < len(batch): - answer = parts[1].strip().lower() - if answer.startswith("yes"): - a, b, _ = batch[idx] - winner = _pick_winner([a, b]) - uf.union(winner["id"], a["id"]) - uf.union(winner["id"], b["id"]) - except Exception as exc: - print(f"[graphify] --dedup-llm batch failed: {exc}", flush=True) -``` - -- [ ] **Step 4: Run tests** - -```bash -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass. - -- [ ] **Step 5: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed. - -- [ ] **Step 6: Commit** - -```bash -git add graphify/dedup.py tests/test_dedup.py -git commit -m "Add --dedup-llm LLM tiebreaker to dedup pipeline" -``` - ---- - -## Task 6: Update CHANGELOG + bump version to 0.7.5 - -**Files:** -- Modify: `CHANGELOG.md` -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add changelog entry** - -Add at the top of `CHANGELOG.md`: - -```markdown -## 0.7.5 (2026-05-04) - -- Feat: `graphify extract` now runs incrementally — auto-detects prior `manifest.json` and re-extracts only changed/new files; semantic results cached by content hash so unchanged docs cost zero LLM tokens on repeat runs (#698) -- Feat: Entity deduplication pipeline runs on every build — entropy gate + MinHash/LSH blocking + Jaro-Winkler verification + same-community boost collapses near-duplicate entities (typos, spacing, plurals) before clustering -- Feat: `--dedup-llm` flag for `graphify extract` — optional LLM tiebreaker for ambiguous entity pairs (~$0.01 for 10k-node graphs), off by default -- Deps: `datasketch` and `rapidfuzz` added as base dependencies -``` - -- [ ] **Step 2: Bump version** - -In `pyproject.toml`, change: -```toml -version = "0.7.4" -``` -to: -```toml -version = "0.7.5" -``` - -- [ ] **Step 3: Run full suite one final time** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed (pre-existing SQL only). - -- [ ] **Step 4: Commit** - -```bash -git add CHANGELOG.md pyproject.toml -git commit -m "bump version to 0.7.5" -``` - ---- - -## Self-Review - -**Spec coverage:** -- [x] Semantic cache wrapping → Task 4 steps 3-4 -- [x] Incremental auto-detection via manifest → Task 4 step 3 -- [x] `build_merge` with `prune_sources` → Task 4 step 5 -- [x] Manifest saved on success only → Task 4 step 6 -- [x] Summary print → Task 4 step 8 -- [x] `dedup.py` new module → Task 2 -- [x] Entropy gate → Task 2 step 3 -- [x] MinHash/LSH blocking → Task 2 step 3 -- [x] Jaro-Winkler verification → Task 2 step 3 -- [x] Same-community boost → Task 2 step 3 -- [x] Union-find merge → Task 2 step 3 -- [x] `--dedup-llm` tiebreaker → Task 5 -- [x] Wire dedup into `build()` and `build_merge()` → Task 3 -- [x] `datasketch` + `rapidfuzz` deps → Task 1 -- [x] Tests for all dedup steps → Task 2 + 3 -- [x] Tests for incremental → Task 4 -- [x] CHANGELOG + version bump → Task 6 - -**Placeholder scan:** None found. - -**Type consistency:** `deduplicate_entities(nodes, edges, *, communities, dedup_llm_backend=None)` used consistently in Task 2, Task 3, Task 5. `build(extractions, *, directed, dedup)` consistent in Task 3. `build_merge(..., dedup=True)` consistent in Task 3 and Task 4. diff --git a/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md b/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md deleted file mode 100644 index 217e24d65..000000000 --- a/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md +++ /dev/null @@ -1,133 +0,0 @@ -# Design: Incremental Updates + Entity Deduplication - -**Date:** 2026-05-04 -**Issues:** #698 (incremental updates), entity deduplication (no issue, proactive) -**Branch:** v7 - ---- - -## Problem - -1. `graphify extract` rebuilds the full graph from scratch every run — re-sends all files to the LLM regardless of what changed. For a 1000-file Markdown corpus updated daily this is expensive. - -2. LLM extraction is chunk-by-chunk — the same real-world concept can get different labels across chunks (`AuthManager`, `AuthenticationManager`, `auth_mgr`). No semantic dedup exists beyond exact string normalization. - ---- - -## Pipeline (every `graphify extract` run) - -``` -detect (full or incremental, auto-detected) - ↓ -AST extract (code files, AST cache-aware) - ↓ -Semantic LLM extract (doc/paper/image files, semantic cache-aware) - ↓ -build_merge (merge into existing graph, prune deleted nodes) - ↓ -deduplicate_entities (normalize → entropy gate → MinHash/LSH → Jaro-Winkler → community boost → optional LLM) - ↓ -cluster (full graph, always re-run) - ↓ -score_all + god_nodes + surprising_connections - ↓ -write graph.json + .graphify_analysis.json + manifest.json -``` - ---- - -## Feature 1: Incremental Updates - -### Auto-detection -If `graphify-out/manifest.json` + `graphify-out/graph.json` both exist → incremental mode. No flag needed. First run is always full. - -### Incremental mode changes -- `detect_incremental(target)` instead of `detect(target)` — returns `new_files`, `unchanged_files`, `deleted_files` -- Only `new_files` go through AST + LLM extraction -- `build_merge(new_chunks, prune_sources=deleted_files)` instead of `build_from_json` — merges into existing graph, prunes nodes from deleted files -- `manifest.json` written only on successful completion (crash mid-run does not corrupt next run's diff) - -### Semantic cache (both full and incremental mode) -- Before LLM call: `check_semantic_cache(files)` splits into `(cached_results, uncached_files)` -- Only `uncached_files` sent to `extract_corpus_parallel` -- After LLM call: `save_semantic_cache(fresh_results)` — keyed by content hash -- On file rename: `source_file` updated to new path in cached result (same pattern as existing AST cache) - -### Output summary -``` -[graphify extract] incremental: 20 changed, 980 cached, 2 deleted -[graphify extract] graph: 4,821 nodes, 12,304 edges, 43 communities -[graphify extract] tokens: 18,432 in / 6,201 out, est. cost: $0.08 -``` - -### Files changed -- `graphify/__main__.py` — `elif cmd == "extract":` block only (~5 targeted changes) - ---- - -## Feature 2: Entity Deduplication - -### New module: `graphify/dedup.py` -Single responsibility. Called from `build.py` after graph construction. Returns deduplicated `(nodes, edges)`. - -### Pipeline - -**Step 1 — Exact normalization** -Wire up the dormant `deduplicate_by_label` in `build.py`. Catches case/punctuation variants across files. Free win, already written. - -**Step 2 — Entropy gate** -Skip fuzzy matching on labels with < 2.5 bits/char entropy. Short ambiguous names (`"AI"`, `"DB"`, `"x"`) are too risky to auto-merge. Only high-entropy labels proceed to steps 3-4. - -**Step 3 — MinHash + LSH blocking** (`datasketch`) -3-gram shingles, 128 permutations, threshold 0.7. Generates candidate pairs in O(n) instead of O(n²). Sub-second at 10k nodes. - -**Step 4 — Jaro-Winkler verification** (`rapidfuzz`) -Each candidate pair verified at ≥ 0.92. Catches typos, plurals, spacing variants. Pairs below threshold discarded. - -**Step 5 — Same-community boost** -Pairs where both nodes share a Leiden community ID get +0.05 score bonus. Graphify-specific advantage — community structure is a strong signal that GraphRAG/LightRAG don't exploit. - -**Step 6 — Union-find merge** -Confirmed pairs fed into union-find → connected components → each component merged into one node. Edges rewired to survivor. Self-loops dropped. Prefer shorter non-chunk-suffixed IDs as survivor. - -**Step 7 — Optional LLM tiebreaker** (`--dedup-llm` flag) -Ambiguous pairs (score 0.75–0.85) batched in groups of 30, one LLM call per batch. ~$0.01 total for 10k nodes. Off by default. - -### Integration point -Dedup runs after `build_merge` / `build_from_json`, before `cluster`. Order matters: cleaner graph → better community detection. - -```python -# in build.py -G = build_merge(...) # or build_from_json -G = deduplicate_entities(G) # new step -communities = cluster(G) # unchanged -``` - -### New dependencies -- `datasketch` — always required (added to `[project.dependencies]`) -- `rapidfuzz` — always required (added to `[project.dependencies]`) -- No `sentence-transformers` / PyTorch dependency - -### Files changed -- `graphify/dedup.py` — new module, full pipeline -- `graphify/build.py` — call `deduplicate_entities` after graph construction; wire dormant `deduplicate_by_label` -- `graphify/__main__.py` — add `--dedup-llm` flag parsing in extract block -- `pyproject.toml` — add `datasketch`, `rapidfuzz` to base dependencies - ---- - -## Testing - -- Unit tests for each dedup step in isolation (`tests/test_dedup.py`) -- Integration test: two chunks with overlapping entity labels → single merged node in output graph -- Incremental test: run extract twice, assert second run makes zero LLM calls for unchanged files -- Rename test: rename a file, assert cache hit and `source_file` updated correctly -- Delete test: delete a file, assert its nodes are pruned from graph - ---- - -## Non-goals - -- `--dedup embed` (MiniLM cosine) — explicitly excluded, no PyTorch dependency -- Incremental support for `graphify update` (AST-only) — already handled by existing AST cache -- Dedup across different graph.json files (merge two graphs) — separate feature diff --git a/docs/translations/README.ar-SA.md b/docs/translations/README.ar-SA.md index f3f4a78c2..e6bd35759 100644 --- a/docs/translations/README.ar-SA.md +++ b/docs/translations/README.ar-SA.md @@ -32,7 +32,7 @@ graphify-out/ ├── graph.html رسم بياني تفاعلي — افتحه في أي متصفح، انقر على العقد، ابحث، صفّ ├── GRAPH_REPORT.md عقد الإله، الاتصالات المفاجئة، الأسئلة المقترحة -├── graph.json رسم بياني دائم — استعلم بعد أسابيع دون إعادة القراءة +├── graph.helix رسم بياني دائم — استعلم بعد أسابيع دون إعادة القراءة └── cache/ ذاكرة تخزين مؤقت SHA256 — إعادة التشغيل تعالج الملفات المتغيرة فقط ``` @@ -56,7 +56,7 @@ dist/ ## كيف يعمل -يعمل graphify في ثلاث مراحل. أولاً، تمريرة AST حتمية تستخرج البنية من ملفات الكود (الفئات، الدوال، الاستيرادات، رسوم بيانية الاستدعاء، docstrings، تعليقات المبرر) — دون الحاجة إلى LLM. ثانياً، يتم نسخ ملفات الفيديو والصوت محلياً باستخدام faster-whisper. ثالثاً، تعمل عوامل Claude الفرعية بالتوازي على المستندات والأوراق البحثية والصور والنصوص المكتوبة لاستخراج المفاهيم والعلاقات ومبررات التصميم. يتم دمج النتائج في رسم بياني NetworkX وتجميعها باستخدام Leiden وتصديرها كـ HTML تفاعلي وJSON قابل للاستعلام وتقرير تدقيق بلغة طبيعية. +يعمل graphify في ثلاث مراحل. أولاً، تمريرة AST حتمية تستخرج البنية من ملفات الكود (الفئات، الدوال، الاستيرادات، رسوم بيانية الاستدعاء، docstrings، تعليقات المبرر) — دون الحاجة إلى LLM. ثانياً، يتم نسخ ملفات الفيديو والصوت محلياً باستخدام faster-whisper. ثالثاً، تعمل عوامل Claude الفرعية بالتوازي على المستندات والأوراق البحثية والصور والنصوص المكتوبة لاستخراج المفاهيم والعلاقات ومبررات التصميم. يتم دمج النتائج في رسم بياني Helix وتجميعها باستخدام Leiden وتصديرها كـ HTML تفاعلي وJSON قابل للاستعلام وتقرير تدقيق بلغة طبيعية. **التجميع مبني على طوبولوجيا الرسم البياني — بدون embeddings.** يجد Leiden المجتمعات بواسطة كثافة الحواف. حواف التشابه الدلالي التي يستخرجها Claude (`semantically_similar_to`، مصنفة INFERRED) موجودة بالفعل في الرسم البياني. بنية الرسم البياني هي إشارة التشابه — لا حاجة لخطوة embedding منفصلة أو قاعدة بيانات متجهية. @@ -162,7 +162,7 @@ graphify watch ./src # تحديث تلقائي للرسم البي ## المكدس التقني -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. استخراج دلالي عبر Claude أو GPT-4 أو نموذج منصتك. نسخ الفيديو عبر faster-whisper + yt-dlp (اختياري). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. استخراج دلالي عبر Claude أو GPT-4 أو نموذج منصتك. نسخ الفيديو عبر faster-whisper + yt-dlp (اختياري). ## مبني على graphify — Penpax diff --git a/docs/translations/README.cs-CZ.md b/docs/translations/README.cs-CZ.md index db4cb2577..7150b6fe2 100644 --- a/docs/translations/README.cs-CZ.md +++ b/docs/translations/README.cs-CZ.md @@ -27,13 +27,13 @@ Plně multimodální. Přidejte kód, PDF, markdown, snímky obrazovky, diagramy graphify-out/ ├── graph.html interaktivní graf — otevřete v libovolném prohlížeči ├── GRAPH_REPORT.md boží uzly, překvapivá propojení, navrhované otázky -├── graph.json trvalý graf — dotazovatelný týdny poté +├── graph.helix trvalý graf — dotazovatelný týdny poté └── cache/ SHA256 cache — opakovaná spuštění zpracovávají pouze změněné soubory ``` ## Jak to funguje -graphify pracuje ve třech průchodech. Nejprve deterministický průchod AST extrahuje strukturu z kódových souborů bez LLM. Poté jsou video a zvukové soubory přepisovány lokálně pomocí faster-whisper. Nakonec sub-agenti Claude běží paralelně na dokumentech, článcích, obrázcích a přepisech. Výsledky jsou sloučeny do grafu NetworkX, clusterovány pomocí Leiden a exportovány jako interaktivní HTML, dotazovatelný JSON a auditní zpráva. +graphify pracuje ve třech průchodech. Nejprve deterministický průchod AST extrahuje strukturu z kódových souborů bez LLM. Poté jsou video a zvukové soubory přepisovány lokálně pomocí faster-whisper. Nakonec sub-agenti Claude běží paralelně na dokumentech, článcích, obrázcích a přepisech. Výsledky jsou sloučeny do grafu Helix, clusterovány pomocí Leiden a exportovány jako interaktivní HTML, dotazovatelný JSON a auditní zpráva. Každý vztah je označen `EXTRACTED`, `INFERRED` (se skóre spolehlivosti) nebo `AMBIGUOUS`. diff --git a/docs/translations/README.da-DK.md b/docs/translations/README.da-DK.md index b6ca65e6c..f022437a5 100644 --- a/docs/translations/README.da-DK.md +++ b/docs/translations/README.da-DK.md @@ -27,13 +27,13 @@ Fuldt multimodal. Tilføj kode, PDF'er, markdown, skærmbilleder, diagrammer, wh graphify-out/ ├── graph.html interaktiv graf — åbn i enhver browser ├── GRAPH_REPORT.md gudknuder, overraskende forbindelser, foreslåede spørgsmål -├── graph.json vedvarende graf — forespørgselsbar uger senere +├── graph.helix vedvarende graf — forespørgselsbar uger senere └── cache/ SHA256-cache — gentagne kørsler behandler kun ændrede filer ``` ## Sådan fungerer det -graphify arbejder i tre gennemløb. Først udtrækker et deterministisk AST-gennemløb struktur fra kodefiler uden LLM. Derefter transskriberes video- og lydfiler lokalt med faster-whisper. Endelig kører Claude-underagenter parallelt på dokumenter, artikler, billeder og transskriptioner. Resultaterne flettes ind i en NetworkX-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørgselsbar JSON og revisionsrapport. +graphify arbejder i tre gennemløb. Først udtrækker et deterministisk AST-gennemløb struktur fra kodefiler uden LLM. Derefter transskriberes video- og lydfiler lokalt med faster-whisper. Endelig kører Claude-underagenter parallelt på dokumenter, artikler, billeder og transskriptioner. Resultaterne flettes ind i en Helix-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørgselsbar JSON og revisionsrapport. Hver relation er mærket `EXTRACTED`, `INFERRED` (med konfidensscore) eller `AMBIGUOUS`. diff --git a/docs/translations/README.de-DE.md b/docs/translations/README.de-DE.md index f81067902..268cd0449 100644 --- a/docs/translations/README.de-DE.md +++ b/docs/translations/README.de-DE.md @@ -28,7 +28,7 @@ Vollständig multimodal. Leg Code, PDFs, Markdown, Screenshots, Diagramme, White graphify-out/ ├── graph.html interaktiver Graph — im Browser öffnen, Knoten anklicken, suchen, filtern ├── GRAPH_REPORT.md Gott-Knoten, überraschende Verbindungen, vorgeschlagene Fragen -├── graph.json persistenter Graph — Wochen später abfragen, ohne neu zu lesen +├── graph.helix persistenter Graph — Wochen später abfragen, ohne neu zu lesen └── cache/ SHA256-Cache — erneute Ausführungen verarbeiten nur geänderte Dateien ``` @@ -46,7 +46,7 @@ Gleiche Syntax wie `.gitignore`. Du kannst eine einzelne `.graphifyignore` im Re ## So funktioniert es -graphify läuft in drei Durchgängen. Zuerst extrahiert ein deterministischer AST-Durchgang Strukturen aus Code-Dateien (Klassen, Funktionen, Importe, Aufrufgraphen, Docstrings, Begründungskommentare) — ohne LLM. Zweitens werden Video- und Audiodateien lokal mit faster-whisper transkribiert, angetrieben durch einen domänenspezifischen Prompt aus Korpus-Gott-Knoten — Transkripte werden gecacht, sodass erneute Ausführungen sofort sind. Drittens laufen Claude-Subagenten parallel über Dokumente, Papers, Bilder und Transkripte, um Konzepte, Beziehungen und Designbegründungen zu extrahieren. Die Ergebnisse werden in einem NetworkX-Graphen zusammengeführt, mit Leiden-Community-Erkennung geclustert und als interaktives HTML, abfragbares JSON und ein Klartext-Audit-Report exportiert. +graphify läuft in drei Durchgängen. Zuerst extrahiert ein deterministischer AST-Durchgang Strukturen aus Code-Dateien (Klassen, Funktionen, Importe, Aufrufgraphen, Docstrings, Begründungskommentare) — ohne LLM. Zweitens werden Video- und Audiodateien lokal mit faster-whisper transkribiert, angetrieben durch einen domänenspezifischen Prompt aus Korpus-Gott-Knoten — Transkripte werden gecacht, sodass erneute Ausführungen sofort sind. Drittens laufen Claude-Subagenten parallel über Dokumente, Papers, Bilder und Transkripte, um Konzepte, Beziehungen und Designbegründungen zu extrahieren. Die Ergebnisse werden in einem Helix-Graphen zusammengeführt, mit Leiden-Community-Erkennung geclustert und als interaktives HTML, abfragbares JSON und ein Klartext-Audit-Report exportiert. **Clustering basiert auf Graph-Topologie — keine Embeddings.** Leiden findet Communities durch Kantendichte. Die semantischen Ähnlichkeitskanten, die Claude extrahiert (`semantically_similar_to`, markiert als INFERRED), sind bereits im Graphen, sodass sie die Community-Erkennung direkt beeinflussen. Die Graphstruktur ist das Ähnlichkeitssignal — kein separater Embedding-Schritt oder Vektordatenbank nötig. @@ -167,7 +167,7 @@ graphify sendet Dateiinhalte an die Modell-API deines KI-Assistenten für semant ## Tech-Stack -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantische Extraktion via Claude, GPT-4 oder welches Modell deine Plattform verwendet. Video-Transkription via faster-whisper + yt-dlp (optional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Semantische Extraktion via Claude, GPT-4 oder welches Modell deine Plattform verwendet. Video-Transkription via faster-whisper + yt-dlp (optional). ## Auf graphify aufgebaut — Penpax diff --git a/docs/translations/README.el-GR.md b/docs/translations/README.el-GR.md index 599e615a1..0c1fa857d 100644 --- a/docs/translations/README.el-GR.md +++ b/docs/translations/README.el-GR.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html διαδραστικός γράφος — ανοίξτε σε οποιοδήποτε πρόγραμμα περιήγησης ├── GRAPH_REPORT.md κόμβοι-θεοί, εκπληκτικές συνδέσεις, προτεινόμενες ερωτήσεις -├── graph.json επίμονος γράφος — μπορεί να υποβληθεί σε ερωτήματα εβδομάδες αργότερα +├── graph.helix επίμονος γράφος — μπορεί να υποβληθεί σε ερωτήματα εβδομάδες αργότερα └── cache/ κρυφή μνήμη SHA256 — επαναλαμβανόμενες εκτελέσεις επεξεργάζονται μόνο τα αλλαγμένα αρχεία ``` ## Πώς λειτουργεί -Το graphify λειτουργεί σε τρεις διελεύσεις. Πρώτα, μια ντετερμινιστική διέλευση AST εξάγει δομή από αρχεία κώδικα χωρίς LLM. Στη συνέχεια, τα αρχεία βίντεο και ήχου μεταγράφονται τοπικά με faster-whisper. Τέλος, οι υπο-πράκτορες Claude εκτελούνται παράλληλα σε έγγραφα, εργασίες, εικόνες και μεταγραφές. Τα αποτελέσματα συγχωνεύονται σε ένα γράφο NetworkX, ομαδοποιούνται με Leiden και εξάγονται ως διαδραστική HTML, JSON για ερωτήματα και αναφορά ελέγχου. +Το graphify λειτουργεί σε τρεις διελεύσεις. Πρώτα, μια ντετερμινιστική διέλευση AST εξάγει δομή από αρχεία κώδικα χωρίς LLM. Στη συνέχεια, τα αρχεία βίντεο και ήχου μεταγράφονται τοπικά με faster-whisper. Τέλος, οι υπο-πράκτορες Claude εκτελούνται παράλληλα σε έγγραφα, εργασίες, εικόνες και μεταγραφές. Τα αποτελέσματα συγχωνεύονται σε ένα γράφο Helix, ομαδοποιούνται με Leiden και εξάγονται ως διαδραστική HTML, JSON για ερωτήματα και αναφορά ελέγχου. Κάθε σχέση επισημαίνεται ως `EXTRACTED`, `INFERRED` (με βαθμολογία εμπιστοσύνης) ή `AMBIGUOUS`. diff --git a/docs/translations/README.es-ES.md b/docs/translations/README.es-ES.md index 7c58a2b27..bd73f7644 100644 --- a/docs/translations/README.es-ES.md +++ b/docs/translations/README.es-ES.md @@ -28,7 +28,7 @@ Totalmente multimodal. Deposita código, PDFs, markdown, capturas de pantalla, d graphify-out/ ├── graph.html grafo interactivo — abrir en cualquier navegador, hacer clic en nodos, buscar ├── GRAPH_REPORT.md nodos dios, conexiones sorprendentes, preguntas sugeridas -├── graph.json grafo persistente — consultar semanas después sin releer +├── graph.helix grafo persistente — consultar semanas después sin releer └── cache/ caché SHA256 — las re-ejecuciones solo procesan archivos modificados ``` @@ -46,7 +46,7 @@ Misma sintaxis que `.gitignore`. Puedes mantener un único `.graphifyignore` en ## Cómo funciona -graphify se ejecuta en tres pasadas. Primero, una pasada AST determinista extrae estructura de los archivos de código (clases, funciones, importaciones, grafos de llamadas, docstrings, comentarios de justificación) sin necesidad de LLM. Segundo, los archivos de video y audio se transcriben localmente con faster-whisper usando un prompt adaptado al dominio derivado de los nodos dios del corpus. Tercero, subagentes de Claude se ejecutan en paralelo sobre documentos, papers, imágenes y transcripciones para extraer conceptos, relaciones y justificaciones de diseño. Los resultados se fusionan en un grafo NetworkX, se agrupan con detección de comunidades Leiden, y se exportan como HTML interactivo, JSON consultable y un informe de auditoría en lenguaje natural. +graphify se ejecuta en tres pasadas. Primero, una pasada AST determinista extrae estructura de los archivos de código (clases, funciones, importaciones, grafos de llamadas, docstrings, comentarios de justificación) sin necesidad de LLM. Segundo, los archivos de video y audio se transcriben localmente con faster-whisper usando un prompt adaptado al dominio derivado de los nodos dios del corpus. Tercero, subagentes de Claude se ejecutan en paralelo sobre documentos, papers, imágenes y transcripciones para extraer conceptos, relaciones y justificaciones de diseño. Los resultados se fusionan en un grafo Helix, se agrupan con detección de comunidades Leiden, y se exportan como HTML interactivo, JSON consultable y un informe de auditoría en lenguaje natural. **El clustering se basa en la topología del grafo — sin embeddings.** Leiden encuentra comunidades por densidad de aristas. Las aristas de similitud semántica que Claude extrae (`semantically_similar_to`, marcadas como INFERRED) ya están en el grafo. La estructura del grafo es la señal de similitud — no se necesita paso de embedding separado ni base de datos vectorial. @@ -157,7 +157,7 @@ graphify envía contenido de archivos a la API del modelo de tu asistente IA par ## Stack técnico -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extracción semántica via Claude, GPT-4 o el modelo de tu plataforma. Transcripción de video via faster-whisper + yt-dlp (opcional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extracción semántica via Claude, GPT-4 o el modelo de tu plataforma. Transcripción de video via faster-whisper + yt-dlp (opcional). ## Construido sobre graphify — Penpax diff --git a/docs/translations/README.fa-IR.md b/docs/translations/README.fa-IR.md index 2851c0803..ae0819440 100644 --- a/docs/translations/README.fa-IR.md +++ b/docs/translations/README.fa-IR.md @@ -40,7 +40,7 @@ graphify-out/ ├── graph.html در هر مرورگری باز کنید — روی گره‌ها کلیک کنید، فیلتر کنید، جستجو کنید ├── GRAPH_REPORT.md نکات کلیدی: مفاهیم محوری، اتصالات شگفت‌انگیز، سؤالات پیشنهادی -└── graph.json گراف کامل — هر زمان بدون نیاز به بازخوانی فایل‌ها پرس‌وجو کنید +└── graph.helix گراف کامل — هر زمان بدون نیاز به بازخوانی فایل‌ها پرس‌وجو کنید ```
@@ -355,14 +355,14 @@ graphify-out/cost.json # فقط محلی ```bash # پرس‌وجو از گراف در ترمینال graphify query "جریان احراز هویت را نشان بده" -graphify query "چه چیزی DigestAuth را به Response متصل می‌کند؟" --graph graphify-out/graph.json +graphify query "چه چیزی DigestAuth را به Response متصل می‌کند؟" --graph graphify-out/graph.helix # نمایش گراف به‌عنوان سرور MCP -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix # یا سرویس‌دهی از طریق HTTP برای دسترسی تیمی: -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
diff --git a/docs/translations/README.fi-FI.md b/docs/translations/README.fi-FI.md index afe725881..8e69d6b22 100644 --- a/docs/translations/README.fi-FI.md +++ b/docs/translations/README.fi-FI.md @@ -27,13 +27,13 @@ Täysin multimodaalinen. Lisää koodia, PDF:iä, markdownia, kuvakaappauksia, k graphify-out/ ├── graph.html interaktiivinen graafi — avaa missä tahansa selaimessa ├── GRAPH_REPORT.md jumalsolmut, yllättävät yhteydet, ehdotetut kysymykset -├── graph.json pysyvä graafi — kyselytavissa viikkojen kuluttua +├── graph.helix pysyvä graafi — kyselytavissa viikkojen kuluttua └── cache/ SHA256-välimuisti — toistuvat ajot käsittelevät vain muuttuneet tiedostot ``` ## Miten se toimii -graphify toimii kolmessa läpiajossa. Ensin deterministinen AST-läpiajo poimii rakenteen kooditiedostoista ilman LLM:ää. Sitten video- ja äänitiedostot litteroidaan paikallisesti faster-whisperillä. Lopuksi Clauden ala-agentit suoritetaan rinnakkain asiakirjoissa, papereissa, kuvissa ja litteroinneissa. Tulokset yhdistetään NetworkX-graafiin, klusteroidaan Leidenillä ja viedään interaktiivisena HTML:nä, kyselytavissa olevana JSON:na ja tarkastusraporttina. +graphify toimii kolmessa läpiajossa. Ensin deterministinen AST-läpiajo poimii rakenteen kooditiedostoista ilman LLM:ää. Sitten video- ja äänitiedostot litteroidaan paikallisesti faster-whisperillä. Lopuksi Clauden ala-agentit suoritetaan rinnakkain asiakirjoissa, papereissa, kuvissa ja litteroinneissa. Tulokset yhdistetään Helix-graafiin, klusteroidaan Leidenillä ja viedään interaktiivisena HTML:nä, kyselytavissa olevana JSON:na ja tarkastusraporttina. Jokainen suhde on merkitty `EXTRACTED`, `INFERRED` (luottamuspisteineen) tai `AMBIGUOUS`. diff --git a/docs/translations/README.fil-PH.md b/docs/translations/README.fil-PH.md index e318eb68e..4c97c1bd4 100644 --- a/docs/translations/README.fil-PH.md +++ b/docs/translations/README.fil-PH.md @@ -27,13 +27,13 @@ Ganap na multimodal. Magdagdag ng code, PDF, markdown, mga screenshot, diagram, graphify-out/ ├── graph.html interactive na graph — buksan sa kahit anong browser ├── GRAPH_REPORT.md mga god node, nakakagulat na koneksyon, mga iminumungkahing tanong -├── graph.json persistent na graph — maaaring i-query kahit pagkalipas ng mga linggo +├── graph.helix persistent na graph — maaaring i-query kahit pagkalipas ng mga linggo └── cache/ SHA256 cache — ang mga pag-uulit ay nagpo-proseso lang ng mga nabagong file ``` ## Paano Gumagana -Gumagana ang graphify sa tatlong pass. Una, isang deterministikong AST pass ang nag-e-extract ng istruktura mula sa mga code file nang walang LLM. Pagkatapos, ang mga video at audio file ay tina-transcribe nang lokal gamit ang faster-whisper. Panghuli, mga Claude sub-agent ang tumatakbo nang magkakasabay sa mga dokumento, papel, imahe, at transkripsyon. Ang mga resulta ay pinagsasama sa isang NetworkX graph, naka-cluster gamit ang Leiden, at ine-export bilang interactive na HTML, queryable na JSON, at audit report. +Gumagana ang graphify sa tatlong pass. Una, isang deterministikong AST pass ang nag-e-extract ng istruktura mula sa mga code file nang walang LLM. Pagkatapos, ang mga video at audio file ay tina-transcribe nang lokal gamit ang faster-whisper. Panghuli, mga Claude sub-agent ang tumatakbo nang magkakasabay sa mga dokumento, papel, imahe, at transkripsyon. Ang mga resulta ay pinagsasama sa isang Helix graph, naka-cluster gamit ang Leiden, at ine-export bilang interactive na HTML, queryable na JSON, at audit report. Ang bawat relasyon ay may label na `EXTRACTED`, `INFERRED` (may confidence score), o `AMBIGUOUS`. diff --git a/docs/translations/README.fr-FR.md b/docs/translations/README.fr-FR.md index eb57cd638..5ae61144b 100644 --- a/docs/translations/README.fr-FR.md +++ b/docs/translations/README.fr-FR.md @@ -28,7 +28,7 @@ Entièrement multimodal. Déposez du code, des PDFs, du markdown, des captures d graphify-out/ ├── graph.html graphe interactif — ouvrir dans un navigateur, cliquer, rechercher, filtrer ├── GRAPH_REPORT.md nœuds dieu, connexions surprenantes, questions suggérées -├── graph.json graphe persistant — interrogeable des semaines plus tard sans relire +├── graph.helix graphe persistant — interrogeable des semaines plus tard sans relire └── cache/ cache SHA256 — les réexécutions ne traitent que les fichiers modifiés ``` @@ -46,7 +46,7 @@ Même syntaxe que `.gitignore`. Un seul `.graphifyignore` à la racine du dépô ## Comment ça fonctionne -graphify s'exécute en trois passes. D'abord, un passage AST déterministe extrait la structure des fichiers de code (classes, fonctions, imports, graphes d'appel, docstrings, commentaires de justification) sans LLM. Ensuite, les fichiers vidéo et audio sont transcrits localement avec faster-whisper. Enfin, des sous-agents Claude s'exécutent en parallèle sur les docs, articles, images et transcriptions pour extraire concepts, relations et justifications de conception. Les résultats sont fusionnés dans un graphe NetworkX, regroupés avec la détection de communautés Leiden, et exportés en HTML interactif, JSON interrogeable et un rapport d'audit en langage naturel. +graphify s'exécute en trois passes. D'abord, un passage AST déterministe extrait la structure des fichiers de code (classes, fonctions, imports, graphes d'appel, docstrings, commentaires de justification) sans LLM. Ensuite, les fichiers vidéo et audio sont transcrits localement avec faster-whisper. Enfin, des sous-agents Claude s'exécutent en parallèle sur les docs, articles, images et transcriptions pour extraire concepts, relations et justifications de conception. Les résultats sont fusionnés dans un graphe Helix, regroupés avec la détection de communautés Leiden, et exportés en HTML interactif, JSON interrogeable et un rapport d'audit en langage naturel. **Le clustering est basé sur la topologie du graphe — pas d'embeddings.** Leiden trouve les communautés par densité d'arêtes. Les arêtes de similarité sémantique extraites par Claude (`semantically_similar_to`, marquées INFERRED) sont déjà dans le graphe. La structure du graphe est le signal de similarité — pas d'étape d'embedding séparée ni de base de données vectorielle nécessaire. @@ -157,7 +157,7 @@ graphify envoie le contenu des fichiers à l'API du modèle de votre assistant I ## Stack technique -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extraction sémantique via Claude, GPT-4 ou le modèle de votre plateforme. Transcription vidéo via faster-whisper + yt-dlp (optionnel). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extraction sémantique via Claude, GPT-4 ou le modèle de votre plateforme. Transcription vidéo via faster-whisper + yt-dlp (optionnel). ## Construit sur graphify — Penpax diff --git a/docs/translations/README.he-IL.md b/docs/translations/README.he-IL.md index 65e9ae75d..8fede1345 100644 --- a/docs/translations/README.he-IL.md +++ b/docs/translations/README.he-IL.md @@ -46,7 +46,7 @@ graphify-out/ ├── graph.html נפתח בכל דפדפן — לחיצה על צמתים, סינון, חיפוש ├── GRAPH_REPORT.md עיקרי הדברים: מושגי מפתח, קשרים מפתיעים, שאלות מוצעות -└── graph.json הגרף המלא — אפשר לשאול עליו בכל רגע בלי לקרוא שוב את הקבצים +└── graph.helix הגרף המלא — אפשר לשאול עליו בכל רגע בלי לקרוא שוב את הקבצים ```
@@ -391,7 +391,7 @@ graphify-out/cost.json # מקומי בלבד **תהליך העבודה:** 1. אחד מחברי הצוות מריץ `‎/graphify .` ומבצע commit ל-`graphify-out/`. 2. כולם מושכים — העוזר שלהם קורא את הגרף מיד. -3. הריצו `graphify hook install` לבנייה אוטומטית מחדש אחרי כל commit ‏(AST בלבד, ללא עלות API). זה גם מגדיר merge driver של git כך ש-`graph.json` לעולם לא יישאר עם סימוני קונפליקט — שני מפתחים שמבצעים commit במקביל מקבלים מיזוג-איחוד אוטומטי של הגרפים. +3. הריצו `graphify hook install` לבנייה אוטומטית מחדש אחרי כל commit ‏(AST בלבד, ללא עלות API). זה גם מגדיר merge driver של git כך ש-`graph.helix` לעולם לא יישאר עם סימוני קונפליקט — שני מפתחים שמבצעים commit במקביל מקבלים מיזוג-איחוד אוטומטי של הגרפים. 4. כשמסמכים או מאמרים משתנים, הריצו `‎/graphify --update` לרענון הצמתים הללו. --- @@ -403,18 +403,18 @@ graphify-out/cost.json # מקומי בלבד ```bash # שאילתת גרף מהטרמינל graphify query "הצג את זרימת האימות" -graphify query "מה מחבר בין DigestAuth ל-Response?" --graph graphify-out/graph.json +graphify query "מה מחבר בין DigestAuth ל-Response?" --graph graphify-out/graph.helix # חשיפת הגרף כשרת MCP (לגישת כלים חוזרת) -python -m graphify.serve graphify-out/graph.json -python -m graphify.serve --graph graphify-out/graph.json # גם הדגל --graph מתקבל +python -m graphify.serve graphify-out/graph.helix +python -m graphify.serve --graph graphify-out/graph.helix # גם הדגל --graph מתקבל # רישום ב-Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix # או הגשה על HTTP כך שכל הצוות מצביע על URL אחד (בלי graphify מקומי): -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
@@ -443,7 +443,7 @@ python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 ```bash docker build -t graphify . docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ - /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" + /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
@@ -495,7 +495,7 @@ python3 -m venv .venv && .venv/bin/pip install "graphifyy[mcp]" | `GRAPHIFY_QUERY_LOG` | דריסת נתיב יומן השאילתות (ברירת מחדל: `~/.cache/graphify-queries.log`) | אופציונלי — ערך ריק או `/dev/null` להשתקה | | `GRAPHIFY_QUERY_LOG_DISABLE` | הגדירו `1` לביטול מוחלט של יומן השאילתות | אופציונלי | | `GRAPHIFY_QUERY_LOG_RESPONSES` | הגדירו `1` לרישום גם של תשובות תת-גרף מלאות (כבוי כברירת מחדל) | אופציונלי | -| `GRAPHIFY_MAX_GRAPH_BYTES` | דריסת תקרת הגודל של graph.json ‏(512 MiB) — למשל `700MB`, ‏`2GB` או בייטים | אופציונלי — שימושי לקורפוסים גדולים מאוד | +| `GRAPHIFY_MAX_GRAPH_BYTES` | דריסת תקרת הגודל של graph.helix ‏(512 MiB) — למשל `700MB`, ‏`2GB` או בייטים | אופציונלי — שימושי לקורפוסים גדולים מאוד | | `GRAPHIFY_LLM_TEMPERATURE` | דריסת טמפרטורת ה-LLM לחילוץ סמנטי — למשל `0.7`, או `none` להשמטה | אופציונלי — מושמט אוטומטית למודלי היסק o1/o3/o4/gpt-5 | --- @@ -579,8 +579,8 @@ graphify query "..."
-**ל-`graph.json` יש סימוני קונפליקט אחרי ששני מפתחים ביצעו commit במקביל** -הריצו `graphify hook install` — הוא מגדיר merge driver של git שממזג-מאחד את `graph.json` אוטומטית כך שקונפליקטים לא קורים בכלל. +**ל-`graph.helix` יש סימוני קונפליקט אחרי ששני מפתחים ביצעו commit במקביל** +הריצו `graphify hook install` — הוא מגדיר merge driver של git שממזג-מאחד את `graph.helix` אוטומטית כך שקונפליקטים לא קורים בכלל. **החילוץ מחזיר צמתים/קשתות ריקים למסמכים או PDF** מסמכים, PDF ותמונות דורשים קריאת LLM — קורפוסים של קוד בלבד אינם דורשים מפתח. ודאו שמפתח ה-API מוגדר וה-backend נכון: @@ -644,7 +644,7 @@ graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome usefu graphify reflect # איחוד תוצאות graphify-out/memory/ אל reflections/LESSONS.md graphify reflect --if-stale # לא עושה דבר אם LESSONS.md כבר חדש מכל הקלטים (זול להרצה בכל סשן) graphify reflect --out docs/LESSONS.md # כתיבת מסמך הלקחים למקום אחר -graphify reflect --graph graphify-out/graph.json # קיבוץ לקחים לפי קהילה + כתיבת שכבת זיכרון העבודה (.graphify_learning.json) +graphify reflect --graph graphify-out/graph.helix # קיבוץ לקחים לפי קהילה + כתיבת שכבת זיכרון העבודה (.graphify_learning.json) # השכבה מתייגת צמתים preferred/tentative/contested (משוקלל-עדכניות, עם מקור); # graphify explain / query מציגים אז רמז "Lesson:", מסומן "code changed — re-verify" כשהמקור התקדם @@ -717,7 +717,7 @@ graphify extract ./docs --google-workspace # ייצוא .gdoc/.gsheet/.gslid graphify extract ./docs --mode deep # חילוץ סמנטי עשיר יותר עם system prompt מורחב graphify extract ./docs --no-cluster # חילוץ גולמי בלבד, דילוג על אשכול graphify extract ./docs --timing # הדפסת זמני ריצה פר-שלב ל-stderr ‏(עובד גם ב-cluster-only) -graphify extract ./docs --force # דריסת graph.json גם אם לגרף החדש פחות צמתים (אחרי refactors או לניקוי כפילויות רפאים) +graphify extract ./docs --force # דריסת graph.helix גם אם לגרף החדש פחות צמתים (אחרי refactors או לניקוי כפילויות רפאים) graphify extract ./docs --dedup-llm # ‏LLM כמכריע לזוגות ישויות עמומים (משתמש באותו מפתח API) graphify extract ./docs --global --as myrepo # חילוץ ורישום בגרף הגלובלי חוצה-הפרויקטים GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # הרמת תקרת הפלט לקורפוסים צפופים @@ -727,7 +727,7 @@ graphify export callflow-html --max-sections 8 # הגבלת מספר קטע graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json --as myrepo # רישום גרף פרויקט אל ~/.graphify/global-graph.json +graphify global add graphify-out/graph.helix --as myrepo # רישום גרף פרויקט אל ~/.graphify/global-graph.helix graphify global remove myrepo # הסרת פרויקט מהגרף הגלובלי graphify global list # הצגת כל המאגרים הרשומים + ספירות צמתים/קשתות graphify global path # הדפסת הנתיב לקובץ הגרף הגלובלי @@ -750,7 +750,7 @@ graphify update ./src graphify update ./src --no-cluster # דילוג על אשכול מחדש, כתיבת גרף AST גולמי בלבד graphify update ./src --force # דריסה גם אם לגרף החדש פחות צמתים graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # מיקום גרף מותאם +graphify cluster-only ./my-project --graph path/to/graph.helix # מיקום גרף מותאם graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # תיוג קהילות מקבילי (גרפים גדולים) graphify cluster-only ./my-project --resolution 1.5 # יותר קהילות, קטנות יותר graphify cluster-only ./my-project --exclude-hubs 99 # החרגת צומתי p99 מהחלוקה diff --git a/docs/translations/README.hi-IN.md b/docs/translations/README.hi-IN.md index 181963c29..b68261174 100644 --- a/docs/translations/README.hi-IN.md +++ b/docs/translations/README.hi-IN.md @@ -28,7 +28,7 @@ graphify-out/ ├── graph.html इंटरेक्टिव ग्राफ — किसी भी ब्राउज़र में खोलें, नोड्स क्लिक करें, खोजें ├── GRAPH_REPORT.md गॉड नोड्स, आश्चर्यजनक कनेक्शन, सुझाए गए प्रश्न -├── graph.json स्थायी ग्राफ — हफ्तों बाद भी क्वेरी करें +├── graph.helix स्थायी ग्राफ — हफ्तों बाद भी क्वेरी करें └── cache/ SHA256 कैश — पुनः चलाने पर केवल बदली हुई फ़ाइलें प्रोसेस होती हैं ``` @@ -46,7 +46,7 @@ dist/ ## यह कैसे काम करता है -graphify तीन चरणों में चलता है। पहले, एक निर्धारक AST पास कोड फ़ाइलों से संरचना निकालता है — बिना किसी LLM के। दूसरे, वीडियो और ऑडियो फ़ाइलों को faster-whisper से स्थानीय रूप से ट्रांसक्राइब किया जाता है। तीसरे, Claude सबएजेंट दस्तावेज़ों, papers, छवियों और ट्रांसक्रिप्ट पर समानांतर में चलते हैं। परिणामों को NetworkX ग्राफ में मर्ज किया जाता है, Leiden कम्युनिटी डिटेक्शन से क्लस्टर किया जाता है, और इंटरेक्टिव HTML, क्वेरी करने योग्य JSON और एक ऑडिट रिपोर्ट के रूप में निर्यात किया जाता है। +graphify तीन चरणों में चलता है। पहले, एक निर्धारक AST पास कोड फ़ाइलों से संरचना निकालता है — बिना किसी LLM के। दूसरे, वीडियो और ऑडियो फ़ाइलों को faster-whisper से स्थानीय रूप से ट्रांसक्राइब किया जाता है। तीसरे, Claude सबएजेंट दस्तावेज़ों, papers, छवियों और ट्रांसक्रिप्ट पर समानांतर में चलते हैं। परिणामों को Helix ग्राफ में मर्ज किया जाता है, Leiden कम्युनिटी डिटेक्शन से क्लस्टर किया जाता है, और इंटरेक्टिव HTML, क्वेरी करने योग्य JSON और एक ऑडिट रिपोर्ट के रूप में निर्यात किया जाता है। **क्लस्टरिंग ग्राफ-टोपोलॉजी आधारित है — कोई embeddings नहीं।** Claude द्वारा निकाले गए सिमेंटिक समानता किनारे पहले से ग्राफ में हैं, इसलिए वे कम्युनिटी डिटेक्शन को सीधे प्रभावित करते हैं। diff --git a/docs/translations/README.hu-HU.md b/docs/translations/README.hu-HU.md index 14c24c1cb..0b241b38d 100644 --- a/docs/translations/README.hu-HU.md +++ b/docs/translations/README.hu-HU.md @@ -27,13 +27,13 @@ Teljesen multimodális. Adjon hozzá kódot, PDF-eket, markdownt, képernyőkép graphify-out/ ├── graph.html interaktív gráf — nyissa meg bármely böngészőben ├── GRAPH_REPORT.md isten-csúcspontok, meglepő kapcsolatok, javasolt kérdések -├── graph.json állandó gráf — hetekkel később is lekérdezhető +├── graph.helix állandó gráf — hetekkel később is lekérdezhető └── cache/ SHA256-gyorsítótár — ismételt futtatások csak a módosított fájlokat dolgozzák fel ``` ## Hogyan működik -A graphify három menetben dolgozik. Először egy determinisztikus AST-menet kinyeri a struktúrát a kódfájlokból LLM nélkül. Ezután a video- és hangfájlokat a faster-whisper segítségével helyben írja át. Végül a Claude alügynökök párhuzamosan futnak dokumentumokon, cikkeken, képeken és átiratokban. Az eredményeket egy NetworkX-gráfba olvasztja össze, Leiden-nel klaszterezik, és interaktív HTML-ként, lekérdezhető JSON-ként és auditjelentésként exportálja. +A graphify három menetben dolgozik. Először egy determinisztikus AST-menet kinyeri a struktúrát a kódfájlokból LLM nélkül. Ezután a video- és hangfájlokat a faster-whisper segítségével helyben írja át. Végül a Claude alügynökök párhuzamosan futnak dokumentumokon, cikkeken, képeken és átiratokban. Az eredményeket egy Helix-gráfba olvasztja össze, Leiden-nel klaszterezik, és interaktív HTML-ként, lekérdezhető JSON-ként és auditjelentésként exportálja. Minden kapcsolat `EXTRACTED`, `INFERRED` (megbízhatósági pontszámmal) vagy `AMBIGUOUS` feliratot kap. diff --git a/docs/translations/README.id-ID.md b/docs/translations/README.id-ID.md index 86691d5fa..25660eccc 100644 --- a/docs/translations/README.id-ID.md +++ b/docs/translations/README.id-ID.md @@ -27,13 +27,13 @@ Sepenuhnya multimodal. Tambahkan kode, PDF, markdown, tangkapan layar, diagram, graphify-out/ ├── graph.html graf interaktif — buka di browser mana saja ├── GRAPH_REPORT.md node dewa, koneksi mengejutkan, pertanyaan yang disarankan -├── graph.json graf persisten — dapat dikueri berminggu-minggu kemudian +├── graph.helix graf persisten — dapat dikueri berminggu-minggu kemudian └── cache/ cache SHA256 — pengulangan hanya memproses file yang berubah ``` ## Cara Kerja -graphify bekerja dalam tiga tahap. Pertama, tahap AST deterministik mengekstrak struktur dari file kode tanpa LLM. Kemudian file video dan audio ditranskrip secara lokal dengan faster-whisper. Terakhir, sub-agen Claude berjalan secara paralel pada dokumen, makalah, gambar, dan transkripsi. Hasilnya digabungkan ke dalam graf NetworkX, dikelompokkan dengan Leiden, dan diekspor sebagai HTML interaktif, JSON yang dapat dikueri, dan laporan audit. +graphify bekerja dalam tiga tahap. Pertama, tahap AST deterministik mengekstrak struktur dari file kode tanpa LLM. Kemudian file video dan audio ditranskrip secara lokal dengan faster-whisper. Terakhir, sub-agen Claude berjalan secara paralel pada dokumen, makalah, gambar, dan transkripsi. Hasilnya digabungkan ke dalam graf Helix, dikelompokkan dengan Leiden, dan diekspor sebagai HTML interaktif, JSON yang dapat dikueri, dan laporan audit. Setiap hubungan diberi label `EXTRACTED`, `INFERRED` (dengan skor kepercayaan), atau `AMBIGUOUS`. diff --git a/docs/translations/README.it-IT.md b/docs/translations/README.it-IT.md index 9cb1ccb91..1f789ffab 100644 --- a/docs/translations/README.it-IT.md +++ b/docs/translations/README.it-IT.md @@ -27,13 +27,13 @@ Completamente multimodale. Aggiungi codice, PDF, markdown, screenshot, diagrammi graphify-out/ ├── graph.html grafo interattivo — apri in qualsiasi browser ├── GRAPH_REPORT.md nodi dio, connessioni sorprendenti, domande suggerite -├── graph.json grafo persistente — interrogabile settimane dopo +├── graph.helix grafo persistente — interrogabile settimane dopo └── cache/ cache SHA256 — le riesecuzioni elaborano solo i file modificati ``` ## Come funziona -graphify esegue in tre passaggi. Prima, un passaggio AST deterministico estrae la struttura dai file di codice senza LLM. Poi, i file video e audio vengono trascritti localmente con faster-whisper. Infine, i subagenti Claude eseguono in parallelo su documenti, paper, immagini e trascrizioni. I risultati vengono uniti in un grafo NetworkX, raggruppati con Leiden e esportati come HTML interattivo, JSON interrogabile e report di audit. +graphify esegue in tre passaggi. Prima, un passaggio AST deterministico estrae la struttura dai file di codice senza LLM. Poi, i file video e audio vengono trascritti localmente con faster-whisper. Infine, i subagenti Claude eseguono in parallelo su documenti, paper, immagini e trascrizioni. I risultati vengono uniti in un grafo Helix, raggruppati con Leiden e esportati come HTML interattivo, JSON interrogabile e report di audit. Ogni relazione è etichettata `EXTRACTED`, `INFERRED` (con punteggio di confidenza) o `AMBIGUOUS`. diff --git a/docs/translations/README.ja-JP.md b/docs/translations/README.ja-JP.md index 467ff6839..332aadb9c 100644 --- a/docs/translations/README.ja-JP.md +++ b/docs/translations/README.ja-JP.md @@ -20,7 +20,7 @@ graphify-out/ ├── graph.html インタラクティブなグラフ - ノードをクリック、検索、コミュニティでフィルタ ├── GRAPH_REPORT.md ゴッドノード、意外なつながり、推奨される質問 -├── graph.json 永続化されたグラフ - 数週間後でも再読み込みなしでクエリ可能 +├── graph.helix 永続化されたグラフ - 数週間後でも再読み込みなしでクエリ可能 └── cache/ SHA256 キャッシュ - 再実行時は変更されたファイルのみ処理 ``` @@ -38,7 +38,7 @@ dist/ ## 仕組み -graphify は 2 パスで動作します。まず、決定論的な AST パスがコードファイルから構造(クラス、関数、インポート、コールグラフ、docstring、根拠コメント)を LLM なしで抽出します。次に、Claude サブエージェントがドキュメント、論文、画像に対して並列に実行され、概念、関係性、設計の根拠を抽出します。結果は NetworkX グラフにマージされ、Leiden コミュニティ検出でクラスタリングされ、インタラクティブ HTML、クエリ可能な JSON、平易な言葉の監査レポートとしてエクスポートされます。 +graphify は 2 パスで動作します。まず、決定論的な AST パスがコードファイルから構造(クラス、関数、インポート、コールグラフ、docstring、根拠コメント)を LLM なしで抽出します。次に、Claude サブエージェントがドキュメント、論文、画像に対して並列に実行され、概念、関係性、設計の根拠を抽出します。結果は Helix グラフにマージされ、Leiden コミュニティ検出でクラスタリングされ、インタラクティブ HTML、クエリ可能な JSON、平易な言葉の監査レポートとしてエクスポートされます。 **クラスタリングはグラフトポロジベース――埋め込みは使いません。** Leiden はエッジ密度によってコミュニティを見つけます。Claude が抽出する意味的類似性エッジ(`semantically_similar_to`、INFERRED とマーク)は既にグラフに含まれているため、コミュニティ検出に直接影響します。グラフ構造そのものが類似性シグナルであり――別途の埋め込みステップやベクターデータベースは不要です。 @@ -97,7 +97,7 @@ Codex ユーザーは並列抽出のために `~/.codex/config.toml` の `[featu 常時有効のフックは `GRAPH_REPORT.md` を表面化します――これはゴッドノード、コミュニティ、意外なつながりを 1 ページにまとめた要約です。アシスタントはファイル検索の前にこれを読み、キーワードマッチではなく構造に基づいてナビゲートします。これで日常的な質問のほとんどをカバーできます。 -`/graphify query`、`/graphify path`、`/graphify explain` はさらに深く踏み込みます:生の `graph.json` をホップごとに辿り、ノード間の正確なパスをトレースし、エッジレベルの詳細(関係タイプ、信頼度スコア、ソース位置)を表面化します。一般的なオリエンテーションではなく、特定の質問をグラフから答えさせたいときに使います。 +`/graphify query`、`/graphify path`、`/graphify explain` はさらに深く踏み込みます:生の `graph.helix` をホップごとに辿り、ノード間の正確なパスをトレースし、エッジレベルの詳細(関係タイプ、信頼度スコア、ソース位置)を表面化します。一般的なオリエンテーションではなく、特定の質問をグラフから答えさせたいときに使います。 こう考えてください:常時有効のフックはアシスタントに地図を与え、`/graphify` コマンドはその地図を正確にナビゲートさせます。 @@ -167,7 +167,7 @@ graphify droid install # AGENTS.md(Factory Droid) graphify query "アテンションとオプティマイザを結ぶものは?" graphify query "認証フローを表示" --dfs graphify query "CfgNode とは?" --budget 500 -graphify query "..." --graph path/to/graph.json +graphify query "..." --graph path/to/graph.helix ``` あらゆるファイルタイプの組み合わせで動作します: @@ -212,7 +212,7 @@ graphify query "..." --graph path/to/graph.json | graphify ソース + Transformer 論文 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx(合成 Python ライブラリ) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -トークン削減はコーパスサイズに応じてスケールします。6 ファイルはいずれにせよコンテキストウィンドウに収まるため、そこでのグラフの価値は圧縮ではなく構造的明瞭さです。52 ファイル(コード + 論文 + 画像)では 71 倍以上が得られます。各 `worked/` フォルダには生の入力ファイルと実際の出力(`GRAPH_REPORT.md`、`graph.json`)があり、自分で実行して数字を検証できます。 +トークン削減はコーパスサイズに応じてスケールします。6 ファイルはいずれにせよコンテキストウィンドウに収まるため、そこでのグラフの価値は圧縮ではなく構造的明瞭さです。52 ファイル(コード + 論文 + 画像)では 71 倍以上が得られます。各 `worked/` フォルダには生の入力ファイルと実際の出力(`GRAPH_REPORT.md`、`graph.helix`)があり、自分で実行して数字を検証できます。 ## プライバシー @@ -220,7 +220,7 @@ graphify はドキュメント、論文、画像の意味的抽出のために ## 技術スタック -NetworkX + Leiden(graspologic) + tree-sitter + vis.js。意味的抽出は Claude(Claude Code)、GPT-4(Codex)、またはプラットフォームが実行するモデルを介して行われます。Neo4j は不要、サーバーも不要、完全にローカルで実行されます。 +Helix + Leiden(native Leiden) + tree-sitter + vis.js。意味的抽出は Claude(Claude Code)、GPT-4(Codex)、またはプラットフォームが実行するモデルを介して行われます。Neo4j は不要、サーバーも不要、完全にローカルで実行されます。 ## スター履歴 diff --git a/docs/translations/README.ko-KR.md b/docs/translations/README.ko-KR.md index 0fa46a211..b9b7d2e4f 100644 --- a/docs/translations/README.ko-KR.md +++ b/docs/translations/README.ko-KR.md @@ -20,7 +20,7 @@ graphify-out/ ├── graph.html 인터랙티브 그래프 - 노드 클릭, 검색, 커뮤니티별 필터 ├── GRAPH_REPORT.md 갓 노드, 의외의 연결, 추천 질문 -├── graph.json 영속 그래프 - 몇 주 후에도 재읽기 없이 쿼리 가능 +├── graph.helix 영속 그래프 - 몇 주 후에도 재읽기 없이 쿼리 가능 └── cache/ SHA256 캐시 - 재실행 시 변경된 파일만 처리 ``` @@ -38,7 +38,7 @@ dist/ ## 동작 원리 -graphify는 두 번의 패스로 실행됩니다. 첫 번째는 결정론적 AST 패스로, 코드 파일에서 구조(클래스, 함수, 임포트, 콜 그래프, docstring, 근거 주석)를 LLM 없이 추출합니다. 두 번째는 Claude 서브에이전트가 문서, 논문, 이미지에 대해 병렬로 실행되어 개념, 관계, 설계 근거를 추출합니다. 결과는 NetworkX 그래프로 병합되고, Leiden 커뮤니티 탐지로 클러스터링되며, 인터랙티브 HTML, 쿼리 가능한 JSON, 그리고 일반 언어 감사 보고서로 내보내집니다. +graphify는 두 번의 패스로 실행됩니다. 첫 번째는 결정론적 AST 패스로, 코드 파일에서 구조(클래스, 함수, 임포트, 콜 그래프, docstring, 근거 주석)를 LLM 없이 추출합니다. 두 번째는 Claude 서브에이전트가 문서, 논문, 이미지에 대해 병렬로 실행되어 개념, 관계, 설계 근거를 추출합니다. 결과는 Helix 그래프로 병합되고, Leiden 커뮤니티 탐지로 클러스터링되며, 인터랙티브 HTML, 쿼리 가능한 JSON, 그리고 일반 언어 감사 보고서로 내보내집니다. **클러스터링은 그래프 토폴로지 기반 — 임베딩을 사용하지 않습니다.** Leiden은 엣지 밀도를 기반으로 커뮤니티를 찾습니다. Claude가 추출하는 의미적 유사성 엣지(`semantically_similar_to`, INFERRED로 표시)는 이미 그래프에 포함되어 있으므로 커뮤니티 탐지에 직접 영향을 줍니다. 그래프 구조 자체가 유사성 신호이며 — 별도의 임베딩 단계나 벡터 데이터베이스가 필요하지 않습니다. @@ -103,13 +103,13 @@ Codex 사용자는 병렬 추출을 위해 `~/.codex/config.toml`의 `[features] 상시 작동 훅은 `GRAPH_REPORT.md`를 노출합니다 — 갓 노드, 커뮤니티, 의외의 연결을 한 페이지로 요약한 것입니다. 어시스턴트는 파일 검색 전에 이것을 읽으므로 키워드 매칭이 아닌 구조 기반으로 탐색합니다. 이것만으로 대부분의 일상적인 질문을 처리할 수 있습니다. -`/graphify query`, `/graphify path`, `/graphify explain`은 더 깊이 들어갑니다: 원시 `graph.json`을 홉 단위로 순회하고, 노드 간의 정확한 경로를 추적하며, 엣지 수준의 세부 정보(관계 유형, 신뢰도 점수, 소스 위치)를 보여줍니다. 일반적인 오리엔테이션이 아닌 그래프에서 특정 질문에 답하고 싶을 때 사용하세요. +`/graphify query`, `/graphify path`, `/graphify explain`은 더 깊이 들어갑니다: 원시 `graph.helix`을 홉 단위로 순회하고, 노드 간의 정확한 경로를 추적하며, 엣지 수준의 세부 정보(관계 유형, 신뢰도 점수, 소스 위치)를 보여줍니다. 일반적인 오리엔테이션이 아닌 그래프에서 특정 질문에 답하고 싶을 때 사용하세요. 이렇게 생각하면 됩니다: 상시 작동 훅은 어시스턴트에게 지도를 주고, `/graphify` 명령은 그 지도를 정확하게 탐색하게 합니다. -## `graph.json`을 LLM과 함께 사용하기 +## `graph.helix`을 LLM과 함께 사용하기 -`graph.json`은 프롬프트에 한 번에 전부 붙여넣기 위한 것이 아닙니다. 유용한 워크플로우는 다음과 같습니다: +`graph.helix`은 프롬프트에 한 번에 전부 붙여넣기 위한 것이 아닙니다. 유용한 워크플로우는 다음과 같습니다: 1. `graphify-out/GRAPH_REPORT.md`로 높은 수준의 개요를 파악합니다. 2. `graphify query`를 사용하여 답하려는 특정 질문에 대한 더 작은 서브그래프를 가져옵니다. @@ -118,8 +118,8 @@ Codex 사용자는 병렬 추출을 위해 `~/.codex/config.toml`의 `[features] 예를 들어, 프로젝트에서 graphify를 실행한 후: ```bash -graphify query "show the auth flow" --graph graphify-out/graph.json -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +graphify query "show the auth flow" --graph graphify-out/graph.helix +graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.helix ``` 출력에는 노드 레이블, 엣지 유형, 신뢰도 태그, 소스 파일, 소스 위치가 포함됩니다. 이는 LLM을 위한 좋은 중간 컨텍스트 블록이 됩니다: @@ -129,10 +129,10 @@ graphify query "what connects DigestAuth to Response?" --graph graphify-out/grap 가능한 경우 소스 파일을 인용하세요. ``` -어시스턴트가 도구 호출이나 MCP를 지원하는 경우, 텍스트를 붙여넣는 대신 그래프를 직접 사용하세요. graphify는 `graph.json`을 MCP 서버로 노출할 수 있습니다: +어시스턴트가 도구 호출이나 MCP를 지원하는 경우, 텍스트를 붙여넣는 대신 그래프를 직접 사용하세요. graphify는 `graph.helix`을 MCP 서버로 노출할 수 있습니다: ```bash -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix ``` 이를 통해 어시스턴트가 `query_graph`, `get_node`, `get_neighbors`, `shortest_path` 같은 반복 쿼리에 구조화된 그래프 접근을 할 수 있습니다. @@ -207,7 +207,7 @@ graphify trae-cn uninstall graphify query "어텐션과 옵티마이저를 연결하는 것은?" graphify query "인증 흐름 보기" --dfs graphify query "CfgNode이 뭐지?" --budget 500 -graphify query "..." --graph path/to/graph.json +graphify query "..." --graph path/to/graph.helix ``` 다양한 파일 유형의 조합과 함께 동작합니다: @@ -252,7 +252,7 @@ graphify query "..." --graph path/to/graph.json | graphify 소스 + Transformer 논문 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx (합성 Python 라이브러리) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -토큰 축소는 코퍼스 크기에 비례하여 확장됩니다. 6개 파일은 어차피 컨텍스트 윈도우에 들어가므로, 그래프의 가치는 압축이 아닌 구조적 명확성에 있습니다. 52개 파일(코드 + 논문 + 이미지)에서는 71배 이상을 달성합니다. 각 `worked/` 폴더에는 원본 입력 파일과 실제 출력(`GRAPH_REPORT.md`, `graph.json`)이 있어 직접 실행하여 수치를 검증할 수 있습니다. +토큰 축소는 코퍼스 크기에 비례하여 확장됩니다. 6개 파일은 어차피 컨텍스트 윈도우에 들어가므로, 그래프의 가치는 압축이 아닌 구조적 명확성에 있습니다. 52개 파일(코드 + 논문 + 이미지)에서는 71배 이상을 달성합니다. 각 `worked/` 폴더에는 원본 입력 파일과 실제 출력(`GRAPH_REPORT.md`, `graph.helix`)이 있어 직접 실행하여 수치를 검증할 수 있습니다. ## 개인정보 보호 @@ -260,7 +260,7 @@ graphify는 문서, 논문, 이미지의 의미적 추출을 위해 파일 내 ## 기술 스택 -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. 의미적 추출은 Claude(Claude Code), GPT-4(Codex), 또는 플랫폼이 실행하는 모델을 통해 수행됩니다. Neo4j 불필요, 서버 불필요, 완전히 로컬에서 실행됩니다. +Helix + Leiden (native Leiden) + tree-sitter + vis.js. 의미적 추출은 Claude(Claude Code), GPT-4(Codex), 또는 플랫폼이 실행하는 모델을 통해 수행됩니다. Neo4j 불필요, 서버 불필요, 완전히 로컬에서 실행됩니다. ## 다음 계획 diff --git a/docs/translations/README.nl-NL.md b/docs/translations/README.nl-NL.md index 2dd8f834f..2dc675067 100644 --- a/docs/translations/README.nl-NL.md +++ b/docs/translations/README.nl-NL.md @@ -27,13 +27,13 @@ Volledig multimodaal. Voeg code, PDF's, markdown, schermafbeeldingen, diagrammen graphify-out/ ├── graph.html interactieve graaf — open in elke browser ├── GRAPH_REPORT.md godknooppunten, verrassende verbindingen, voorgestelde vragen -├── graph.json persistente graaf — weken later opvraagbaar +├── graph.helix persistente graaf — weken later opvraagbaar └── cache/ SHA256-cache — herhaalde runs verwerken alleen gewijzigde bestanden ``` ## Hoe het werkt -graphify werkt in drie passes. Eerst extraheert een deterministische AST-pass structuur uit codebestanden zonder LLM. Vervolgens worden video- en audiobestanden lokaal getranscribeerd met faster-whisper. Ten slotte werken Claude-subagenten parallel over documenten, papers, afbeeldingen en transcripties. De resultaten worden samengevoegd in een NetworkX-graaf, geclusterd met Leiden en geëxporteerd als interactieve HTML, opvraagbare JSON en een auditrapport. +graphify werkt in drie passes. Eerst extraheert een deterministische AST-pass structuur uit codebestanden zonder LLM. Vervolgens worden video- en audiobestanden lokaal getranscribeerd met faster-whisper. Ten slotte werken Claude-subagenten parallel over documenten, papers, afbeeldingen en transcripties. De resultaten worden samengevoegd in een Helix-graaf, geclusterd met Leiden en geëxporteerd als interactieve HTML, opvraagbare JSON en een auditrapport. Elke relatie is gelabeld als `EXTRACTED`, `INFERRED` (met betrouwbaarheidsscore) of `AMBIGUOUS`. diff --git a/docs/translations/README.no-NO.md b/docs/translations/README.no-NO.md index 5bfcd59be..589a3ad2b 100644 --- a/docs/translations/README.no-NO.md +++ b/docs/translations/README.no-NO.md @@ -27,13 +27,13 @@ Fullt multimodal. Legg til kode, PDF-er, markdown, skjermbilder, diagrammer, whi graphify-out/ ├── graph.html interaktiv graf — åpne i en hvilken som helst nettleser ├── GRAPH_REPORT.md gudnoder, overraskende forbindelser, foreslåtte spørsmål -├── graph.json vedvarende graf — forespørselbar uker senere +├── graph.helix vedvarende graf — forespørselbar uker senere └── cache/ SHA256-cache — gjentatte kjøringer behandler bare endrede filer ``` ## Hvordan det fungerer -graphify arbeider i tre gjennomganger. Først ekstraherer et deterministisk AST-gjennomgang struktur fra kodefiler uten LLM. Deretter transkriberes video- og lydfiler lokalt med faster-whisper. Til slutt kjører Claude-underagenter parallelt på dokumenter, artikler, bilder og transkripsjoner. Resultatene slås sammen i en NetworkX-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørselbar JSON og revisjonsrapport. +graphify arbeider i tre gjennomganger. Først ekstraherer et deterministisk AST-gjennomgang struktur fra kodefiler uten LLM. Deretter transkriberes video- og lydfiler lokalt med faster-whisper. Til slutt kjører Claude-underagenter parallelt på dokumenter, artikler, bilder og transkripsjoner. Resultatene slås sammen i en Helix-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørselbar JSON og revisjonsrapport. Hver relasjon er merket `EXTRACTED`, `INFERRED` (med konfidenspoeng) eller `AMBIGUOUS`. diff --git a/docs/translations/README.pl-PL.md b/docs/translations/README.pl-PL.md index 555dd4c03..0bbc47c3a 100644 --- a/docs/translations/README.pl-PL.md +++ b/docs/translations/README.pl-PL.md @@ -27,13 +27,13 @@ W pełni multimodalny. Dodaj kod, PDF, markdown, zrzuty ekranu, diagramy, zdjęc graphify-out/ ├── graph.html interaktywny graf — otwórz w dowolnej przeglądarce ├── GRAPH_REPORT.md węzły boga, zaskakujące połączenia, sugerowane pytania -├── graph.json trwały graf — zapytaj tygodnie później +├── graph.helix trwały graf — zapytaj tygodnie później └── cache/ cache SHA256 — ponowne uruchomienia przetwarzają tylko zmienione pliki ``` ## Jak to działa -graphify działa w trzech przebiegach. Najpierw deterministyczny przebieg AST wyodrębnia strukturę z plików kodu bez LLM. Następnie pliki wideo i audio są transkrybowane lokalnie za pomocą faster-whisper. Na koniec subagenci Claude działają równolegle na dokumentach, artykułach, obrazach i transkrypcjach. Wyniki są łączone w graf NetworkX, grupowane za pomocą Leiden i eksportowane jako interaktywny HTML, JSON i raport audytu. +graphify działa w trzech przebiegach. Najpierw deterministyczny przebieg AST wyodrębnia strukturę z plików kodu bez LLM. Następnie pliki wideo i audio są transkrybowane lokalnie za pomocą faster-whisper. Na koniec subagenci Claude działają równolegle na dokumentach, artykułach, obrazach i transkrypcjach. Wyniki są łączone w graf Helix, grupowane za pomocą Leiden i eksportowane jako interaktywny HTML, JSON i raport audytu. Każda relacja jest oznaczona `EXTRACTED`, `INFERRED` (z wynikiem pewności) lub `AMBIGUOUS`. diff --git a/docs/translations/README.pt-BR.md b/docs/translations/README.pt-BR.md index 37e7d3f6e..2364b755b 100644 --- a/docs/translations/README.pt-BR.md +++ b/docs/translations/README.pt-BR.md @@ -28,7 +28,7 @@ Totalmente multimodal. Adicione código, PDFs, markdown, capturas de tela, diagr graphify-out/ ├── graph.html grafo interativo — abrir em qualquer navegador, clicar em nós, pesquisar ├── GRAPH_REPORT.md nós deus, conexões surpreendentes, perguntas sugeridas -├── graph.json grafo persistente — consultar semanas depois sem reler +├── graph.helix grafo persistente — consultar semanas depois sem reler └── cache/ cache SHA256 — re-execuções processam apenas arquivos modificados ``` @@ -46,7 +46,7 @@ Mesma sintaxe do `.gitignore`. ## Como funciona -graphify executa em três passes. Primeiro, uma passagem AST determinística extrai estrutura de arquivos de código (classes, funções, importações, grafos de chamadas, docstrings, comentários de justificativa) sem LLM. Segundo, arquivos de vídeo e áudio são transcritos localmente com faster-whisper. Terceiro, subagentes Claude executam em paralelo sobre documentos, papers, imagens e transcrições para extrair conceitos, relações e justificativas de design. Os resultados são mesclados em um grafo NetworkX, agrupados com detecção de comunidades Leiden, e exportados como HTML interativo, JSON consultável e um relatório de auditoria em linguagem natural. +graphify executa em três passes. Primeiro, uma passagem AST determinística extrai estrutura de arquivos de código (classes, funções, importações, grafos de chamadas, docstrings, comentários de justificativa) sem LLM. Segundo, arquivos de vídeo e áudio são transcritos localmente com faster-whisper. Terceiro, subagentes Claude executam em paralelo sobre documentos, papers, imagens e transcrições para extrair conceitos, relações e justificativas de design. Os resultados são mesclados em um grafo Helix, agrupados com detecção de comunidades Leiden, e exportados como HTML interativo, JSON consultável e um relatório de auditoria em linguagem natural. **O clustering é baseado em topologia de grafo — sem embeddings.** Leiden encontra comunidades por densidade de arestas. As arestas de similaridade semântica que Claude extrai (`semantically_similar_to`, marcadas INFERRED) já estão no grafo. A estrutura do grafo é o sinal de similaridade — nenhum passo de embedding separado ou banco de dados vetorial é necessário. @@ -157,7 +157,7 @@ graphify envia conteúdo de arquivos para a API do modelo do seu assistente IA p ## Stack técnico -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extração semântica via Claude, GPT-4 ou o modelo da sua plataforma. Transcrição de vídeo via faster-whisper + yt-dlp (opcional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extração semântica via Claude, GPT-4 ou o modelo da sua plataforma. Transcrição de vídeo via faster-whisper + yt-dlp (opcional). ## Construído sobre graphify — Penpax diff --git a/docs/translations/README.ro-RO.md b/docs/translations/README.ro-RO.md index 159354118..8faa94f0d 100644 --- a/docs/translations/README.ro-RO.md +++ b/docs/translations/README.ro-RO.md @@ -27,13 +27,13 @@ Complet multimodal. Adăugați cod, PDF-uri, markdown, capturi de ecran, diagram graphify-out/ ├── graph.html graf interactiv — deschideți în orice browser ├── GRAPH_REPORT.md noduri-zeu, conexiuni surprinzătoare, întrebări sugerate -├── graph.json graf persistent — interogabil săptămâni mai târziu +├── graph.helix graf persistent — interogabil săptămâni mai târziu └── cache/ cache SHA256 — rulările repetate procesează doar fișierele modificate ``` ## Cum funcționează -graphify lucrează în trei treceri. Mai întâi, o trecere AST deterministă extrage structura din fișierele de cod fără LLM. Apoi fișierele video și audio sunt transcrise local cu faster-whisper. În final, sub-agenții Claude rulează în paralel pe documente, lucrări, imagini și transcrieri. Rezultatele sunt îmbinate într-un graf NetworkX, grupate cu Leiden și exportate ca HTML interactiv, JSON interogabil și raport de audit. +graphify lucrează în trei treceri. Mai întâi, o trecere AST deterministă extrage structura din fișierele de cod fără LLM. Apoi fișierele video și audio sunt transcrise local cu faster-whisper. În final, sub-agenții Claude rulează în paralel pe documente, lucrări, imagini și transcrieri. Rezultatele sunt îmbinate într-un graf Helix, grupate cu Leiden și exportate ca HTML interactiv, JSON interogabil și raport de audit. Fiecare relație este etichetată `EXTRACTED`, `INFERRED` (cu scor de încredere) sau `AMBIGUOUS`. diff --git a/docs/translations/README.ru-RU.md b/docs/translations/README.ru-RU.md index 7518aea41..dd0d0abcf 100644 --- a/docs/translations/README.ru-RU.md +++ b/docs/translations/README.ru-RU.md @@ -28,7 +28,7 @@ graphify-out/ ├── graph.html интерактивный граф — открыть в браузере, кликать по узлам, искать, фильтровать ├── GRAPH_REPORT.md бог-узлы, неожиданные связи, предлагаемые вопросы -├── graph.json постоянный граф — запрашивать через недели без повторного чтения +├── graph.helix постоянный граф — запрашивать через недели без повторного чтения └── cache/ SHA256-кэш — повторные запуски обрабатывают только изменённые файлы ``` @@ -46,7 +46,7 @@ dist/ ## Как это работает -graphify работает в три прохода. Сначала детерминированный AST-проход извлекает структуру из файлов кода (классы, функции, импорты, графы вызовов, docstrings, комментарии с обоснованием) — без LLM. Затем видео и аудиофайлы транскрибируются локально с faster-whisper. Наконец, Claude-субагенты запускаются параллельно над документами, статьями, изображениями и транскриптами для извлечения концепций, связей и обоснований дизайна. Результаты объединяются в граф NetworkX, кластеризуются с помощью Leiden-детекции сообществ и экспортируются как интерактивный HTML, запрашиваемый JSON и аудит-отчёт на естественном языке. +graphify работает в три прохода. Сначала детерминированный AST-проход извлекает структуру из файлов кода (классы, функции, импорты, графы вызовов, docstrings, комментарии с обоснованием) — без LLM. Затем видео и аудиофайлы транскрибируются локально с faster-whisper. Наконец, Claude-субагенты запускаются параллельно над документами, статьями, изображениями и транскриптами для извлечения концепций, связей и обоснований дизайна. Результаты объединяются в граф Helix, кластеризуются с помощью Leiden-детекции сообществ и экспортируются как интерактивный HTML, запрашиваемый JSON и аудит-отчёт на естественном языке. **Кластеризация основана на топологии графа — без эмбеддингов.** Leiden находит сообщества по плотности рёбер. Рёбра семантического сходства, извлечённые Claude (`semantically_similar_to`, помечены как INFERRED), уже в графе. Структура графа — это сигнал сходства. Отдельный шаг с эмбеддингами или векторная база данных не нужны. @@ -157,7 +157,7 @@ graphify отправляет содержимое файлов в API моде ## Технологический стек -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Семантическое извлечение через Claude, GPT-4 или модель вашей платформы. Транскрипция видео через faster-whisper + yt-dlp (опционально). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Семантическое извлечение через Claude, GPT-4 или модель вашей платформы. Транскрипция видео через faster-whisper + yt-dlp (опционально). ## Построено на graphify — Penpax diff --git a/docs/translations/README.sv-SE.md b/docs/translations/README.sv-SE.md index 63e015d95..d184051cb 100644 --- a/docs/translations/README.sv-SE.md +++ b/docs/translations/README.sv-SE.md @@ -27,13 +27,13 @@ Helt multimodal. Lägg till kod, PDF:er, markdown, skärmdumpar, diagram, whiteb graphify-out/ ├── graph.html interaktivt diagram — öppna i valfri webbläsare ├── GRAPH_REPORT.md gudnoder, överraskande kopplingar, föreslagna frågor -├── graph.json beständigt diagram — kan frågas veckor senare +├── graph.helix beständigt diagram — kan frågas veckor senare └── cache/ SHA256-cache — upprepade körningar behandlar bara ändrade filer ``` ## Hur det fungerar -graphify arbetar i tre pass. Först extraherar ett deterministiskt AST-pass struktur från kodfiler utan LLM. Sedan transkriberas video- och ljudfiler lokalt med faster-whisper. Slutligen kör Claude-subagenter parallellt på dokument, papper, bilder och transkriptioner. Resultaten slås samman i ett NetworkX-diagram, klustras med Leiden och exporteras som interaktiv HTML, frågebar JSON och revisionsrapport. +graphify arbetar i tre pass. Först extraherar ett deterministiskt AST-pass struktur från kodfiler utan LLM. Sedan transkriberas video- och ljudfiler lokalt med faster-whisper. Slutligen kör Claude-subagenter parallellt på dokument, papper, bilder och transkriptioner. Resultaten slås samman i ett Helix-diagram, klustras med Leiden och exporteras som interaktiv HTML, frågebar JSON och revisionsrapport. Varje relation är märkt `EXTRACTED`, `INFERRED` (med konfidenspoäng) eller `AMBIGUOUS`. diff --git a/docs/translations/README.th-TH.md b/docs/translations/README.th-TH.md index 8c1b05969..86612e82b 100644 --- a/docs/translations/README.th-TH.md +++ b/docs/translations/README.th-TH.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html กราฟแบบโต้ตอบ — เปิดในเบราว์เซอร์ใดก็ได้ ├── GRAPH_REPORT.md โหนดพระเจ้า, การเชื่อมต่อที่น่าประหลาดใจ, คำถามที่แนะนำ -├── graph.json กราฟถาวร — สามารถสืบค้นได้หลายสัปดาห์ต่อมา +├── graph.helix กราฟถาวร — สามารถสืบค้นได้หลายสัปดาห์ต่อมา └── cache/ SHA256-cache — การรันซ้ำประมวลผลเฉพาะไฟล์ที่เปลี่ยนแปลง ``` ## วิธีการทำงาน -graphify ทำงานใน 3 รอบ ก่อนอื่น AST pass แบบ deterministic ดึงโครงสร้างจากไฟล์โค้ดโดยไม่ต้องใช้ LLM จากนั้นไฟล์วิดีโอและเสียงถูกถอดเสียงในเครื่องด้วย faster-whisper สุดท้าย Claude sub-agent ทำงานแบบขนานกันบนเอกสาร, งานวิจัย, รูปภาพ และบทถอดเสียง ผลลัพธ์ถูกรวมเข้ากับกราฟ NetworkX, จัดกลุ่มด้วย Leiden และส่งออกเป็น HTML แบบโต้ตอบ, JSON ที่สืบค้นได้ และรายงานการตรวจสอบ +graphify ทำงานใน 3 รอบ ก่อนอื่น AST pass แบบ deterministic ดึงโครงสร้างจากไฟล์โค้ดโดยไม่ต้องใช้ LLM จากนั้นไฟล์วิดีโอและเสียงถูกถอดเสียงในเครื่องด้วย faster-whisper สุดท้าย Claude sub-agent ทำงานแบบขนานกันบนเอกสาร, งานวิจัย, รูปภาพ และบทถอดเสียง ผลลัพธ์ถูกรวมเข้ากับกราฟ Helix, จัดกลุ่มด้วย Leiden และส่งออกเป็น HTML แบบโต้ตอบ, JSON ที่สืบค้นได้ และรายงานการตรวจสอบ ความสัมพันธ์แต่ละอย่างถูกติดป้าย `EXTRACTED`, `INFERRED` (พร้อมคะแนนความเชื่อมั่น) หรือ `AMBIGUOUS` diff --git a/docs/translations/README.tr-TR.md b/docs/translations/README.tr-TR.md index 77f30742e..734f42772 100644 --- a/docs/translations/README.tr-TR.md +++ b/docs/translations/README.tr-TR.md @@ -27,13 +27,13 @@ Tamamen çok modlu. Kod, PDF, markdown, ekran görüntüleri, diyagramlar, beyaz graphify-out/ ├── graph.html etkileşimli grafik — herhangi bir tarayıcıda açın ├── GRAPH_REPORT.md tanrı düğümleri, şaşırtıcı bağlantılar, önerilen sorular -├── graph.json kalıcı grafik — haftalar sonra sorgulanabilir +├── graph.helix kalıcı grafik — haftalar sonra sorgulanabilir └── cache/ SHA256 önbelleği — tekrarlanan çalışmalar yalnızca değiştirilen dosyaları işler ``` ## Nasıl çalışır -graphify üç geçişte çalışır. Önce deterministik bir AST geçişi, LLM olmadan kod dosyalarından yapı çıkarır. Ardından video ve ses dosyaları faster-whisper ile yerel olarak transkribe edilir. Son olarak Claude alt ajanları belgeler, makaleler, görüntüler ve transkriptler üzerinde paralel olarak çalışır. Sonuçlar bir NetworkX grafiğinde birleştirilir, Leiden ile kümelenir ve etkileşimli HTML, sorgulanabilir JSON ve denetim raporu olarak dışa aktarılır. +graphify üç geçişte çalışır. Önce deterministik bir AST geçişi, LLM olmadan kod dosyalarından yapı çıkarır. Ardından video ve ses dosyaları faster-whisper ile yerel olarak transkribe edilir. Son olarak Claude alt ajanları belgeler, makaleler, görüntüler ve transkriptler üzerinde paralel olarak çalışır. Sonuçlar bir Helix grafiğinde birleştirilir, Leiden ile kümelenir ve etkileşimli HTML, sorgulanabilir JSON ve denetim raporu olarak dışa aktarılır. Her ilişki `EXTRACTED`, `INFERRED` (güven puanıyla) veya `AMBIGUOUS` olarak etiketlenir. diff --git a/docs/translations/README.uk-UA.md b/docs/translations/README.uk-UA.md index 2fa632544..528e4599f 100644 --- a/docs/translations/README.uk-UA.md +++ b/docs/translations/README.uk-UA.md @@ -37,7 +37,7 @@ graphify-out/ ├── graph.html відкрийте в будь-якому браузері — клікайте по вузлах, фільтруйте, шукайте ├── GRAPH_REPORT.md основне: ключові концепції, неочікувані зв’язки, запропоновані запитання -└── graph.json повний граф — запитуйте його будь-коли без повторного перечитування ваших файлів +└── graph.helix повний граф — запитуйте його будь-коли без повторного перечитування ваших файлів ``` Для читабельної сторінки архітектури з діаграмами викликів Mermaid виконайте: @@ -291,7 +291,7 @@ graphify-out/cost.json # лише локальний **Робочий процес:** 1. Одна людина запускає `/graphify .` і комітить `graphify-out/`. 2. Усі виконують pull — їхній асистент одразу читає граф. -3. Запустіть `graphify hook install` для автоматичного перебудування після кожного коміту (лише AST, без витрат API). Це також налаштовує git merge driver, щоб `graph.json` ніколи не залишався з маркерами конфліктів — два розробники, що комітять одночасно, отримають автоматично об'єднані графи. +3. Запустіть `graphify hook install` для автоматичного перебудування після кожного коміту (лише AST, без витрат API). Це також налаштовує git merge driver, щоб `graph.helix` ніколи не залишався з маркерами конфліктів — два розробники, що комітять одночасно, отримають автоматично об'єднані графи. 4. Коли документи або статті змінюються, запустіть `/graphify --update`, щоб оновити ці вузли. --- @@ -301,13 +301,13 @@ graphify-out/cost.json # лише локальний ```bash # запит до графу з терміналу graphify query "покажи потік автентифікації" -graphify query "що пов'язує DigestAuth з Response?" --graph graphify-out/graph.json +graphify query "що пов'язує DigestAuth з Response?" --graph graphify-out/graph.helix # відкрити граф як MCP-сервер (для повторного доступу через інструменти) -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix # зареєструвати в Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix ``` MCP-сервер надає асистенту структурований доступ: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. @@ -394,8 +394,8 @@ graphify cluster-only ./my-project --no-viz graphify query "..." ``` -**`graph.json` має маркери конфліктів після одночасного коміту двох розробників** -Запустіть `graphify hook install` — це налаштовує git merge driver, який автоматично об'єднує `graph.json`, щоб конфліктів ніколи не виникало. +**`graph.helix` має маркери конфліктів після одночасного коміту двох розробників** +Запустіть `graphify hook install` — це налаштовує git merge driver, який автоматично об'єднує `graph.helix`, щоб конфліктів ніколи не виникало. **Вилучення повертає порожні вузли/ребра для документів або PDF** Документи та PDF потребують LLM-виклику. Перевірте, що API-ключ встановлено і backend правильний: @@ -479,7 +479,7 @@ graphify extract ./docs --max-concurrency 2 # менше паралельни graphify extract ./docs --api-timeout 900 # довший HTTP тайм-аут для повільних локальних моделей (типово 600с) graphify extract ./docs --google-workspace # експортувати .gdoc/.gsheet/.gslides через gws перед витягуванням graphify extract ./docs --no-cluster # лише сире витягування, пропустити кластеризацію -graphify extract ./docs --force # перезаписати graph.json навіть якщо новий граф має менше вузлів (використовуйте після рефакторингу або для очищення фантомних дублікатів) +graphify extract ./docs --force # перезаписати graph.helix навіть якщо новий граф має менше вузлів (використовуйте після рефакторингу або для очищення фантомних дублікатів) graphify extract ./docs --dedup-llm # LLM-арбітр для неоднозначних пар сутностей (використовує той самий API-ключ) graphify extract ./docs --global --as myrepo # витягнути і зареєструвати в крос-проектний глобальний граф GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # підвищити ліміт виводу для щільних корпусів @@ -489,7 +489,7 @@ graphify export callflow-html --max-sections 8 # обмежити кіль graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json myrepo # зареєструвати граф проекту в ~/.graphify/global.json +graphify global add graphify-out/graph.helix myrepo # зареєструвати граф проекту в ~/.graphify/global.json graphify global remove myrepo # видалити проект з глобального графу graphify global list # показати всі зареєстровані репо + кількість вузлів/ребер graphify global path # вивести шлях до файлу глобального графу @@ -512,7 +512,7 @@ graphify update ./src graphify update ./src --no-cluster # пропустити рекластеризацію, записати лише сирий AST граф graphify update ./src --force # перезаписати навіть якщо новий граф має менше вузлів graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # власне розташування графу +graphify cluster-only ./my-project --graph path/to/graph.helix # власне розташування графу graphify cluster-only ./my-project --resolution 1.5 # більше, менших спільнот graphify cluster-only ./my-project --exclude-hubs 99 # виключити вузли p99 ступеня з розбиття ``` diff --git a/docs/translations/README.uz-UZ.md b/docs/translations/README.uz-UZ.md index 0e5f6e745..20c68ab0f 100644 --- a/docs/translations/README.uz-UZ.md +++ b/docs/translations/README.uz-UZ.md @@ -28,7 +28,7 @@ To'liq multimodal. Kod, PDF, markdown, ekran tasvirlari, diagrammalar, doska sur graphify-out/ ├── graph.html interaktiv graf — brauzerda oching, tugunlarni bosing, qidiring, filtrlang ├── GRAPH_REPORT.md god-tugunlar, kutilmagan aloqalar, taklif qilingan savollar -├── graph.json doimiy graf — haftalardan keyin qayta o'qimasdan so'rov qiling +├── graph.helix doimiy graf — haftalardan keyin qayta o'qimasdan so'rov qiling └── cache/ SHA256-kesh — qayta ishga tushirish faqat o'zgargan fayllarni qayta ishlaydi ``` @@ -46,7 +46,7 @@ Sintaksisi `.gitignore` ga o'xshash. ## Qanday ishlaydi -graphify uch bosqichda ishlaydi. Birinchi navbatda, deterministik AST bosqichi kod fayllaridan tuzilmani chiqaradi (klasslar, funksiyalar, importlar, chaqiruv graflari, docstring lar, sabab izohlari) — LLM ishtirokisiz. Keyin video va audio fayllar faster-whisper yordamida mahalliy ravishda transkripsiya qilinadi. Nihoyat, Claude subagentlari hujjatlar, maqolalar, tasvirlar va transkriptlar ustida parallel ishlab, tushunchalar, aloqalar va dizayn asoslarini chiqarib oladi. Natijalar NetworkX grafiga birlashtiriladi, Leiden hamjamiyat aniqlash algoritmi bilan klasterlanadi va interaktiv HTML, so'rov qilinadigan JSON hamda tabiiy tildagi audit hisoboti sifatida eksport qilinadi. +graphify uch bosqichda ishlaydi. Birinchi navbatda, deterministik AST bosqichi kod fayllaridan tuzilmani chiqaradi (klasslar, funksiyalar, importlar, chaqiruv graflari, docstring lar, sabab izohlari) — LLM ishtirokisiz. Keyin video va audio fayllar faster-whisper yordamida mahalliy ravishda transkripsiya qilinadi. Nihoyat, Claude subagentlari hujjatlar, maqolalar, tasvirlar va transkriptlar ustida parallel ishlab, tushunchalar, aloqalar va dizayn asoslarini chiqarib oladi. Natijalar Helix grafiga birlashtiriladi, Leiden hamjamiyat aniqlash algoritmi bilan klasterlanadi va interaktiv HTML, so'rov qilinadigan JSON hamda tabiiy tildagi audit hisoboti sifatida eksport qilinadi. **Klasterlash graf topologiyasiga asoslangan — embedding ishlatilmaydi.** Leiden hamjamiyatlarni qirralar zichligi bo'yicha topadi. Claude tomonidan chiqarilgan semantik o'xshashlik qirralari (`semantically_similar_to`, INFERRED deb belgilangan) allaqachon grafda. Graf tuzilmasining o'zi o'xshashlik signali. Alohida embedding bosqichi yoki vektor ma'lumotlar bazasi shart emas. @@ -157,7 +157,7 @@ graphify hujjatlar, maqolalar va tasvirlardan semantik chiqarish uchun fayl mazm ## Texnologiyalar to'plami -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantik chiqarish Claude, GPT-4 yoki sizning platformangiz modeli orqali. Video transkripsiyasi faster-whisper + yt-dlp orqali (opsional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Semantik chiqarish Claude, GPT-4 yoki sizning platformangiz modeli orqali. Video transkripsiyasi faster-whisper + yt-dlp orqali (opsional). ## graphify ustida qurilgan — Penpax diff --git a/docs/translations/README.vi-VN.md b/docs/translations/README.vi-VN.md index 5c44dc6aa..f7e39468d 100644 --- a/docs/translations/README.vi-VN.md +++ b/docs/translations/README.vi-VN.md @@ -27,13 +27,13 @@ Hoàn toàn đa phương thức. Thêm code, PDF, markdown, ảnh chụp màn h graphify-out/ ├── graph.html đồ thị tương tác — mở trong bất kỳ trình duyệt nào ├── GRAPH_REPORT.md nút thần, kết nối bất ngờ, câu hỏi được đề xuất -├── graph.json đồ thị liên tục — có thể truy vấn sau nhiều tuần +├── graph.helix đồ thị liên tục — có thể truy vấn sau nhiều tuần └── cache/ bộ nhớ đệm SHA256 — các lần chạy lại chỉ xử lý các tệp đã thay đổi ``` ## Cách hoạt động -graphify hoạt động theo ba lần duyệt. Đầu tiên, một lần duyệt AST xác định trích xuất cấu trúc từ các tệp code mà không cần LLM. Sau đó, các tệp video và âm thanh được phiên âm cục bộ bằng faster-whisper. Cuối cùng, các sub-agent Claude chạy song song trên các tài liệu, bài báo, hình ảnh và bản phiên âm. Kết quả được hợp nhất vào đồ thị NetworkX, phân cụm với Leiden và xuất dưới dạng HTML tương tác, JSON có thể truy vấn và báo cáo kiểm tra. +graphify hoạt động theo ba lần duyệt. Đầu tiên, một lần duyệt AST xác định trích xuất cấu trúc từ các tệp code mà không cần LLM. Sau đó, các tệp video và âm thanh được phiên âm cục bộ bằng faster-whisper. Cuối cùng, các sub-agent Claude chạy song song trên các tài liệu, bài báo, hình ảnh và bản phiên âm. Kết quả được hợp nhất vào đồ thị Helix, phân cụm với Leiden và xuất dưới dạng HTML tương tác, JSON có thể truy vấn và báo cáo kiểm tra. Mỗi mối quan hệ được gắn nhãn `EXTRACTED`, `INFERRED` (với điểm tin cậy) hoặc `AMBIGUOUS`. diff --git a/docs/translations/README.zh-CN.md b/docs/translations/README.zh-CN.md index e41832293..0be06a10c 100644 --- a/docs/translations/README.zh-CN.md +++ b/docs/translations/README.zh-CN.md @@ -19,13 +19,13 @@ graphify-out/ ├── graph.html 可交互图谱:可点节点、搜索、按社区过滤 ├── GRAPH_REPORT.md God nodes、意外连接、建议提问 -├── graph.json 持久化图谱:数周后仍可查询,无需重新读原始文件 +├── graph.helix 持久化图谱:数周后仍可查询,无需重新读原始文件 └── cache/ SHA256 缓存:重复运行时只处理变更过的文件 ``` ## 工作原理 -graphify 分两轮执行。第一轮是确定性的 AST 提取,对代码文件做结构分析(类、函数、导入、调用图、docstring、解释性注释),这一轮不需要 LLM。第二轮会并行调用 Claude 子代理处理文档、论文和图片,从中提取概念、关系和设计动机。最后把两边结果合并到一个 NetworkX 图里,用 Leiden 社区发现算法做聚类,并导出成可交互 HTML、可查询 JSON,以及一份人类可读的审计报告。 +graphify 分两轮执行。第一轮是确定性的 AST 提取,对代码文件做结构分析(类、函数、导入、调用图、docstring、解释性注释),这一轮不需要 LLM。第二轮会并行调用 Claude 子代理处理文档、论文和图片,从中提取概念、关系和设计动机。最后把两边结果合并到一个 Helix 图里,用 Leiden 社区发现算法做聚类,并导出成可交互 HTML、可查询 JSON,以及一份人类可读的审计报告。 **聚类是基于图拓扑完成的,不依赖 embeddings。** Leiden 按边密度发现社区。Claude 抽取出的语义相似边(`semantically_similar_to`,标记为 `INFERRED`)本来就存在于图中,所以会直接影响社区划分。图结构本身就是相似性信号,不需要额外的 embedding 步骤,也不需要向量数据库。 @@ -93,7 +93,7 @@ Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `mult 常驻 hook 会优先暴露 `GRAPH_REPORT.md` —— 这是一页式总结,包含 god nodes、社区结构和意外连接。你的助手在搜索文件前会先读它,因此会按结构导航,而不是按关键字乱搜。这已经能覆盖大部分日常问题。 -`/graphify query`、`/graphify path` 和 `/graphify explain` 会更深入:它们会逐跳遍历底层 `graph.json`,追踪节点之间的精确路径,并展示边级别细节(关系类型、置信度、源位置)。当你想从图谱里精确回答某个问题,而不仅仅是获得整体感知时,就该用这些命令。 +`/graphify query`、`/graphify path` 和 `/graphify explain` 会更深入:它们会逐跳遍历底层 `graph.helix`,追踪节点之间的精确路径,并展示边级别细节(关系类型、置信度、源位置)。当你想从图谱里精确回答某个问题,而不仅仅是获得整体感知时,就该用这些命令。 可以这样理解:常驻 hook 是先给助手一张地图,`/graphify` 这几个命令则是让它沿着地图精确导航。 @@ -204,7 +204,7 @@ graphify trae-cn uninstall | graphify 源码 + Transformer 论文 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx(合成 Python 库) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -Token 压缩效果会随着语料规模增大而更明显。6 个文件本来就塞得进上下文窗口,所以 graphify 在这种场景里的价值更多是结构清晰度,而不是 token 压缩。到了 52 个文件(代码 + 论文 + 图片)这种规模,就能做到 71x+。每个 `worked/` 目录里都带了原始输入和真实输出(`GRAPH_REPORT.md`、`graph.json`),你可以自己跑一遍核对数字。 +Token 压缩效果会随着语料规模增大而更明显。6 个文件本来就塞得进上下文窗口,所以 graphify 在这种场景里的价值更多是结构清晰度,而不是 token 压缩。到了 52 个文件(代码 + 论文 + 图片)这种规模,就能做到 71x+。每个 `worked/` 目录里都带了原始输入和真实输出(`GRAPH_REPORT.md`、`graph.helix`),你可以自己跑一遍核对数字。 ## 隐私 @@ -212,7 +212,7 @@ graphify 会把文档、论文和图片的内容发送给你所用 AI 编码助 ## 技术栈 -NetworkX + Leiden(graspologic)+ tree-sitter + vis.js。语义提取由 Claude(Claude Code)、GPT-4(Codex)或你当前平台所运行的模型完成。不需要 Neo4j,不需要 server,整体是纯本地运行。 +Helix + Leiden(native Leiden)+ tree-sitter + vis.js。语义提取由 Claude(Claude Code)、GPT-4(Codex)或你当前平台所运行的模型完成。不需要 Neo4j,不需要 server,整体是纯本地运行。
贡献 diff --git a/docs/translations/README.zh-TW.md b/docs/translations/README.zh-TW.md index 2bb90b4d4..0eae5fb12 100644 --- a/docs/translations/README.zh-TW.md +++ b/docs/translations/README.zh-TW.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html 互動式圖譜 — 在任何瀏覽器中開啟 ├── GRAPH_REPORT.md 神級節點、令人驚訝的連接、建議問題 -├── graph.json 持久圖譜 — 幾週後仍可查詢 +├── graph.helix 持久圖譜 — 幾週後仍可查詢 └── cache/ SHA256 快取 — 重複執行只處理已變更的檔案 ``` ## 運作原理 -graphify 分三個階段工作。首先,確定性 AST 遍歷在不使用 LLM 的情況下從程式碼檔案中提取結構。然後使用 faster-whisper 在本地轉錄視訊和音訊檔案。最後,Claude 子代理並行處理文件、論文、圖片和轉錄文字。結果被合併到 NetworkX 圖譜中,使用 Leiden 進行聚類,並匯出為互動式 HTML、可查詢 JSON 和審計報告。 +graphify 分三個階段工作。首先,確定性 AST 遍歷在不使用 LLM 的情況下從程式碼檔案中提取結構。然後使用 faster-whisper 在本地轉錄視訊和音訊檔案。最後,Claude 子代理並行處理文件、論文、圖片和轉錄文字。結果被合併到 Helix 圖譜中,使用 Leiden 進行聚類,並匯出為互動式 HTML、可查詢 JSON 和審計報告。 每個關係都標記為 `EXTRACTED`、`INFERRED`(帶有置信度分數)或 `AMBIGUOUS`。 diff --git a/graphify/__init__.py b/graphify/__init__.py index 41d4b1f3d..b537b8928 100644 --- a/graphify/__init__.py +++ b/graphify/__init__.py @@ -6,7 +6,7 @@ def __getattr__(name): _map = { "extract": ("graphify.extract", "extract"), "collect_files": ("graphify.extract", "collect_files"), - "build_from_json": ("graphify.build", "build_from_json"), + "build_from_extraction": ("graphify.build", "build_from_extraction"), "cluster": ("graphify.cluster", "cluster"), "score_all": ("graphify.cluster", "score_all"), "cohesion_score": ("graphify.cluster", "cohesion_score"), @@ -14,7 +14,6 @@ def __getattr__(name): "surprising_connections": ("graphify.analyze", "surprising_connections"), "suggest_questions": ("graphify.analyze", "suggest_questions"), "generate": ("graphify.report", "generate"), - "to_json": ("graphify.export", "to_json"), "to_html": ("graphify.export", "to_html"), "to_svg": ("graphify.export", "to_svg"), "to_canvas": ("graphify.export", "to_canvas"), diff --git a/graphify/__main__.py b/graphify/__main__.py index 924ae986d..f5d8e757d 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -123,7 +123,7 @@ _StageTimer, _clone_repo, _default_graph_path, - _enforce_graph_size_cap_or_exit, + _validate_store_or_exit, _run_hook_guard, _SEARCH_NUDGE, _READ_NUDGE, @@ -510,24 +510,11 @@ def _run_cli() -> None: print(" install [--platform P] copy skill to platform config dir (claude|windows|codebuddy|codex|opencode|aider|amp|agents|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi|devin)") print(" uninstall remove graphify from all detected platforms in one shot") print(" --purge also delete graphify-out/ directory") - print(" path \"A\" \"B\" shortest path between two nodes in graph.json") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" path \"A\" \"B\" shortest path between two nodes in graph.helix") + print(" --store Helix store (default graphify-out/graph.helix)") print(" explain \"X\" plain-language explanation of a node and its neighbors") - print(" --graph path to graph.json (default graphify-out/graph.json)") - print(" diagnose multigraph report same-endpoint edge collapse risk in graph.json") - print(" --graph path to graph/extraction JSON") - print(" (default graphify-out/graph.json)") - print(" --json emit machine-readable JSON") - print(" --max-examples N max same-endpoint examples to print (default 5)") - print(" --directed force directed post-build simulation") - print(" --undirected force undirected post-build simulation") - print(" (default follows JSON directed flag;") - print(" raw extraction with no flag defaults directed)") - print(" --extract-path PATH extractor source for suppression scan") + print(" --store Helix store (default graphify-out/graph.helix)") print(" clone clone a GitHub repo locally and print its path for /graphify") - print(" merge-driver git merge driver: union-merge two graph.json files (set up via hook install)") - print(" merge-graphs merge two or more graph.json files into one cross-repo graph") - print(" --out output path (default: graphify-out/merged-graph.json)") print(" --branch checkout a specific branch (default: repo default)") print(" --out clone to a custom directory (default: ~/.graphify/repos//)") print(" add fetch a URL and save it to ./raw, then update the graph") @@ -536,12 +523,14 @@ def _run_cli() -> None: print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") print(" update re-extract code files and update the graph (no LLM needed)") - print(" --force overwrite graph.json even if the rebuild has fewer nodes") - print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") + print(" --retain-rollback retain one previous native generation") + print(" rollback [--store PATH] activate the explicitly retained previous generation") + print(" doctor [--deep] verify native counts or perform a full checksum audit") + print(" --force force a full source rescan") print(" --no-cluster skip clustering, write raw extraction only") - print(" cluster-only rerun clustering on an existing graph.json and regenerate report") + print(" cluster-only rerun clustering on an existing graph.helix generation") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") - print(" --graph path to graph.json (default /graphify-out/graph.json)") + print(" --store Helix store (default /graphify-out/graph.helix)") print(" --no-label keep 'Community N' placeholders (skip LLM community naming)") print(" --backend= backend to use for community naming (default: auto-detect)") print(" --model= model to use for community naming") @@ -553,18 +542,17 @@ def _run_cli() -> None: print(" --model= model to use for community naming") print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") - print(" query \"\" BFS traversal of graph.json for a question") + print(" query \"\" native traversal of graph.helix for a question") print(" --dfs use depth-first instead of breadth-first") print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --store Helix store (default graphify-out/graph.helix)") print(" affected \"X\" reverse traversal to find nodes impacted by X") print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") - print(" --graph path to graph.json (default graphify-out/graph.json)") print(" god-nodes list the most connected nodes (architectural hubs)") print(" --top N how many to show (default 10)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --store Helix store (default graphify-out/graph.helix)") print(" --json emit JSON instead of text") print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") print(" --question Q the question asked") @@ -579,14 +567,12 @@ def _run_cli() -> None: print(" reflect aggregate graphify-out/memory/ outcomes into a deterministic lessons doc") print(" --memory-dir DIR memory directory (default: graphify-out/memory)") print(" --out FILE output path (default: graphify-out/reflections/LESSONS.md)") - print(" --graph PATH graph.json, for community grouping + dropping stale nodes (optional)") - print(" --analysis PATH .graphify_analysis.json (optional, auto-detected next to --graph)") - print(" --labels PATH .graphify_labels.json (optional, auto-detected next to --graph)") + print(" --store PATH graph.helix store for community grouping (optional)") print(" --half-life-days N signal weight halves every N days (default 30)") print(" --min-corroboration N distinct useful results to prefer a node (default 2)") print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") - print(" tree emit a D3 v7 collapsible-tree HTML for graph.json") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" tree emit a D3 v7 collapsible-tree HTML from graph.helix") + print(" --store PATH Helix store (default graphify-out/graph.helix)") print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") print(" --root PATH filesystem root for the hierarchy") print(" --max-children N cap children per node (default 200)") @@ -618,12 +604,12 @@ def _run_cli() -> None: print(" --cargo extract crate→crate deps from Cargo.toml") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") - print(" global add add/update a project graph in the global graph (~/.graphify/global-graph.json)") + print(" global add add/update a project in ~/.graphify/global-graph.helix") + print(" --retain-rollback retain one previous global generation") print(" --as repo tag (default: parent directory name)") print(" global remove remove a repo's nodes from the global graph") print(" global list list repos in the global graph") - print(" global path print path to the global graph file") - print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") + print(" benchmark [graph.helix] measure token reduction vs naive full-corpus approach") print(" export callflow-html emit Mermaid-based architecture/call-flow HTML") print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") diff --git a/graphify/_minhash.py b/graphify/_minhash.py index aabeb654f..c3a94cb43 100644 --- a/graphify/_minhash.py +++ b/graphify/_minhash.py @@ -44,7 +44,7 @@ def __init__(self, num_perm: int = 128) -> None: self._a, self._b = _mh_coeffs(num_perm) def update(self, v: bytes) -> None: - hv = np.uint64(struct.unpack(" str: - data = graph.nodes[node_id] +def _node_label(graph: Any, node_id: Any) -> str: + data = node_attributes(graph, node_id) return str(data.get("label") or node_id) @@ -61,8 +61,8 @@ def _normalize_label(label: str) -> str: def _prefer_file_node( - graph: nx.Graph, - node_ids: list[str], + graph: Any, + node_ids: list[Any], query: str, ) -> str | None: """Return the file-level node when a source_file query matches many nodes.""" @@ -70,8 +70,8 @@ def _prefer_file_node( exact_file_nodes = [ node_id for node_id in node_ids - if str(graph.nodes[node_id].get("source_location", "")) == "L1" - and _normalize_label(str(graph.nodes[node_id].get("label", ""))) == query_basename + if str(node_attributes(graph, node_id).get("source_location", "")) == "L1" + and _normalize_label(str(node_attributes(graph, node_id).get("label", ""))) == query_basename ] if len(exact_file_nodes) == 1: return exact_file_nodes[0] @@ -79,7 +79,7 @@ def _prefer_file_node( l1_nodes = [ node_id for node_id in node_ids - if str(graph.nodes[node_id].get("source_location", "")) == "L1" + if str(node_attributes(graph, node_id).get("source_location", "")) == "L1" ] if len(l1_nodes) == 1: return l1_nodes[0] @@ -87,7 +87,7 @@ def _prefer_file_node( basename_nodes = [ node_id for node_id in node_ids - if _normalize_label(str(graph.nodes[node_id].get("label", ""))) == query_basename + if _normalize_label(str(node_attributes(graph, node_id).get("label", ""))) == query_basename ] if len(basename_nodes) == 1: return basename_nodes[0] @@ -95,17 +95,25 @@ def _prefer_file_node( return None -def resolve_seed(graph: nx.Graph, query: str) -> str | None: +def resolve_seed(graph: Any, query: str, *, node_query: Any) -> Any | None: + """Resolve a seed from a bounded native predicate projection.""" # A trailing path separator must not change a source-file match — serve's # _find_node tokenizes the path (which drops it), so strip it here for parity # (otherwise `affected "src/x.ts/"` returned None while `explain` resolved it). query = query.rstrip("/\\") or query - if query in graph: + if graph.contains_node(query): return query + if node_query is None: + raise RuntimeError("affected analysis requires the native Helix query interface") query_lower = _normalize_label(query) + candidate_terms = [query] + if query_lower.endswith("()"): + candidate_terms.append(query[:-2]) + candidate_ids = node_query.candidate_ids(candidate_terms) exact_label_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _normalize_label(str(data.get("label", ""))) == query_lower ] if len(exact_label_matches) == 1: @@ -115,15 +123,17 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: # contains pass. Match on the undecorated name before giving up. query_bare = _bare_name(query_lower) bare_name_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _bare_name(str(data.get("label", ""))) == query_bare ] if len(bare_name_matches) == 1: return bare_name_matches[0] exact_source_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _normalize_label(str(data.get("source_file", ""))) == query_lower ] if len(exact_source_matches) == 1: @@ -133,8 +143,9 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: if preferred_file_node is not None: return preferred_file_node contains_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if query_lower in _normalize_label(str(data.get("label", ""))) ] if len(contains_matches) == 1: @@ -143,16 +154,15 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: def affected_nodes( - graph: nx.Graph, - seed: str, + graph: Any, + seed: Any, *, relations: Iterable[str] = DEFAULT_AFFECTED_RELATIONS, depth: int = 2, ) -> list[AffectedHit]: - relation_set = set(relations) - seen = {seed} - queue: deque[tuple[str, int]] = deque([(seed, 0)]) - hits: list[AffectedHit] = [] + from helixdb.graph import TraversalOptions + + relation_list = tuple(dict.fromkeys(str(relation) for relation in relations)) # #1669: seed the reverse walk with the root's own member nodes (one outward # `method`/`contains` hop). A caller can bind to a class's method node rather @@ -161,64 +171,51 @@ def affected_nodes( # otherwise. The member nodes are seeds only (not reported as hits), and # `method`/`contains` stay out of the general relation-filtered walk, so this # adds no forward noise anywhere else. - if hasattr(graph, "out_edges"): - member_edges = graph.out_edges(seed, data=True) - else: - member_edges = ( - (s, t, d) for s, t, d in graph.edges(data=True) if s == seed - ) - for _s, member, data in member_edges: - if str(data.get("relation", "")) not in ("method", "contains"): + member_seeds: list[Any] = [] + member_edges = (graph.edge(edge_id) for edge_id in graph.out_edge_ids(seed)) + for edge in member_edges: + if edge is None: continue - member = str(member) - if member not in seen: - seen.add(member) - queue.append((member, 0)) - - while queue: - current, current_depth = queue.popleft() - if current_depth >= depth: + member, data = edge.target, edge_attributes(edge) + if str(data.get("relation", "")) not in ("method", "contains"): continue - if hasattr(graph, "in_edges"): - incoming = graph.in_edges(current, data=True) - else: - incoming = ( - (source, target, data) - for source, target, data in graph.edges(data=True) - if target == current - ) - for source, _target, data in incoming: - relation = str(data.get("relation", "")) - if relation not in relation_set: - continue - source = str(source) - if source in seen: - continue - seen.add(source) - # Carry the matched edge's location (taken from the SAME edge dict - # whose relation passed the filter, so relation and location stay - # consistent) — that is the call/import/reference site in `source`'s - # own file, which is where the user should click (#BUG1). - hit = AffectedHit( - source, current_depth + 1, relation, - via_file=str(data.get("source_file") or "") or None, - via_location=str(data.get("source_location") or "") or None, - ) - hits.append(hit) - queue.append((source, current_depth + 1)) - - return hits + if member != seed and member not in member_seeds: + member_seeds.append(member) + + seeds = (seed, *member_seeds) + result = graph.traverse(TraversalOptions( + seeds=seeds, + max_depth=max(0, depth), + direction="in", + allowed_labels=relation_list, + )) + seed_set = set(seeds) + via: dict[Any, tuple[str, str | None, str | None]] = {} + for traversed in result.discovery_edges: + edge = graph.edge(traversed.edge_id) + data = edge_attributes(edge) if edge is not None else {} + via[traversed.source] = ( + str(data.get("relation") or traversed.label or ""), + str(data.get("source_file") or "") or None, + str(data.get("source_location") or "") or None, + ) + return [ + AffectedHit(visit.node_id, visit.depth, *via.get(visit.node_id, ("", None, None))) + for visit in result.visits + if visit.node_id not in seed_set + ] def format_affected( - graph: nx.Graph, + graph: Any, query: str, *, + node_query: Any, relations: Iterable[str] = DEFAULT_AFFECTED_RELATIONS, depth: int = 2, ) -> str: relation_list = tuple(relations) - seed = resolve_seed(graph, query) + seed = resolve_seed(graph, query, node_query=node_query) if seed is None: return f"No unique node match for {query}" @@ -233,7 +230,7 @@ def format_affected( return "\n".join(lines) for hit in hits: - data = graph.nodes[hit.node_id] + data = node_attributes(graph, hit.node_id) if hit.via_location: # The relation SITE in this node's file (call/import/reference line), # labeled by [via_relation] so it's never mistaken for a def line. @@ -246,27 +243,12 @@ def format_affected( return "\n".join(lines) -def load_graph(path: Path) -> nx.Graph: - import json - from networkx.readwrite import json_graph - +def load_graph(path: Path): + """Open the active immutable graph from a Helix store directory.""" try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: + return load_helix_graph(path).graph + except (OSError, RuntimeError, ValueError) as exc: raise RuntimeError( - f"Cannot read graph file {path}: {exc}. " + f"Cannot open Helix store {path}: {exc}. " "Re-run 'graphify extract' to regenerate it." ) from exc - # Force directed so stored caller→callee direction survives the round-trip; - # mirrors serve.py and __main__.py (#1174). - raw = {**raw, "directed": True} - # Normalize the edge key: graphify's `extract` output uses "edges" while - # networkx's node_link_data default is "links". Without this, an edges-keyed - # graph.json raises an uncaught KeyError: 'links' here — every other loader - # (__main__.py) already normalizes this (#738; same class as #1198). - if "links" not in raw and "edges" in raw: - raw = dict(raw, links=raw["edges"]) - try: - return json_graph.node_link_graph(raw, edges="links") - except TypeError: - return json_graph.node_link_graph(raw) diff --git a/graphify/always_on/agents-md.md b/graphify/always_on/agents-md.md index 6511cd1dd..f3725dd93 100644 --- a/graphify/always_on/agents-md.md +++ b/graphify/always_on/agents-md.md @@ -5,7 +5,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community st When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. diff --git a/graphify/always_on/antigravity-rules.md b/graphify/always_on/antigravity-rules.md index 0fc786414..1b2f06f24 100644 --- a/graphify/always_on/antigravity-rules.md +++ b/graphify/always_on/antigravity-rules.md @@ -8,7 +8,7 @@ description: Consult the graphify knowledge graph at graphify-out/ for codebase This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/graphify/always_on/claude-md.md b/graphify/always_on/claude-md.md index 417efeb27..f0b390010 100644 --- a/graphify/always_on/claude-md.md +++ b/graphify/always_on/claude-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/gemini-md.md b/graphify/always_on/gemini-md.md index 417efeb27..f0b390010 100644 --- a/graphify/always_on/gemini-md.md +++ b/graphify/always_on/gemini-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/kiro-steering.md b/graphify/always_on/kiro-steering.md index cb6f4543d..0e4221868 100644 --- a/graphify/always_on/kiro-steering.md +++ b/graphify/always_on/kiro-steering.md @@ -2,4 +2,4 @@ inclusion: always --- -graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/graphify/always_on/vscode-instructions.md b/graphify/always_on/vscode-instructions.md index 9cb983c95..aa7a32840 100644 --- a/graphify/always_on/vscode-instructions.md +++ b/graphify/always_on/vscode-instructions.md @@ -1,7 +1,7 @@ ## graphify For any question about this repo's architecture, structure, components, or how to add/modify/find -code, your first action should be `graphify query ""` when `graphify-out/graph.json` +code, your first action should be `graphify query ""` when `graphify-out/graph.helix` exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` for focused-concept questions. These return a scoped subgraph, usually much smaller than the full report or raw grep output. diff --git a/graphify/analyze.py b/graphify/analyze.py index ec1e61a99..ed9fdf0b6 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -1,9 +1,23 @@ """Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" from __future__ import annotations from pathlib import Path -import networkx as nx +from typing import Any -from graphify.build import edge_data +from graphify.helix.access import ( + degree as _degree, + degree_map, + edge_rows as _edges, + first_edge_attributes as edge_data, + node_attributes, + node_ids, +) +from graphify.helix.model import graphify_attributes + + +def _graphify_betweenness_options(): + from helixdb.graph import BetweennessOptions + + return BetweennessOptions(mode="auto", sample_count=100, seed=42, exact_through=1_000) # Builtin/mock names that can appear as annotation-derived nodes in pre-existing # graphs. Excluded from god-node ranking so they don't displace real abstractions @@ -52,7 +66,7 @@ def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: return {n: cid for cid, nodes in communities.items() for n in nodes} -def _is_file_node(G: nx.Graph, node_id: str) -> bool: +def _is_file_node(G: Any, node_id: str) -> bool: """ Return True if this node is a file-level hub node (e.g. 'client', 'models') or an AST method stub (e.g. '.auth_flow()', '.__init__()'). @@ -60,7 +74,7 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: These are synthetic nodes created by the AST extractor and should be excluded from god nodes, surprising connections, and knowledge gap reporting. """ - attrs = G.nodes[node_id] + attrs = node_attributes(G, node_id) label = attrs.get("label", "") if not label: return False @@ -76,7 +90,7 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: return True # Module-level function stub: labeled 'function_name()' - only has a contains edge # These are real functions but structurally isolated by definition; not a gap worth flagging - if label.endswith("()") and G.degree(node_id) <= 1: + if label.endswith("()") and _degree(G, node_id) <= 1: return True return False @@ -89,8 +103,8 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: }) -def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: - attrs = G.nodes[node_id] +def _is_json_key_node(G: Any, node_id: str) -> bool: + attrs = node_attributes(G, node_id) src = (attrs.get("source_file") or "").lower() if not src.endswith(".json"): return False @@ -98,23 +112,23 @@ def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: return label in _JSON_NOISE_LABELS -def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: +def god_nodes(G: Any, top_n: int = 10) -> list[dict]: """Return the top_n most-connected real entities - the core abstractions. File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) result = [] for node_id, deg in sorted_nodes: if _is_file_node(G, node_id) or _is_concept_node(G, node_id) or _is_json_key_node(G, node_id): continue - if G.nodes[node_id].get("label", "") in _BUILTIN_NOISE_LABELS: + if node_attributes(G, node_id).get("label", "") in _BUILTIN_NOISE_LABELS: continue result.append({ "id": node_id, - "label": G.nodes[node_id].get("label", node_id), + "label": node_attributes(G, node_id).get("label", node_id), "degree": deg, }) if len(result) >= top_n: @@ -123,7 +137,7 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: def surprising_connections( - G: nx.Graph, + G: Any, communities: dict[int, list[str]] | None = None, top_n: int = 5, ) -> list[dict]: @@ -143,7 +157,8 @@ def surprising_connections( # Identify unique source files (ignore empty/null source_file) source_files = { data.get("source_file", "") - for _, data in G.nodes(data=True) + for node in G.nodes() + if (data := graphify_attributes(node.attributes)) is not None if data.get("source_file", "") } is_multi_source = len(source_files) > 1 @@ -154,7 +169,7 @@ def surprising_connections( return _cross_community_surprises(G, communities or {}, top_n) -def _is_concept_node(G: nx.Graph, node_id: str) -> bool: +def _is_concept_node(G: Any, node_id: str) -> bool: """ Return True if this node is a manually-injected semantic concept node rather than a real entity found in source code. @@ -163,7 +178,7 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool: - Empty source_file - source_file doesn't look like a real file path (no extension) """ - data = G.nodes[node_id] + data = node_attributes(G, node_id) source = data.get("source_file", "") if not source: return True @@ -193,7 +208,7 @@ def _top_level_dir(path: str) -> str: def _surprise_score( - G: nx.Graph, + G: Any, u: str, v: str, data: dict, @@ -242,7 +257,7 @@ def _surprise_score( score += 2 reasons.append("connects across different repos/directories") - # 4. Cross-community bonus - Leiden says these are structurally distant + # 4. Cross-community bonus - Leiden says these are structurally distant cid_u = node_community.get(u) cid_v = node_community.get(v) if cid_u is not None and cid_v is not None and cid_u != cid_v and not _suppress_structural: @@ -255,18 +270,18 @@ def _surprise_score( reasons.append("semantically similar concepts with no structural link") # 5. Peripheral→hub: a low-degree node connecting to a high-degree one - deg_u = degrees[u] if degrees is not None else G.degree(u) - deg_v = degrees[v] if degrees is not None else G.degree(v) + deg_u = degrees[u] if degrees is not None else _degree(G, u) + deg_v = degrees[v] if degrees is not None else _degree(G, v) if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5: score += 1 - peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v) - hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u) + peripheral = node_attributes(G, u).get("label", u) if deg_u <= 2 else node_attributes(G, v).get("label", v) + hub = node_attributes(G, v).get("label", v) if deg_u <= 2 else node_attributes(G, u).get("label", u) reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`") return score, reasons -def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]: +def _cross_file_surprises(G: Any, communities: dict[int, list[str]], top_n: int) -> list[dict]: """ Cross-file edges between real code/doc entities, ranked by a composite surprise score rather than confidence alone. @@ -281,10 +296,10 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: Each result includes a 'why' field explaining what makes it non-obvious. """ node_community = _node_community_map(communities) - degrees = dict(G.degree()) + degrees = degree_map(G) candidates = [] - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): relation = data.get("relation", "") if relation in ("imports", "imports_from", "contains", "method"): continue @@ -293,26 +308,26 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: if _is_file_node(G, u) or _is_file_node(G, v): continue - u_source = G.nodes[u].get("source_file", "") - v_source = G.nodes[v].get("source_file", "") + u_source = node_attributes(G, u).get("source_file", "") + v_source = node_attributes(G, v).get("source_file", "") if not u_source or not v_source or u_source == v_source: continue score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source, degrees) src_id = data.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v candidates.append({ "_score": score, - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), + "source": node_attributes(G, src_id).get("label", src_id), + "target": node_attributes(G, tgt_id).get("label", tgt_id), "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), + node_attributes(G, src_id).get("source_file", ""), + node_attributes(G, tgt_id).get("source_file", ""), ], "confidence": data.get("confidence", "EXTRACTED"), "relation": relation, @@ -330,7 +345,7 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: def _cross_community_surprises( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], top_n: int, ) -> list[dict]: @@ -343,21 +358,28 @@ def _cross_community_surprises( """ if not communities: # No community info - use edge betweenness centrality - if G.number_of_edges() == 0: + if G.edge_count == 0: return [] - if G.number_of_nodes() > 5000: + if G.node_count > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) - top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] + top_edges = sorted( + G.edge_betweenness_centrality(_graphify_betweenness_options()), + key=lambda row: row.score, + reverse=True, + )[:top_n] result = [] - for (u, v), score in top_edges: + for row in top_edges: + edge = G.edge(row.edge_id) + if edge is None: + continue + u, v, score = edge.source, edge.target, row.score data = edge_data(G, u, v) result.append({ - "source": G.nodes[u].get("label", u), - "target": G.nodes[v].get("label", v), + "source": node_attributes(G, u).get("label", u), + "target": node_attributes(G, v).get("label", v), "source_files": [ - G.nodes[u].get("source_file", ""), - G.nodes[v].get("source_file", ""), + node_attributes(G, u).get("source_file", ""), + node_attributes(G, v).get("source_file", ""), ], "confidence": data.get("confidence", "EXTRACTED"), "relation": data.get("relation", ""), @@ -369,7 +391,7 @@ def _cross_community_surprises( node_community = _node_community_map(communities) surprises = [] - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): cid_u = node_community.get(u) cid_v = node_community.get(v) if cid_u is None or cid_v is None or cid_u == cid_v: @@ -383,17 +405,17 @@ def _cross_community_surprises( # This edge crosses community boundaries - interesting confidence = data.get("confidence", "EXTRACTED") src_id = data.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v surprises.append({ - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), + "source": node_attributes(G, src_id).get("label", src_id), + "target": node_attributes(G, tgt_id).get("label", tgt_id), "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), + node_attributes(G, src_id).get("source_file", ""), + node_attributes(G, tgt_id).get("source_file", ""), ], "confidence": confidence, "relation": relation, @@ -418,7 +440,7 @@ def _cross_community_surprises( def suggest_questions( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], community_labels: dict[int, str], top_n: int = 7, @@ -435,10 +457,10 @@ def suggest_questions( node_community = _node_community_map(communities) # 1. AMBIGUOUS edges → unresolved relationship questions - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): if data.get("confidence") == "AMBIGUOUS": - ul = G.nodes[u].get("label", u) - vl = G.nodes[v].get("label", v) + ul = node_attributes(G, u).get("label", u) + vl = node_attributes(G, v).get("label", v) relation = data.get("relation", "related to") questions.append({ "type": "ambiguous_edge", @@ -447,9 +469,11 @@ def suggest_questions( }) # 2. Bridge nodes (high betweenness) → cross-cutting concern questions - if G.number_of_edges() > 0: - k = min(100, G.number_of_nodes()) if G.number_of_nodes() > 1000 else None - betweenness = nx.betweenness_centrality(G, k=k, seed=42) + if G.edge_count > 0: + betweenness = { + row.node_id: row.score + for row in G.betweenness_centrality(_graphify_betweenness_options()) + } # Top bridge nodes that are NOT file-level hubs bridges = sorted( [(n, s) for n, s in betweenness.items() @@ -458,11 +482,15 @@ def suggest_questions( reverse=True, )[:3] for node_id, score in bridges: - label = G.nodes[node_id].get("label", node_id) + label = node_attributes(G, node_id).get("label", node_id) cid = node_community.get(node_id) comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown" neighbors = list(G.neighbors(node_id)) - neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid} + neighbor_comms = { + neighbor_cid + for n in neighbors + if (neighbor_cid := node_community.get(n)) is not None and neighbor_cid != cid + } if neighbor_comms: other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms] questions.append({ @@ -472,7 +500,7 @@ def suggest_questions( }) # 3. God nodes with many INFERRED edges → verification questions - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} top_nodes = sorted( [(n, d) for n, d in degree.items() if not _is_file_node(G, n)], key=lambda x: x[1], @@ -480,22 +508,22 @@ def suggest_questions( )[:5] for node_id, _ in top_nodes: inferred = [ - (u, v, d) for u, v, d in G.edges(node_id, data=True) + (u, v, d) for u, v, d, _ in _edges(G, node_id) if d.get("confidence") == "INFERRED" ] if len(inferred) >= 2: - label = G.nodes[node_id].get("label", node_id) + label = node_attributes(G, node_id).get("label", node_id) # Use _src/_tgt to get the correct direction; fall back to v (the other node) others = [] for u, v, d in inferred[:2]: src_id = d.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = d.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v other_id = tgt_id if src_id == node_id else src_id - others.append(G.nodes[other_id].get("label", other_id)) + others.append(node_attributes(G, other_id).get("label", other_id)) questions.append({ "type": "verify_inferred", "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?", @@ -504,14 +532,14 @@ def suggest_questions( # 4. Isolated or weakly-connected nodes → exploration questions isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 + n for n in node_ids(G) + if _degree(G, n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + and node_attributes(G, n).get("file_type") != "rationale" ] if isolated: - labels = [G.nodes[n].get("label", n) for n in isolated[:3]] + labels = [node_attributes(G, n).get("label", n) for n in isolated[:3]] questions.append({ "type": "isolated_nodes", "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?", @@ -545,7 +573,7 @@ def suggest_questions( return questions[:top_n] -def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: +def graph_diff(G_old: Any, G_new: Any) -> dict: """Compare two graph snapshots and return what changed. Returns: @@ -557,40 +585,40 @@ def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: "summary": "3 new nodes, 5 new edges, 1 node removed" } """ - old_nodes = set(G_old.nodes()) - new_nodes = set(G_new.nodes()) + old_nodes = {node.id for node in G_old.nodes()} + new_nodes = {node.id for node in G_new.nodes()} added_node_ids = new_nodes - old_nodes removed_node_ids = old_nodes - new_nodes new_nodes_list = [ - {"id": n, "label": G_new.nodes[n].get("label", n)} + {"id": n, "label": node_attributes(G_new, n).get("label", n)} for n in added_node_ids ] removed_nodes_list = [ - {"id": n, "label": G_old.nodes[n].get("label", n)} + {"id": n, "label": node_attributes(G_old, n).get("label", n)} for n in removed_node_ids ] - def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: - if G.is_directed(): + def edge_key(G: Any, u: str, v: str, data: dict) -> tuple: + if G.directed: return (u, v, data.get("relation", "")) return (min(u, v), max(u, v), data.get("relation", "")) old_edge_keys = { edge_key(G_old, u, v, d) - for u, v, d in G_old.edges(data=True) + for u, v, d, _ in _edges(G_old) } new_edge_keys = { edge_key(G_new, u, v, d) - for u, v, d in G_new.edges(data=True) + for u, v, d, _ in _edges(G_new) } added_edge_keys = new_edge_keys - old_edge_keys removed_edge_keys = old_edge_keys - new_edge_keys new_edges_list = [] - for u, v, d in G_new.edges(data=True): + for u, v, d, _ in _edges(G_new): if edge_key(G_new, u, v, d) in added_edge_keys: new_edges_list.append({ "source": u, @@ -600,7 +628,7 @@ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: }) removed_edges_list = [] - for u, v, d in G_old.edges(data=True): + for u, v, d, _ in _edges(G_old): if edge_key(G_old, u, v, d) in removed_edge_keys: removed_edges_list.append({ "source": u, @@ -630,7 +658,7 @@ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: def find_import_cycles( - G: nx.Graph, + G: Any, max_cycle_length: int = 5, top_n: int = 20, ) -> list[dict]: @@ -654,15 +682,17 @@ def find_import_cycles( } """ def _endpoint_source_file(node_id: str) -> str: - attrs = G.nodes.get(node_id, {}) + if not G.contains_node(node_id): + return "" + attrs = node_attributes(G, node_id) src_file = attrs.get("source_file", "") return src_file if isinstance(src_file, str) else "" # Step 1: Build a directed file-level graph from import/re-export edges. # IMPORTANT: resolve endpoints using source_file only; never infer from label/id. - file_graph = nx.DiGraph() + adjacency: dict[str, set[str]] = {} - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): rel = data.get("relation", "") if rel not in ("imports_from", "re_exports"): continue @@ -694,21 +724,27 @@ def _endpoint_source_file(node_id: str) -> str: if not tgt_file: continue - file_graph.add_edge(src_file_attr, tgt_file) + adjacency.setdefault(src_file_attr, set()).add(tgt_file) - if not file_graph.edges(): + if not adjacency: return [] # Step 2: Find simple cycles, bounded by length. - # Pass length_bound so networkx prunes during enumeration rather than - # enumerating all elementary cycles and post-filtering — avoids exponential - # blowup on dense graphs with many long cycles (#1196). + # Bound enumeration to avoid exponential blowup on dense graphs. cycles: list[list[str]] = [] - for cycle in nx.simple_cycles(file_graph, length_bound=max_cycle_length): - if len(cycle) <= max_cycle_length: - cycles.append(cycle) + + def walk(start: str, current: str, path: list[str], seen: set[str]) -> None: + if len(cycles) >= top_n * 10 or len(path) > max_cycle_length: + return + for target in sorted(adjacency.get(current, ())): + if target == start: + cycles.append(path.copy()) + elif target not in seen and len(path) < max_cycle_length: + walk(start, target, [*path, target], {*seen, target}) + + for start in sorted(adjacency): + walk(start, start, [start], {start}) if len(cycles) >= top_n * 10: - # Stop early to avoid combinatorial explosion break # Step 3: Sort by length (shortest = tightest coupling), then deduplicate. diff --git a/graphify/benchmark.py b/graphify/benchmark.py index ee9c3925e..7f07d02aa 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -1,14 +1,12 @@ """Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach.""" from __future__ import annotations -import json import sys -from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph +from typing import Any from graphify.build import edge_data +from graphify.helix.model import graphify_attributes, node_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, load_graph from graphify.serve import _query_terms -from graphify.paths import default_graph_json as _default_graph_json _CHARS_PER_TOKEN = 4 # standard approximation @@ -37,11 +35,12 @@ def _estimate_tokens(text: str) -> int: return max(1, len(text) // _CHARS_PER_TOKEN) -def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int: +def _query_subgraph_tokens(G: Any, question: str, depth: int = 3) -> int: """Run BFS from best-matching nodes and return estimated tokens in the subgraph context.""" terms = _query_terms(question) scored = [] - for nid, data in G.nodes(data=True): + for node in G.nodes(): + nid, data = node.id, graphify_attributes(node.attributes) label = data.get("label", "").lower() score = sum(1 for t in terms if t in label) if score > 0: @@ -51,27 +50,22 @@ def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int: if not start_nodes: return 0 - visited: set[str] = set(start_nodes) - frontier = set(start_nodes) - edges_seen: list[tuple] = [] - for _ in range(depth): - next_frontier: set[str] = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in visited: - next_frontier.add(neighbor) - edges_seen.append((n, neighbor)) - visited.update(next_frontier) - frontier = next_frontier + from helixdb.graph import TraversalOptions + + traversal = G.traverse(TraversalOptions( + seeds=tuple(start_nodes), max_depth=depth, direction="both" + )) + visited = {visit.node_id for visit in traversal.visits} lines = [] for nid in visited: - d = G.nodes[nid] + d = node_attributes(G, nid) lines.append(f"NODE {d.get('label', nid)} src={d.get('source_file', '')} loc={d.get('source_location', '')}") - for u, v in edges_seen: + for traversed in traversal.discovery_edges: + u, v = traversed.source, traversed.target if u in visited and v in visited: d = edge_data(G, u, v) - lines.append(f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')}--> {G.nodes[v].get('label', v)}") + lines.append(f"EDGE {node_attributes(G, u).get('label', u)} --{d.get('relation', '')}--> {node_attributes(G, v).get('label', v)}") return _estimate_tokens("\n".join(lines)) @@ -99,19 +93,13 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - graph_path = graph_path or _default_graph_json() - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(Path(graph_path)) - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) + G = load_graph(graph_path or DEFAULT_PROJECT_STORE).graph if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context - corpus_words = G.number_of_nodes() * 50 + corpus_words = G.node_count * 50 + assert corpus_words is not None corpus_tokens = corpus_words * 100 // 75 # words → tokens (100 words ≈ 133 tokens) qs = questions or _SAMPLE_QUESTIONS @@ -130,8 +118,8 @@ def run_benchmark( return { "corpus_tokens": corpus_tokens, "corpus_words": corpus_words, - "nodes": G.number_of_nodes(), - "edges": G.number_of_edges(), + "nodes": G.node_count, + "edges": G.edge_count, "avg_query_tokens": avg_query_tokens, "reduction_ratio": reduction_ratio, "per_question": per_question, diff --git a/graphify/build.py b/graphify/build.py index 44e8c441c..5a7cacdd8 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1,4 +1,4 @@ -# assemble node+edge dicts into a NetworkX graph, preserving edge direction +# Assemble extraction records into Helix build data, preserving edge direction. # # Node deduplication — three layers: # @@ -6,9 +6,8 @@ # emitted at most once per file, so duplicate class/function definitions in # the same source file are collapsed to the first occurrence. # -# 2. Between files (build): NetworkX G.add_node() is idempotent — calling it -# twice with the same ID overwrites the attributes with the second call's -# values. Nodes are added in extraction order (AST first, then semantic), +# 2. Between files (build): the transient Helix build DTO is last-writer-wins +# for duplicate IDs. Nodes are processed in extraction order (AST first, then semantic), # so if the same entity is extracted by both passes the semantic node # silently overwrites the AST node. This is intentional: semantic nodes # carry richer labels and cross-file context, while AST nodes have precise @@ -28,9 +27,15 @@ import sys import unicodedata from pathlib import Path -import networkx as nx +from typing import Any from .ids import make_id, normalize_id as _normalize_id -from .paths import default_graph_json as _default_graph_json +from .helix.access import all_edge_attributes, first_edge_attributes +from .helix.model import ( + EdgeData, + GraphBuildData, + NodeData, +) +from .helix.persistence import DEFAULT_PROJECT_STORE from .validate import validate_extraction @@ -75,7 +80,7 @@ # Hyperedge member lists are canonically keyed `nodes` (see graphify/llm.py -# extraction spec), but LLM/subagent drift and externally-supplied graph.json +# extraction spec), but LLM/subagent drift and externally supplied payloads # sometimes emit `members` or `node_ids`. _normalize_hyperedge_members folds # those aliases into `nodes` at ingest so every downstream consumer reads one # canonical key — mirroring the `from`/`to` edge-endpoint tolerance below. @@ -226,13 +231,24 @@ def _file_label_reassignments(items: "list[tuple]") -> dict: return out -def _disambiguate_file_node_labels(G: "nx.Graph") -> None: - """Relabel colliding-basename file nodes on a graph (#2032). Ids/edges are - never changed — only display labels. Idempotent (labels derive from - source_file, not the current possibly-qualified label).""" - items = [(nid, a.get("label"), a.get("source_file")) for nid, a in G.nodes(data=True)] - for nid, new_label in _file_label_reassignments(items).items(): - G.nodes[nid]["label"] = new_label +def _disambiguate_file_node_labels(graph: GraphBuildData) -> None: + """Relabel colliding-basename file nodes on transient native build data.""" + items = [ + (node.id, node.attributes.get("label"), node.attributes.get("source_file")) + for node in graph.nodes + ] + replacements = _file_label_reassignments(items) + if not replacements: + return + graph.nodes = [ + NodeData( + node.id, + {**dict(node.attributes), "label": replacements[node.id]} + if node.id in replacements + else node.attributes, + ) + for node in graph.nodes + ] def disambiguate_file_labels_in_nodes(nodes: "list") -> None: @@ -253,8 +269,8 @@ def _infer_merge_root(graph_path: Path) -> str | None: Prefers the committed ``graphify-out/.graphify_root`` marker — the authoritative scan root graphify records at build/watch time (#686/#1423) — then falls back to - the directory that contains the output dir (``graph.json``'s grandparent, i.e. - ``/graphify-out/graph.json`` -> ````). Returns None if neither + the directory that contains the output dir (the store's grandparent, i.e. + ``/graphify-out/graph.helix`` -> ````). Returns None if neither resolves, in which case normalization is a no-op (prior behavior). """ try: @@ -271,31 +287,20 @@ def _infer_merge_root(graph_path: Path) -> str | None: return None -def edge_data(G: nx.Graph, u: str, v: str) -> dict: - """Return one edge attribute dict for (u, v), tolerating MultiGraph. - - For MultiGraph/MultiDiGraph there can be multiple parallel edges; - this returns the first one (sufficient for callers that only need - relation/confidence for rendering). Fixes #796. - """ - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return next(iter(raw.values()), {}) - return raw +def edge_data(G, u: str, v: str) -> dict: + """Return one native Helix edge attribute projection for ``(u, v)``.""" + return first_edge_attributes(G, u, v) -def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: +def edge_datas(G, u: str, v: str) -> list[dict]: """Return every edge attribute dict for (u, v); always a list.""" - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return list(raw.values()) - return [raw] + return all_edge_attributes(G, u, v) def dedupe_nodes(nodes: list[dict]) -> list[dict]: """Collapse nodes sharing an ``id``, last-writer-wins on attributes. - Mirrors what ``build_from_json``'s ``G.add_node`` does implicitly (idempotent; + Mirrors what ``build_from_extraction``'s ``G.add_node`` does implicitly (idempotent; a later node overwrites an earlier one's attributes). The ``--no-cluster`` write path dumps the raw node list without building a graph, so same-id nodes — e.g. a Swift ``type=module`` anchor emitted once per importing file (#1327) @@ -315,13 +320,9 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: """Collapse exact parallel edges by ``(source, target, relation)``, keeping the first occurrence. - The clustered build path runs edges through a NetworkX ``DiGraph``, which - collapses parallel edges automatically. The ``--no-cluster`` and incremental - ``update`` write paths bypass NetworkX and concatenate edge lists raw, so - duplicates accumulate and edge counts become non-deterministic across build - modes / repeated updates (#1317). Deduping on the connectivity identity is - zero-signal-loss and restores idempotency. Callers that intentionally keep - parallel edges (multigraph output) must not use this. + Raw extraction and incremental updates can concatenate edge lists, so exact + duplicates would otherwise accumulate across repeated builds. Callers that + intentionally keep parallel edges (multigraph output) must not use this. """ seen: set[tuple] = set() out: list[dict] = [] @@ -334,6 +335,127 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: return out +def build_unclustered_extraction( + extraction: dict, + *, + root: str | Path | None = None, +) -> GraphBuildData: + """Preserve the historical ``--no-cluster`` multigraph semantics. + + The v8 fast path wrote the deduplicated extraction records directly and + consumers rehydrated them as an undirected multigraph. Rehydration also + created empty external/stdlib endpoint nodes for otherwise dangling import + edges. Building a simple graph here silently collapsed relation-distinct + parallel edges and discarded those endpoints, so keep the raw topology in + the transient DTO and let Helix persist it natively. + """ + build_root = str(Path(root).resolve()) if root else None + raw_nodes = dedupe_nodes(list(extraction.get("nodes", []))) + raw_edges = dedupe_edges( + list(extraction.get("edges", extraction.get("links", []))) + ) + + node_attrs: dict[object, dict[str, Any]] = {} + for raw in raw_nodes: + if not isinstance(raw, dict) or "id" not in raw: + continue + node_id = raw["id"] + try: + hash(node_id) + except TypeError: + continue + attrs = {key: value for key, value in raw.items() if key != "id"} + if "source_file" in attrs: + attrs["source_file"] = _norm_source_file( + attrs["source_file"], build_root + ) + node_attrs[node_id] = attrs + + # Match the historical v8 multigraph loader's automatic per-pair integer keys. Explicit + # keys overwrite the same pair/key record, while unkeyed records receive + # the next unused integer for that undirected pair. + used_keys: dict[frozenset[object], set[object]] = {} + edge_by_identity: dict[tuple[frozenset[object], object], EdgeData] = {} + for raw in raw_edges: + if not isinstance(raw, dict): + continue + source = raw.get("source", raw.get("from")) + target = raw.get("target", raw.get("to")) + try: + hash(source) + hash(target) + except TypeError: + continue + if source is None or target is None: + continue + node_attrs.setdefault(source, {}) + node_attrs.setdefault(target, {}) + pair = frozenset((source, target)) + pair_keys = used_keys.setdefault(pair, set()) + if "key" in raw: + key = raw["key"] + try: + hash(key) + except TypeError: + continue + else: + key = len(pair_keys) + while key in pair_keys: + key += 1 + pair_keys.add(key) + attrs = { + attr: value + for attr, value in raw.items() + if attr + not in {"source", "target", "from", "to", "key", "target_file"} + } + relation = attrs.get("relation") + if not isinstance(relation, str) or not relation: + attrs["relation"] = "related_to" + if "source_file" in attrs: + attrs["source_file"] = _norm_source_file( + attrs["source_file"], build_root + ) + edge_by_identity[(pair, key)] = EdgeData(source, target, attrs, key) + + raw_hyperedges = extraction.get("hyperedges", []) + hyperedges = [] + if isinstance(raw_hyperedges, list): + for raw in raw_hyperedges: + if not isinstance(raw, dict): + hyperedges.append(raw) + continue + record = dict(raw) + if "source_file" in record: + record["source_file"] = _norm_source_file( + record["source_file"], build_root + ) + hyperedges.append(record) + graph_attributes = ( + {"hyperedges": list(hyperedges)} + if hyperedges + else {} + ) + reserved = { + "directed", + "multigraph", + "graph", + "nodes", + "edges", + "links", + "hyperedges", + } + return GraphBuildData( + kind="multigraph", + nodes=[NodeData(node_id, attrs) for node_id, attrs in node_attrs.items()], + edges=list(edge_by_identity.values()), + attributes=graph_attributes, + extras={ + key: value for key, value in extraction.items() if key not in reserved + }, + ) + + def _old_file_stems(rel: Path) -> list[str]: """Pre-migration stem forms a semantic fragment may have used for ``rel``. @@ -487,8 +609,14 @@ def _doc_twin_remap(nodes: list) -> dict[str, str]: return remap -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: - """Build a NetworkX graph from an extraction dict. +def build_from_extraction( + extraction: dict, + *, + directed: bool = False, + multigraph: bool | None = None, + root: str | Path | None = None, +) -> GraphBuildData: + """Build transient Helix records from an extraction dict. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. @@ -496,7 +624,10 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat relative to root so all nodes share a consistent path key (#932). """ _root = str(Path(root).resolve()) if root else None - # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. + directed = directed or bool(extraction.get("directed", False)) + if multigraph is None: + multigraph = bool(extraction.get("multigraph", False)) + # Accept the historical node-link spelling while normalizing in memory. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) @@ -518,7 +649,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat file=sys.stderr, ) node["source_file"] = node.pop("source") - # Default missing/None file_type to "concept" so legacy graph.json + # Default missing/None file_type to "concept" so legacy native graph # entries (and stub nodes preserved by `_rebuild_code` from older # graphify versions that didn't always populate file_type) don't # trigger spurious "invalid file_type 'None'" validator warnings (#660). @@ -531,14 +662,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Canonicalize hyperedge member lists (#1561): producers sometimes key the # member list `members`/`node_ids` instead of `nodes`. Fold aliases onto # `nodes` here — BEFORE validation and the semantic-rekey loop below — so - # every downstream consumer (rekey, source_file relativize, to_json) reads + # every downstream consumer reads # one canonical key, the same way edge endpoints alias from/to at build. for he in extraction.get("hyperedges", []) or []: _normalize_hyperedge_members(he) errors = validate_extraction(extraction) - # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. - real_errors = [e for e in errors if "does not match any node id" not in e] + real_errors = [error for error in errors if "does not match any node id" not in error] if real_errors: print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) # Deterministic semantic re-key (#1504/#1509): the node-ID stem is now the @@ -594,11 +724,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + node_attrs: dict[object, dict] = {} for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted - # by a buggy LLM extraction) so NetworkX add_node never raises - # TypeError: unhashable type. Non-dict nodes are deliberately left to + # by a buggy LLM extraction) so construction never raises a late + # TypeError. Non-dict nodes are deliberately left to # raise as before, so callers that probe build for shape errors (e.g. # the multigraph diagnostic) still observe the malformed shape. if isinstance(node, dict): @@ -615,8 +745,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat continue if "source_file" in node: node["source_file"] = _norm_source_file(node["source_file"], _root) - G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - node_set = set(G.nodes()) + node_attrs[node["id"]] = {k: v for k, v in node.items() if k != "id"} + node_set = set(node_attrs) # #1145 (extended): merge LLM ghost-duplicate nodes into AST canonical nodes. # Original bug: AST uses parent-qualified IDs (mingpt_bpe_get_pairs) while LLM @@ -639,8 +769,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # canonical winner and the ambiguity decisions below don't flip run-to-run # with CPython's per-process string-hash seed (#1753) — the same reason the # edge-iteration loop further down sorts on purpose. - for nid in sorted(node_set): - attrs = G.nodes[nid] + for nid in sorted(node_set, key=repr): + attrs = node_attrs[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) if not label or not sf: @@ -659,7 +789,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if is_ast: # Two AST nodes on the same key (same file, same label) is an # ambiguous collision. - if key in _loc_nodes and G.nodes[_loc_nodes[key]].get("_origin") == "ast": + if key in _loc_nodes and node_attrs[_loc_nodes[key]].get("_origin") == "ast": _loc_collisions.add(key) # AST-origin nodes always overwrite a prior non-AST entry. _loc_nodes[key] = nid @@ -670,8 +800,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _loc_nodes.setdefault(key, nid) # Pass 2: find ghosts — non-AST nodes that have an AST canonical twin. - for nid in sorted(node_set): - attrs = G.nodes[nid] + for nid in sorted(node_set, key=repr): + attrs = node_attrs[nid] if attrs.get("_origin") == "ast": continue # AST nodes are never ghosts label = str(attrs.get("label", "")).strip() @@ -691,13 +821,15 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _ghost_remap[sem_id] = ast_id # Remove ghost nodes from the graph; edges will be re-pointed via norm_to_id. for ghost_id in _ghost_remap: - G.remove_node(ghost_id) + node_attrs.pop(ghost_id, None) node_set.discard(ghost_id) # Normalized ID map: lets edges survive when the LLM generates IDs with # slightly different casing or punctuation than the AST extractor. # e.g. "Session_ValidateToken" maps to "session_validatetoken". - norm_to_id: dict[str, str] = {_normalize_id(nid): nid for nid in node_set} + norm_to_id: dict[str, object] = { + _normalize_id(nid): nid for nid in node_set if isinstance(nid, str) + } # Also map ghost IDs to their canonical AST replacements. for ghost_id, canonical_id in _ghost_remap.items(): norm_to_id[_normalize_id(ghost_id)] = canonical_id @@ -734,7 +866,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat from graphify.extractors.base import _file_stem as _fs _alias_candidates: dict[str, set[str]] = {} for nid in node_set: - attrs = G.nodes[nid] + if not isinstance(nid, str): + continue + attrs = node_attrs[nid] sf = attrs.get("source_file") if not sf: continue @@ -757,6 +891,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for alias_key, candidates in _alias_candidates.items(): if len(candidates) == 1: norm_to_id.setdefault(alias_key, next(iter(candidates))) + edge_by_pair: dict[object, EdgeData] = {} + # Iterate edges in a deterministic order. The graph is undirected and stores # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run @@ -810,11 +946,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Sanitize numeric edge fields (#1960): an explicit ``"weight": null`` in # the extraction JSON survives ``.get("weight", 1.0)`` (the key is present, # so the default never applies) and reaches Louvain/Leiden as None, - # crashing modularity arithmetic with a TypeError (graspologic's Leiden + # crashing native modularity arithmetic with a TypeError (Leiden # even panics on NaN). Coerce to float and fall back to the schema default # of 1.0 for anything the clustering backends reject — None, non-numeric # strings, NaN/inf, negatives — while numeric strings coerce cleanly. - # Repair (not drop) the key so graph.json round-trips a clean value and a + # Repair (not drop) the key so native graph round-trips a clean value and a # cluster-only/--update reload never re-ingests the null. attrs = {k: v for k, v in edge.items() if k not in ("source", "target", "target_file", "local_alias")} for _num_key in ("weight", "confidence_score"): @@ -826,13 +962,19 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if not math.isfinite(_num_val) or _num_val < 0: _num_val = 1.0 attrs[_num_key] = _num_val + if "confidence_score" not in attrs: + attrs["confidence_score"] = { + "EXTRACTED": 1.0, + "INFERRED": 0.5, + "AMBIGUOUS": 0.2, + }.get(str(attrs.get("confidence", "EXTRACTED")), 1.0) # Backfill source_file from the endpoint nodes (every node carries one). # Semantic/LLM edges occasionally omit it, which downstream validation # flags and leaves query results with no file reference (#1279). if not attrs.get("source_file"): attrs["source_file"] = ( - G.nodes[src].get("source_file") - or G.nodes[tgt].get("source_file") + node_attrs[src].get("source_file") + or node_attrs[tgt].get("source_file") or "" ) if "source_file" in attrs: @@ -844,8 +986,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Python `import time` must not bind to a `time.ts`, #1749). _edge_rel = attrs.get("relation") if _edge_rel in ("calls", "imports", "imports_from", "references"): - src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() - tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() + src_ext = Path(node_attrs[src].get("source_file") or "").suffix.lower() + tgt_ext = Path(node_attrs[tgt].get("source_file") or "").suffix.lower() src_fam = _EDGE_LANG_FAMILY.get(src_ext) tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext) if _edge_rel == "calls": @@ -883,25 +1025,29 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # earlier one's _src/_tgt, silently flipping the surviving edge's caller # and callee. First-seen direction wins instead — drop the redundant # reverse-direction duplicate so the original direction is preserved (#1061). - if not G.is_directed() and G.has_edge(src, tgt): - existing = edge_data(G, src, tgt) - if existing.get("relation") == attrs.get("relation") and ( - existing.get("_src") == tgt and existing.get("_tgt") == src + edge_key = edge.get("key") if multigraph else None + pair_base: object = (src, tgt) if directed else frozenset((src, tgt)) + pair: object = (pair_base, edge_key) if multigraph else pair_base + existing = edge_by_pair.get(pair) + if not directed and existing is not None: + existing_attrs = dict(existing.attributes) + if existing_attrs.get("relation") == attrs.get("relation") and ( + existing_attrs.get("_src") == tgt and existing_attrs.get("_tgt") == src ): continue - G.add_edge(src, tgt, **attrs) + edge_by_pair[pair] = EdgeData(src, tgt, attrs, edge_key) hyperedges = extraction.get("hyperedges", []) if hyperedges: # Relativize hyperedge source_file the same way nodes and edges are - # (above), so to_json — which has no root and writes G.graph["hyperedges"] - # verbatim — never leaks an absolute path from a semantic subagent (#1418). + # (above), so native persistence never leaks an absolute path from a + # semantic subagent (#1418). kept_hyperedges = [] for he in hyperedges: if isinstance(he, dict) and he.get("source_file"): he["source_file"] = _norm_source_file(he["source_file"], _root) # Validate members against the built node set (#1916): a hyperedge # member absent from the graph used to be copied into - # G.graph["hyperedges"] verbatim and reach graph.json dangling, + # G.graph["hyperedges"] verbatim and reach native graph dangling, # even from a live (non-cache) extraction. Mirror the pairwise-edge # handling above: remap mismatched ids via normalization first, # then drop members that still don't resolve; drop the hyperedge @@ -930,24 +1076,40 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if valid_members != he["nodes"]: he["nodes"] = valid_members kept_hyperedges.append(he) - if kept_hyperedges: - G.graph["hyperedges"] = kept_hyperedges # Runs LAST, after the alias-competition above (which relies on file-node # labels still being bare basenames): give colliding-basename file nodes a # directory-qualified display label so lookup/discovery can disambiguate # them (#2032). Labels only — ids and edges are untouched. - _disambiguate_file_node_labels(G) - return G + items = [ + (node_id, attrs.get("label"), attrs.get("source_file")) + for node_id, attrs in node_attrs.items() + ] + for node_id, new_label in _file_label_reassignments(items).items(): + node_attrs[node_id]["label"] = new_label + graph_attributes = {"hyperedges": kept_hyperedges} if hyperedges and kept_hyperedges else {} + kind = ( + "multidigraph" if directed and multigraph else + "digraph" if directed else + "multigraph" if multigraph else + "graph" + ) + return GraphBuildData( + kind=kind, + nodes=[NodeData(node_id, attrs) for node_id, attrs in node_attrs.items()], + edges=list(edge_by_pair.values()), + attributes=graph_attributes, + ) def build( extractions: list[dict], *, directed: bool = False, + multigraph: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: +) -> GraphBuildData: """Merge multiple extraction results into one graph. directed=True produces a DiGraph that preserves edge direction (source→target). @@ -976,7 +1138,9 @@ def build( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed, root=root) + return build_from_extraction( + combined, directed=directed, multigraph=multigraph, root=root + ) def _norm_label(label: str | None) -> str: @@ -1046,48 +1210,75 @@ def build_merge( graph_path: str | Path | None = None, prune_sources: list[str] | None = None, *, - directed: bool = False, + directed: bool | None = None, + multigraph: bool | None = None, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: - """Load existing graph.json, merge new chunks into it, and save back. - - Re-extracted files REPLACE their prior contribution: any source_file present - in new_chunks is dropped from the loaded graph before merging, so a changed - file's stale nodes/edges don't accumulate. Files absent from new_chunks are - preserved unchanged; deleted files are removed via prune_sources. - Safe to call repeatedly. + base_graph: GraphBuildData | None = None, + base_root: str | Path | None = None, + replace_origin_sources: dict[str, set[str]] | None = None, +) -> GraphBuildData: + """Merge extraction chunks into an explicit transient extraction DTO. + + Production updates rebuild this DTO from the durable extraction cache. This + helper accepts an explicit transient base only; it never projects an active + ``NativeGraph`` back into Python. root: if given, absolute source_file paths in new_chunks are made relative (#932). """ - graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) - if graph_path.exists(): - # Read JSON directly instead of going through node_link_graph(). - # The latter rebuilds an undirected nx.Graph and then enumerating - # edges() yields endpoints based on node insertion order, which - # silently flips directional edges (e.g. `calls`) when the callee - # was inserted before the caller. The _src/_tgt direction-preserving - # attrs are popped before saving in export.py, so going through the - # NetworkX round-trip loses direction permanently (#760). - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot read {graph_path} for incremental merge: {exc}. " - "Delete the file and run a full rebuild." - ) from exc - links_key = "links" if "links" in data else "edges" - existing_nodes = list(data.get("nodes", [])) - existing_edges = list(data.get(links_key, [])) - existing_hyperedges = list(data.get("hyperedges", [])) + graph_path = Path(graph_path if graph_path is not None else DEFAULT_PROJECT_STORE) + if base_graph is not None: + existing_nodes = [ + {"id": node.id, **dict(node.attributes)} + for node in base_graph.nodes + ] + existing_edges = [ + { + "source": edge.source, + "target": edge.target, + **dict(edge.attributes), + **({"key": edge.key} if base_graph.multigraph else {}), + } + for edge in base_graph.edges + ] + raw_hyperedges = base_graph.attributes.get("hyperedges", []) + existing_hyperedges = list(raw_hyperedges) if isinstance(raw_hyperedges, list) else [] + if directed is None: + directed = base_graph.directed + if multigraph is None: + multigraph = base_graph.multigraph had_graph = True else: existing_nodes = [] existing_edges = [] existing_hyperedges = [] had_graph = False + if directed is None: + directed = False + if multigraph is None: + multigraph = False + + # Preserve source identity when invocation style changes (for example an + # absolute ``src`` build followed by a project-relative ``src`` update). + # The active generation's paths are relative to ``base_root``; fresh paths + # are relative to ``root``. Rebase only paths that fit under the new root. + old_root = Path(base_root).resolve() if base_root is not None else None + new_root = Path(root).resolve() if root is not None else None + if old_root is not None and new_root is not None and old_root != new_root: + def _rebase_source(item: dict) -> None: + source = item.get("source_file") + if not source: + return + source_path = Path(str(source)) + absolute = source_path if source_path.is_absolute() else old_root / source_path + try: + item["source_file"] = absolute.resolve().relative_to(new_root).as_posix() + except (OSError, ValueError): + return + + for item in [*existing_nodes, *existing_edges, *existing_hyperedges]: + if isinstance(item, dict): + _rebase_source(item) # Effective root for relativizing absolute source_file / prune paths back to the # stored relative source_file keys. When the caller passes root we use it; @@ -1112,26 +1303,63 @@ def build_merge( # absolute win32 paths while the stored graph keeps relative posix (#1007). _replace_root = _eff_root new_sources: set[str] = set() + replaced_origins: dict[str, set[str]] = {} + for source, origins in (replace_origin_sources or {}).items(): + identities = {source} + norm = _norm_source_file(source, _replace_root) + if norm: + identities.add(norm) + new_sources.update(identities) + for identity in identities: + replaced_origins.setdefault(identity, set()).update(origins) for ch in new_chunks: - for n in ch.get("nodes", []): - sf = n.get("source_file") - if not sf: - continue - new_sources.add(sf) - norm = _norm_source_file(sf, _replace_root) - if norm: - new_sources.add(norm) + for bucket in ("nodes", "edges", "hyperedges"): + for item in ch.get(bucket, []): + sf = item.get("source_file") + if not sf: + continue + identities = {sf} + norm = _norm_source_file(sf, _replace_root) + if norm: + identities.add(norm) + new_sources.update(identities) + origin = item.get("_origin") + if isinstance(origin, str) and origin: + for identity in identities: + replaced_origins.setdefault(identity, set()).add(origin) if new_sources: def _kept(item: dict) -> bool: sf = item.get("source_file") - return sf not in new_sources and _norm_source_file(sf, _replace_root) not in new_sources + norm = _norm_source_file(sf, _replace_root) + if sf not in new_sources and norm not in new_sources: + return True + origin = item.get("_origin") + if not isinstance(origin, str) or not origin: + return True + return ( + origin not in replaced_origins.get(sf, set()) + and origin not in replaced_origins.get(norm, set()) + ) existing_nodes = [n for n in existing_nodes if _kept(n)] existing_edges = [e for e in existing_edges if _kept(e)] + if prune_sources: + existing_nodes = [ + node + for node in existing_nodes + if node.get("source_file") or node.get("_origin") != "ast" + ] base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, + directed=directed, + multigraph=multigraph, + dedup=dedup, + dedup_llm_backend=dedup_llm_backend, + root=root, + ) # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes @@ -1192,22 +1420,36 @@ def _prune_match(sf: "str | None") -> bool: continue sf = he.get("source_file") norm = _norm_source_file(sf, _eff_root) - if sf in new_sources or norm in new_sources: - continue # re-extracted — replaced by the new chunk's version + origin = he.get("_origin") + if ( + (sf in new_sources or norm in new_sources) + and isinstance(origin, str) + and ( + origin in replaced_origins.get(sf, set()) + or origin in replaced_origins.get(norm, set()) + ) + ): + continue # this extraction tier was replaced if _prune_match(sf): continue # deleted — pruned carried.append(he) if carried: - from graphify.export import attach_hyperedges - attach_hyperedges(G, carried) + current = G.attributes.setdefault("hyperedges", []) + current_ids = { + item.get("id") for item in current if isinstance(item, dict) + } + current.extend( + item for item in carried + if item.get("id") not in current_ids + ) # Prune nodes and edges from deleted source files if prune_sources: - to_remove = [ - n for n, d in G.nodes(data=True) - if _prune_match(d.get("source_file")) - ] - G.remove_nodes_from(to_remove) + to_remove = { + node.id for node in G.nodes + if _prune_match(node.attributes.get("source_file")) + } + G.nodes = [node for node in G.nodes if node.id not in to_remove] n_files = len(prune_sources) n_nodes = len(to_remove) if n_nodes: @@ -1217,11 +1459,14 @@ def _prune_match(sf: "str | None") -> bool: ) edges_to_remove = [ - (u, v) for u, v, d in G.edges(data=True) - if _prune_match(d.get("source_file")) + edge for edge in G.edges + if edge.source in to_remove + or edge.target in to_remove + or _prune_match(edge.attributes.get("source_file")) ] if edges_to_remove: - G.remove_edges_from(edges_to_remove) + removed = set(id(edge) for edge in edges_to_remove) + G.edges = [edge for edge in G.edges if id(edge) not in removed] print( f"[graphify] Pruned {len(edges_to_remove)} edge(s) from deleted source file(s).", file=sys.stderr, @@ -1238,7 +1483,7 @@ def _prune_match(sf: "str | None") -> bool: # Skip when dedup or prune_sources is active — shrinkage is intentional there. if graph_path.exists() and not dedup and not prune_sources: existing_n = len(existing_nodes) - new_n = G.number_of_nodes() + new_n = G.node_count if new_n < existing_n: raise ValueError( f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " @@ -1248,7 +1493,7 @@ def _prune_match(sf: "str | None") -> bool: return G -def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: +def prefix_graph_for_global(G: GraphBuildData, repo_tag: str) -> GraphBuildData: """Return a copy of G with all node IDs prefixed with repo_tag::. Labels are preserved unchanged (for display). A 'local_id' attribute @@ -1256,22 +1501,39 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: rewritten to match the new prefixed IDs. The 'repo' attribute is set on every node. """ - relabel = {n: f"{repo_tag}::{n}" for n in G.nodes} - H = nx.relabel_nodes(G, relabel, copy=True) - for node, data in H.nodes(data=True): - data["repo"] = repo_tag - data.setdefault("local_id", node.split("::", 1)[1]) - return H + relabel = {node.id: f"{repo_tag}::{node.id}" for node in G.nodes} + nodes = [] + for node in G.nodes: + attributes = dict(node.attributes) + attributes["repo"] = repo_tag + attributes.setdefault("local_id", node.id) + nodes.append(NodeData(relabel[node.id], attributes)) + edges = [ + EdgeData( + relabel[edge.source], + relabel[edge.target], + dict(edge.attributes), + edge.key, + ) + for edge in G.edges + ] + return GraphBuildData( + kind=G.kind, + nodes=nodes, + edges=edges, + attributes=dict(G.attributes), + extras=dict(G.extras), + ) def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": - """Return a unique, human-meaningful repo tag per input graph for merge-graphs. + """Return a unique, human-meaningful repo tag per input graph. The naive tag (the ``graphify-out`` parent dir name) is NOT unique across inputs: ``src/graphify-out`` and ``frontend/src/graphify-out`` both yield ``src``. Prefixing both node sets with ``src::`` then makes same-stem nodes (a backend ``src/app.js`` and a frontend ``App.jsx``, both bare ``app``) - collide, so ``nx.compose`` silently merges two unrelated entities and invents + collide, so a merge silently combines two unrelated entities and invents cross-runtime edges (#1729). Colliding tags are widened with their own parent dir (``frontend_src``), then an index suffix guarantees uniqueness so no two graphs ever share a prefix. @@ -1292,8 +1554,12 @@ def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": return unique -def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: - """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" - to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] - G.remove_nodes_from(to_remove) +def prune_repo_from_graph(G: GraphBuildData, repo_tag: str) -> int: + """Remove all records tagged with repo_tag from build data. Returns node count.""" + to_remove = {node.id for node in G.nodes if node.attributes.get("repo") == repo_tag} + G.nodes = [node for node in G.nodes if node.id not in to_remove] + G.edges = [ + edge for edge in G.edges + if edge.source not in to_remove and edge.target not in to_remove + ] return len(to_remove) diff --git a/graphify/cache.py b/graphify/cache.py index 0a86a7e80..1367232cc 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -1,808 +1,277 @@ -# per-file extraction cache - skip unchanged files on re-run +"""Extraction caches stored inside Helix generation state. + +The caller owns the cache dictionary and persists it under +``state["incremental"]["extraction_cache"]`` with the topology generation. +No cache sidecars are read or written. +""" + from __future__ import annotations -import atexit +import copy import hashlib -import json -import os import re -import tempfile import warnings from collections.abc import Iterable +from importlib.metadata import PackageNotFoundError, version as package_version from pathlib import Path +from typing import Any, Callable -# Output directory name — override with GRAPHIFY_OUT env var for worktrees or -# shared-output setups. Accepts a relative name ("graphify-out-feature") or an -# absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths -# (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT -# AST cache entries are the output of graphify's own extractor code, so they -# are only valid for the version that wrote them: keying purely on file -# content means extractor fixes shipped in a new release keep serving stale -# pre-fix results. The AST cache is therefore namespaced by package version -# (cache/ast/v{version}/), with entries from other versions removed on first -# use. The semantic cache is deliberately NOT versioned — its entries are -# produced by the LLM from file contents, and invalidating them on every -# release would re-bill extraction for unchanged files. -try: - from importlib.metadata import version as _pkg_version +_FRONTMATTER_DELIM = re.compile(r"^---[ \t]*\r?$", re.MULTILINE) - _EXTRACTOR_VERSION = _pkg_version("graphifyy") -except Exception: +try: + _EXTRACTOR_VERSION = package_version("graphifyy") +except PackageNotFoundError: _EXTRACTOR_VERSION = "unknown" -# Version dirs already swept this process — cleanup runs once per (base, version). -_cleaned_ast_dirs: set[str] = set() - - -def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: - """Remove AST cache entries left behind by other graphify versions. - - Sweeps sibling ``v*/`` directories and unversioned ``*.json`` entries - (the pre-versioning layout) under ``cache/ast/``. Best-effort: failures - are ignored, stragglers are retried on the next run. - """ - key = str(current_dir) - if key in _cleaned_ast_dirs: - return - _cleaned_ast_dirs.add(key) - if not ast_base.is_dir(): - return - import shutil - - for child in ast_base.iterdir(): - if child == current_dir: - continue - try: - if child.is_dir() and child.name.startswith("v"): - shutil.rmtree(child, ignore_errors=True) - elif child.suffix == ".json": - child.unlink() - except OSError: - pass - - -# Semantic cache entries are LLM output, so they depend on the extraction prompt -# that produced them, not just on file contents. Keying purely on content means a -# release that changes the prompt keeps replaying entries from the older prompt on -# every unchanged file, silently mixing extraction vintages in one graph (#1939). -# Versioning them by package version (as the AST cache does) would re-bill LLM -# extraction on every patch release — the reason #1252 deliberately left them -# unversioned. Fingerprinting the prompt itself keeps both properties: entries -# survive releases that don't touch the prompt, and invalidate only when it -# actually changed. Entries live under cache/semantic/p{fingerprint}/ when the -# caller supplies its prompt; callers that don't keep the historical flat layout. -_PROMPT_FP_LEN = 12 - -# Count of pre-fingerprint (flat-layout) entries served this process, so -# check_semantic_cache can report N to the user (#1939). _legacy_semantic_hits = 0 -# Prompt-file fingerprints already computed, keyed by (path, size, mtime_ns) — -# the same stat signature the hash index uses. check_semantic_cache resolves the -# prompt once per FILE in the corpus, so without this a 500-doc run re-reads and -# re-hashes the same spec 500 times (and warns 500 times when it is unreadable). -_prompt_fp_cache: dict[tuple, str] = {} - - -def prompt_fingerprint(prompt: "str | Path") -> str: - """Return a short stable fingerprint of an extraction prompt. - - ``prompt`` is either the prompt text itself (the Python extraction path owns - its system prompt, :func:`graphify.llm._extraction_system`) or a Path to the - prompt file an agent loaded (the skill path's - ``references/extraction-spec.md``). - - Line endings and trailing whitespace are normalized before hashing: the same - spec file checked out with CRLF on Windows must not fingerprint differently - from the LF checkout that wrote the cache, or every Windows run would look - like a prompt change and re-bill extraction. - """ - if isinstance(prompt, Path): - text = prompt.read_text(encoding="utf-8", errors="replace") - else: - text = prompt - normalized = "\n".join( - line.rstrip() for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - ).strip() - return hashlib.sha256(normalized.encode()).hexdigest()[:_PROMPT_FP_LEN] - - -def _resolve_prompt_fp(prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None) -> str | None: - """Fingerprint the caller's extraction prompt, or None when it supplied none. - - ``prompt`` is prompt TEXT; ``prompt_file`` is a path to a file CONTAINING the - prompt. They are separate parameters rather than one overloaded argument - because the skill-driven callers are markdown snippets an agent copies with a - path substituted in — passing that path as ``prompt`` would hash the path - string itself, yielding a fingerprint that is stable, plausible, and tracks - nothing about the prompt. A silent wrong fingerprint is the exact failure - class #1939 is about, so the two are not inferred from each other. - - Best-effort: an unreadable ``prompt_file`` falls back to the flat, unattributed - layout rather than failing the run — a cache is never worth aborting an - extraction over. It warns rather than falling back quietly, because that - fallback silently restores the very behavior this fixes, and the skill-side - caller substitutes this path by hand. - """ - memo_key = None - if prompt_file is not None: - prompt = Path(prompt_file) - try: - st = prompt.stat() - memo_key = (str(prompt), st.st_size, st.st_mtime_ns) - if memo_key in _prompt_fp_cache: - return _prompt_fp_cache[memo_key] - except OSError: - pass # unreadable — fall through to the warning below - if prompt is None: - return None - try: - fp = prompt_fingerprint(prompt) - if memo_key is not None: - _prompt_fp_cache[memo_key] = fp - return fp - except (OSError, UnicodeError) as exc: - warnings.warn( - f"could not read extraction prompt {str(prompt)!r} ({exc}); semantic cache " - "entries cannot be attributed to a prompt version and fall back to the " - "unversioned layout, so this run may replay entries from an older " - "extraction prompt (#1939).", - RuntimeWarning, - stacklevel=3, - ) - return None - - -# A frontmatter delimiter is a whole line of exactly three dashes (optional -# trailing whitespace). Substring checks like startswith("---") / -# find("\n---") also match `----` thematic breaks and `--- text` prose, -# silently dropping everything above them from the hash (#1259). -_FRONTMATTER_DELIM = re.compile(r"^---[ \t]*\r?$", re.MULTILINE) - def _body_content(content: bytes) -> bytes: - """Strip YAML frontmatter from Markdown content, returning only the body.""" + """Strip a complete Markdown YAML frontmatter block for stable hashing.""" text = content.decode(errors="replace") opener = _FRONTMATTER_DELIM.match(text) if opener is None: return content closer = _FRONTMATTER_DELIM.search(text, opener.end()) - if closer is None: - return content - # Slice right after the closing `---` (not after its line) so the output - # stays byte-identical with the historical implementation for well-formed - # frontmatter -- existing semantic-cache hashes must not churn. - return text[closer.start() + 3:].encode() - - -# Stat-based index: maps absolute path → {size, mtime_ns, hash}. -# Loaded once per process, flushed via atexit. Skips full file reads when -# size+mtime_ns are unchanged — same trade-off as make(1). -# Correctness risks: `touch` causes a harmless extra re-hash; same-size edits -# within NFS second-resolution mtime have a 1-second window (same as make). -# `graphify extract --force` / `graphify update --force` (or GRAPHIFY_FORCE=1) -# skip the cache reads and re-dispatch everything when needed (#1894). -_stat_index: dict[str, dict] = {} -_stat_index_root: Path | None = None -_stat_index_dirty: bool = False + return text[closer.start() + 3:].encode() if closer is not None else content -def _stat_index_file(root: Path) -> Path: - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - return base / "cache" / "stat-index.json" +def file_hash(path: Path, root: Path = Path(".")) -> str: + """Hash file content; Markdown metadata-only changes do not invalidate extraction.""" + content = Path(path).read_bytes() + if Path(path).suffix.lower() in {".md", ".mdx"}: + content = _body_content(content) + return hashlib.sha256(content).hexdigest() -def _ensure_stat_index(root: Path, cache_root: "Path | None" = None) -> None: - global _stat_index, _stat_index_root, _stat_index_dirty - if _stat_index_root is not None: - return - # The stat index only determines the cache FILE location (entry keys are - # absolute paths), so honoring an explicit cache_root keeps detect()'s - # word-count cache under the requested --out dir instead of polluting the - # scanned corpus with a stray graphify-out/ (#1747). - _stat_index_root = Path(cache_root if cache_root is not None else root).resolve() - p = _stat_index_file(_stat_index_root) - if p.exists(): - try: - _stat_index = json.loads(p.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - _stat_index = {} - else: - _stat_index = {} - atexit.register(_flush_stat_index) +def prompt_fingerprint(prompt: str | Path) -> str: + text = Path(prompt).read_text(encoding="utf-8", errors="replace") if isinstance(prompt, Path) else prompt + normalized = "\n".join( + line.rstrip() + for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ).strip() + return hashlib.sha256(normalized.encode()).hexdigest()[:12] -def _flush_stat_index() -> None: - global _stat_index_dirty, _stat_index_root - if not _stat_index_dirty or _stat_index_root is None: - return - p = _stat_index_file(_stat_index_root) - try: - p.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") +def _prompt_fp(prompt: str | Path | None, prompt_file: str | Path | None) -> str: + if prompt_file is not None: try: - os.write(fd, json.dumps(_stat_index, separators=(",", ":")).encode()) - os.close(fd) - os.replace(tmp, p) - except Exception: - try: - os.close(fd) - except OSError: - pass - try: - os.unlink(tmp) - except OSError: - pass - except OSError: - pass - _stat_index_dirty = False - - -def _normalize_path(path: Path) -> Path: - """Normalize path for consistent cache keys across Windows path spellings.""" - import sys - if sys.platform != "win32": - return path - s = str(path) - if s.startswith("\\\\?\\"): - s = s[4:] # strip extended-length prefix \\?\ - return Path(os.path.normcase(s)) - - -def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = None) -> str: - """SHA256 of file contents + path relative to root. - - Uses a stat-based fastpath (size + mtime_ns) to skip full reads when the - file hasn't changed. Falls through to full SHA256 on first encounter or - when stat changes. Index is flushed atomically at process exit. - - Using a relative path (not absolute) makes cache entries portable across - machines and checkout directories, so shared caches and CI work correctly. - Falls back to the resolved absolute path if the file is outside root. + return prompt_fingerprint(Path(prompt_file)) + except (OSError, UnicodeError) as exc: + warnings.warn( + f"could not read extraction prompt {str(prompt_file)!r} ({exc}); " + "falling back to unattributed semantic cache state", + RuntimeWarning, + stacklevel=3, + ) + return "unscoped" + return prompt_fingerprint(prompt) if prompt is not None else "unscoped" - For Markdown files (.md), only the body below the YAML frontmatter is hashed, - so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache. - """ - global _stat_index_dirty - p = _normalize_path(Path(path)) - root = _normalize_path(Path(root)) - if not p.is_file(): - raise IsADirectoryError(f"file_hash requires a file, got: {p}") - # The stat index is a cache artifact, so it must follow the cache location - # (cache_root), not the key-anchor root — otherwise it leaves a stray - # graphify-out/cache/stat-index.json inside the analyzed source tree even when - # the AST cache itself is redirected to CWD (#1774 completion). - _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() - abs_key = str(resolved) - # The salt is the path component that enters the digest (relative to root, or - # the absolute-path fallback). The stat-index memo MUST be keyed by it too: - # the same file hashed under two different roots yields two different digests - # (this happens within one `--out` run), and a memo keyed only by absolute - # path served whichever was computed first — making file_hash order-dependent - # and poisoning the persisted stat-index across runs (#1989). Store one digest - # per salt so alternating roots don't force re-reads. +def _relative(path: Path, root: Path) -> str: + # Do not resolve the source path itself: an in-root symlink must retain the + # caller-visible alias in portable durable state, not leak its target path. + absolute = Path(path).absolute() try: - salt = resolved.relative_to(Path(root).resolve()).as_posix().lower() + return absolute.relative_to(Path(root).absolute()).as_posix() except ValueError: - salt = resolved.as_posix().lower() - - st: "os.stat_result | None" = None - try: - st = p.stat() - entry = _stat_index.get(abs_key) - if (isinstance(entry, dict) - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - hashes = entry.get("hashes") - if isinstance(hashes, dict): - cached = hashes.get(salt) - if isinstance(cached, str): - return cached - # Legacy single-digest entries ("hash") don't record which salt - # produced them, so they are never trusted (#1989) — recompute once. - except OSError: - pass - - raw = p.read_bytes() - content = _body_content(raw) if p.suffix.lower() == ".md" else raw - h = hashlib.sha256() - h.update(content) - h.update(b"\x00") - h.update(salt.encode()) - digest = h.hexdigest() - - if st is not None: - entry = _stat_index.get(abs_key) - if (isinstance(entry, dict) - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - hashes = entry.get("hashes") - if not isinstance(hashes, dict): - hashes = {} - entry["hashes"] = hashes - hashes[salt] = digest # preserve a co-located word_count / other salts - entry.pop("hash", None) # retire the un-salted legacy digest - else: - _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, - "hashes": {salt: digest}} - _stat_index_dirty = True - - return digest - - -def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" = None) -> int: - """Word count with the same (size, mtime_ns) stat-fastpath cache as - :func:`file_hash`, persisted in the shared stat index. - - ``detect()`` counts words in every PDF/docx/text file to size the corpus, - which re-opens and re-parses every binary on each run — minutes on a large - docs corpus even when only a handful of files changed (#1656). This caches - the count against the file's stat signature so an unchanged file is counted - once and read from the index thereafter. ``compute(path)`` produces the - count on a miss. A file that can't be stat'd (e.g. a Windows long path the - index normalization can't reach) simply recomputes and isn't cached — - correct, just not accelerated. - """ - global _stat_index_dirty - p = _normalize_path(Path(path)) - root = _normalize_path(Path(root)) - _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) - st: "os.stat_result | None" = None - try: - st = p.stat() - entry = _stat_index.get(abs_key) - if (entry - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns - and "word_count" in entry): - return entry["word_count"] - except OSError: - pass - - wc = compute(Path(path)) - - if st is not None: - entry = _stat_index.get(abs_key) - if (entry - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - entry["word_count"] = wc # augment the existing hash entry in place - else: - _stat_index[abs_key] = { - "size": st.st_size, "mtime_ns": st.st_mtime_ns, "word_count": wc, - } - _stat_index_dirty = True - - return wc - - -def _relativize_source_files_in(payload: dict, root: Path) -> None: - """Mutate ``payload`` to rewrite absolute ``source_file`` fields as - forward-slash relative paths from ``root``. - - Mirror of :func:`graphify.watch._relativize_source_files` so cached - extraction fragments persist in portable form (#777). Already-relative - fields and out-of-root paths pass through unchanged. - - Only ``root`` is resolved — ``source_file`` itself is relativized - symbolically so in-root symlinks keep their original name rather than - pointing at the resolved target. Same reasoning as - :func:`graphify.detect._to_relative_for_storage`. - """ - try: - root_resolved = Path(root).resolve() - except OSError: - return - # raw_calls (#: Pascal/Delphi cross-file inherited-call resolution) carries - # source_file the same way nodes/edges/hyperedges do, so it needs the same - # portable-path treatment for cache entries to round-trip correctly across - # machines/checkout directories. + return absolute.as_posix() + + +def _cache_key( + path: Path, + root: Path, + kind: str, + prompt: str | Path | None, + prompt_file: str | Path | None, +) -> str: + scope = ( + f"v{_EXTRACTOR_VERSION}" + if kind == "ast" + else _prompt_fp(prompt, prompt_file) + ) + return f"{kind}:{scope}:{_relative(path, root)}" + + +def _drop_stale_ast_entries( + cache: dict[str, dict[str, Any]], path: Path, root: Path +) -> None: + suffix = ":" + _relative(path, root) + current_prefix = f"ast:v{_EXTRACTOR_VERSION}:" + for key in list(cache): + if key.startswith("ast:") and key.endswith(suffix) and not key.startswith(current_prefix): + del cache[key] + + +def _portable_result(result: dict[str, Any], root: Path) -> dict[str, Any]: + value = copy.deepcopy(result) for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): - for item in payload.get(bucket, []): - if not isinstance(item, dict): + for item in value.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): continue - source = item.get("source_file") - if not source: - continue - sp = Path(source) - if not sp.is_absolute(): - continue - try: - rel = os.path.relpath(sp, root_resolved) - except (ValueError, OSError): - continue # out-of-root (e.g. Windows cross-drive) - if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): - continue # escaped root — keep absolute - item["source_file"] = rel.replace(os.sep, "/") - + source = Path(str(item["source_file"])) + if source.is_absolute(): + item["source_file"] = _relative(source, root) + return value -def _absolutize_source_files_in(payload: dict, root: Path) -> None: - """Inverse of :func:`_relativize_source_files_in`. - Re-anchor relative ``source_file`` fields against ``root`` so callers - that load a cached fragment see the same absolute-path shape that a - fresh in-process extraction would produce. Legacy cache entries with - absolute ``source_file`` values pass through unchanged. - """ - try: - root_resolved = Path(root).resolve() - except OSError: - return +def _runtime_result(result: dict[str, Any], root: Path) -> dict[str, Any]: + """Restore the absolute source paths emitted by fresh AST extraction.""" + value = copy.deepcopy(result) for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): - for item in payload.get(bucket, []): - if not isinstance(item, dict): - continue - source = item.get("source_file") - if not source: - continue - sp = Path(source) - if sp.is_absolute(): - continue - try: - item["source_file"] = str(root_resolved / sp) - except (TypeError, OSError): + for item in value.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): continue + source = Path(str(item["source_file"])) + if not source.is_absolute(): + item["source_file"] = str((root / source).resolve()) + return value -def cache_dir(root: Path = Path("."), kind: str = "ast", - prompt_fp: str | None = None) -> Path: - """Returns the cache directory for ``kind`` - creates it if needed. +def _group_has_partial_marker(result: dict[str, Any]) -> bool: + return any( + isinstance(item, dict) and item.get("_partial") is True + for bucket in ("nodes", "edges", "hyperedges") + for item in result.get(bucket, []) + ) - kind is "ast", "semantic", or a mode-namespaced semantic kind such as - "semantic-deep" (#1894). Separate subdirectories prevent semantic cache - entries from overwriting AST cache entries for the same source_file (#582). - AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by - graphify version because they depend on extractor code, not just file - contents. Semantic entries are still NOT version-namespaced (re-extraction - costs LLM calls, #1252): they live in graphify-out/cache/semantic/, with - deep-mode entries beside them in graphify-out/cache/semantic-deep/. - - ``prompt_fp`` (semantic kinds only) adds a p{fingerprint}/ subdirectory so - entries are attributed to the extraction prompt that produced them (#1939). - Omitting it yields the historical flat layout, where entries of unknown - vintage live. - """ - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - d = base / "cache" / kind - if kind == "ast": - d = d / f"v{_EXTRACTOR_VERSION}" - _cleanup_stale_ast_entries(d.parent, d) - elif prompt_fp: - d = d / f"p{prompt_fp}" - d.mkdir(parents=True, exist_ok=True) - return d - - -def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", - cache_root: Path | None = None, prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, - allow_legacy: bool = True, - allow_partial: bool = False) -> dict | None: - """Return cached extraction for this file if hash matches, else None. - - Cache key: SHA256 of file contents. - Cache value: stored as graphify-out/cache/{kind}/{hash}.json (AST entries - under the per-version subdirectory, see :func:`cache_dir`). - - ``root`` anchors the content-hash key and source_file relativization (it - must stay the inferred common parent so keys remain portable). ``cache_root`` - decouples *where* the cache directory lives from that anchor — the cache is - an output and must not land inside a read-only/analyzed source tree (#1774). - When ``cache_root`` is None the location falls back to ``root`` (unchanged - behavior for existing callers). - - AST entries written by other graphify versions — including the legacy - flat cache/ layout (pre-0.5.3) and the unversioned cache/ast/ layout — - are deliberately not consulted: they were produced by a different - extractor and may be stale. - - ``prompt`` (semantic kinds) is the extraction prompt — text, or a Path to - the prompt file — that the caller is about to extract with. It selects the - p{fingerprint}/ namespace, so an entry produced by a different prompt is a - miss rather than a silent stale hit (#1939). When it is given and the - fingerprinted namespace misses, ``allow_legacy`` (default True) falls back - to a flat-layout entry: those predate fingerprinting, so their vintage is - unknowable — they are served rather than re-billed, and the hit is counted - so :func:`check_semantic_cache` can report N to the user. Callers that must - not mix vintages within one entry (see :func:`save_semantic_cache`'s - ``merge_existing``) pass allow_legacy=False. - Returns None if no cache entry or file has changed. - """ +def load_cached( + path: Path, + root: Path = Path("."), + kind: str = "ast", + cache_root: Path | None = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + allow_legacy: bool = True, + allow_partial: bool = False, + allow_stale: bool = False, + *, + cache: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any] | None: global _legacy_semantic_hits - location = cache_root if cache_root is not None else root - try: - h = file_hash(path, root, cache_root=cache_root) - except OSError: + del cache_root + if cache is None or not Path(path).is_file(): return None - prompt_fp = _resolve_prompt_fp(prompt, prompt_file) - entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" - legacy_hit = False - if prompt_fp and not entry.exists() and allow_legacy: - legacy = cache_dir(location, kind) / f"{h}.json" - if legacy.exists(): - entry, legacy_hit = legacy, True - if entry.exists(): - try: - result = json.loads(entry.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return None - # A ``partial`` entry was produced from a truncated LLM response and - # covers only part of the file's symbols. Serving it as authoritative - # would return the incomplete node set forever until the file is - # re-extracted. Treat it as a cache MISS (the normal read path) so the - # file is re-dispatched and retried. Self-heals: a later complete - # extraction overwrites the same content-hash key with a non-partial - # entry. ``allow_partial`` is the one exception — the merge_existing - # checkpoint peeks at a partial prev so it can accumulate a file's slices - # across chunks without losing the truncated one (it stays partial). - if not allow_partial and isinstance(result, dict) and result.get("partial"): - return None - if legacy_hit: + if kind == "ast": + _drop_stale_ast_entries(cache, Path(path), Path(root)) + key = _cache_key(Path(path), Path(root), kind, prompt, prompt_file) + entry = cache.get(key) + if ( + not isinstance(entry, dict) + and kind.startswith("semantic") + and (prompt is not None or prompt_file is not None) + and allow_legacy + ): + legacy_key = f"{kind}:unscoped:{_relative(Path(path), Path(root))}" + legacy = cache.get(legacy_key) + if isinstance(legacy, dict): + entry = legacy _legacy_semantic_hits += 1 - # Re-anchor relative source_file fields so callers see the same - # absolute-path shape that a fresh in-process extraction produces - # (#777). Legacy entries with absolute source_file pass through. - if isinstance(result, dict): - _absolutize_source_files_in(result, root) - return result - return None - - -def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast", - cache_root: Path | None = None, prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None) -> None: - """Save extraction result for this file. - - Stores as graphify-out/cache/{kind}/{hash}.json where hash = SHA256 of current file contents. - result should be a dict with 'nodes' and 'edges' lists. - - ``root`` anchors the content-hash key and source_file relativization; - ``cache_root`` (when given) is where the cache directory is written, decoupled - from ``root`` so the cache never lands inside the analyzed source tree (#1774). + if not isinstance(entry, dict) or ( + not allow_stale + and entry.get("content_hash") != file_hash(Path(path), Path(root)) + ): + return None + if entry.get("partial") and not allow_partial: + return None + result = entry.get("result") + return _runtime_result(result, Path(root)) if isinstance(result, dict) else None - ``prompt`` (semantic kinds) is the extraction prompt that produced ``result`` - — text, or a Path to the prompt file. It stamps the entry into the - p{fingerprint}/ namespace so a later run under a different prompt does not - replay it (#1939). Writes always land in the fingerprinted namespace when a - prompt is given: an entry of known vintage is never written back into the - flat unknown-vintage layout. - No-ops if `path` is not a regular file. Subagent-produced semantic fragments - occasionally carry a directory path in `source_file`; skipping them prevents - IsADirectoryError from aborting the whole batch. - """ - p = Path(path) - if not p.is_file(): +def save_cached( + path: Path, + result: dict[str, Any], + root: Path = Path("."), + kind: str = "ast", + cache_root: Path | None = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + *, + cache: dict[str, dict[str, Any]] | None = None, + partial: bool | None = None, +) -> None: + del cache_root + if cache is None or not Path(path).is_file(): return - # Relativize source_file fields against ``root`` before write so the - # cache file on disk is portable across machines and checkout - # directories (#777). The cache key is content-hashed so lookup is - # already path-independent; this fixes the embedded path leak. - # - # Serialize a relativized copy rather than mutating the caller's dict — - # downstream pipeline steps (notably extract.py's AST prefix remap, which - # looks up Path(source_file).resolve() in a prefix table) depend on the - # source_file field's original absolute form. Mutating the input here would - # silently break those remaps on the first extraction pass. - on_disk = result - if isinstance(result, dict) and any(result.get(k) for k in ("nodes", "edges", "hyperedges", "raw_calls")): - import copy as _copy - on_disk = _copy.deepcopy(result) - _relativize_source_files_in(on_disk, root) - h = file_hash(p, root, cache_root=cache_root) - location = cache_root if cache_root is not None else root - target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) - entry = target_dir / f"{h}.json" - fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") - try: - os.write(fd, json.dumps(on_disk).encode()) - os.close(fd) - try: - os.replace(tmp_path, entry) - except PermissionError: - # Windows: os.replace can fail with WinError 5 if the target is - # briefly locked. Fall back to copy-then-delete. - import shutil - shutil.copy2(tmp_path, entry) - os.unlink(tmp_path) - except Exception: - try: - os.close(fd) - except OSError: - pass - try: - os.unlink(tmp_path) - except OSError: - pass - raise - - -def cached_files(root: Path = Path(".")) -> set[str]: - """Return set of file hashes that have a valid cache entry (any kind).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" - hashes: set[str] = set() - # Legacy flat entries - if base.is_dir(): - hashes.update(p.stem for p in base.glob("*.json")) - # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, - # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds - # have per-prompt-fingerprint subdirs alongside pre-fingerprint flat entries - # (#1939). - for kind in ("ast", "semantic", "semantic-deep"): - d = base / kind - if d.is_dir(): - hashes.update(p.stem for p in d.glob("**/*.json")) - return hashes - - -def clear_cache(root: Path = Path(".")) -> None: - """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy - flat entries).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" - # Legacy flat entries - if base.is_dir(): - for f in base.glob("*.json"): - f.unlink() - # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, - # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds - # have per-prompt-fingerprint subdirs (#1939). - for kind in ("ast", "semantic", "semantic-deep"): - d = base / kind - if d.is_dir(): - for f in d.glob("**/*.json"): - f.unlink() + if kind == "ast": + _drop_stale_ast_entries(cache, Path(path), Path(root)) + key = _cache_key(Path(path), Path(root), kind, prompt, prompt_file) + portable = _portable_result(result, Path(root)) + cache[key] = { + "content_hash": file_hash(Path(path), Path(root)), + "kind": kind, + "prompt_fingerprint": _prompt_fp(prompt, prompt_file), + "partial": _group_has_partial_marker(portable) if partial is None else partial, + "result": portable, + } -def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: - """Remove orphaned semantic cache entries, returning the count pruned. +def cached_files(cache: dict[str, dict[str, Any]]) -> set[str]: + return {str(entry.get("content_hash")) for entry in cache.values() if entry.get("content_hash")} - The semantic cache is content-hash-keyed (``{file_hash}.json`` under - ``cache/semantic/``) and deliberately UNVERSIONED — entries are produced by - the LLM from file contents, so invalidating them on every release would - re-bill extraction. Because it is unversioned it is also never swept by the - AST version-cleanup, so every content change or file deletion leaves a - permanent orphan entry that accumulates unbounded. - This sweeps ``cache/semantic/*.json`` AND ``cache/semantic-deep/*.json`` - (the ``--mode deep`` namespace, #1894) and deletes any entry whose stem - (the content hash) is not in ``live_hashes`` — the hashes of the current - live document set. Both namespaces are pruned against the SAME live set: - liveness is content-based and mode-independent, so a hash that is live for - one namespace is live for both. Skipping the deep namespace would re-grow - the unbounded-orphan problem this function fixed (#1527). ``*.tmp`` - atomic-write temporaries are skipped, and only these directories are - touched (never ``cache/ast/**`` or anything else). The unversioned design - is preserved: we prune by liveness, not by version. +def clear_cache(cache: dict[str, dict[str, Any]]) -> None: + cache.clear() - The sweep recurses into the per-prompt-fingerprint subdirs (#1939) for the - same reason it covers the deep namespace: a glob that stopped at the top - level would leave every fingerprinted entry permanently unprunable. Entries - under a fingerprint other than the current one are pruned by liveness only, - never swept wholesale the way :func:`_cleanup_stale_ast_entries` sweeps old - AST versions — two hosts with different prompts (verbose vs compact - extraction-spec) can share one graphify-out/, and a wholesale sweep would - have each run delete the other's entries and re-bill extraction on every - alternation. Liveness keeps the total bounded by live docs × prompts seen. - Best-effort, mirroring :func:`_cleanup_stale_ast_entries`: each unlink is - wrapped in ``try/except OSError`` and a failure is ignored. The worst-case - failure mode is benign — a surviving orphan costs only one re-extraction of - one doc on a future run, never incorrect output. - """ - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - pruned = 0 - for kind in ("semantic", "semantic-deep"): - semantic_dir = base / "cache" / kind - if not semantic_dir.is_dir(): - continue - for entry in semantic_dir.glob("**/*.json"): - if entry.stem in live_hashes: - continue - try: - entry.unlink() - pruned += 1 - except OSError: - pass - return pruned +def cached_word_count( + path: Path, + root: Path, + compute: Callable[[Path], int], + cache_root: Path | None = None, +) -> int: + """Word counts are cheap and are not persisted outside the active generation.""" + del root, cache_root + return int(compute(path)) def check_semantic_cache( files: list[str], + cache: dict[str, dict[str, Any]], + *, root: Path = Path("."), mode: str | None = None, - prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, - cache_root: "Path | None" = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + allow_stale: bool = False, ) -> tuple[list[dict], list[dict], list[dict], list[str]]: - """Check semantic extraction cache for a list of absolute file paths. - - Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files). - Uncached files need Claude extraction; cached files are merged directly. - - ``mode`` selects the cache namespace: ``None`` (the default) reads - ``cache/semantic/`` — byte-identical to the historical behavior, so - existing callers that omit it (including older installed skill flows) - are unaffected. A non-None mode (e.g. ``"deep"``) reads - ``cache/semantic-{mode}/`` instead, so deep-mode results never shadow - (or get shadowed by) standard-mode entries for the same content (#1894). - - ``prompt`` is the extraction prompt this run will use for the uncached - files — the prompt text (Python path) or a Path to the prompt file the - agent loaded (skill path, ``references/extraction-spec.md``). Supplying it - restricts hits to entries produced by that same prompt, so an upgrade that - changed the prompt re-extracts instead of replaying the older vintage - (#1939). Entries written before fingerprinting existed still hit — their - vintage is unknowable and dropping them would re-bill a whole corpus — but - a warning reports how many were served. Omitting ``prompt`` keeps the - historical behavior for existing callers. - - ``cache_root`` decouples *where* the cache is read from the key-anchor - ``root``, mirroring :func:`load_cached` and :func:`save_semantic_cache` - (#1774 / #1990). With ``--out``, pass the corpus as ``root`` (so content-hash - keys and relative-path resolution stay anchored to the source tree) and the - output directory as ``cache_root``. Omitting it keeps ``root`` for both. - """ global _legacy_semantic_hits kind = "semantic" if mode is None else f"semantic-{mode}" - cached_nodes: list[dict] = [] - cached_edges: list[dict] = [] - cached_hyperedges: list[dict] = [] - uncached: list[str] = [] legacy_before = _legacy_semantic_hits - - for fpath in files: - p = Path(fpath) - if not p.is_absolute(): - p = Path(root) / p - result = load_cached(p, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file) - if result is not None: - cached_nodes.extend(result.get("nodes", [])) - cached_edges.extend(result.get("edges", [])) - cached_hyperedges.extend(result.get("hyperedges", [])) + nodes: list[dict] = [] + edges: list[dict] = [] + hyperedges: list[dict] = [] + uncached: list[str] = [] + for raw in files: + path = Path(raw) + if not path.is_absolute(): + path = Path(root) / path + result = load_cached( + path, + root, + kind=kind, + prompt=prompt, + prompt_file=prompt_file, + allow_stale=allow_stale, + cache=cache, + ) + if result is None: + uncached.append(raw) else: - uncached.append(fpath) - - legacy = _legacy_semantic_hits - legacy_before - if legacy: + nodes.extend(result.get("nodes", [])) + edges.extend(result.get("edges", [])) + hyperedges.extend(result.get("hyperedges", [])) + legacy_hits = _legacy_semantic_hits - legacy_before + if legacy_hits: warnings.warn( - f"{legacy} semantic cache entr{'y' if legacy == 1 else 'ies'} predate " - "extraction-prompt fingerprinting and were written by an unknown prompt " - "version; they were replayed as-is, so this graph may mix extraction " - "vintages. Re-run with --force (or GRAPHIFY_FORCE=1) to re-extract them " - "with the current prompt (#1939).", + f"{legacy_hits} semantic cache entries predate extraction-prompt " + "fingerprinting and were reused from unattributed durable state", RuntimeWarning, stacklevel=2, ) - - return cached_nodes, cached_edges, cached_hyperedges, uncached - - -def _group_has_partial_marker(group: dict) -> bool: - """True if any node/edge/hyperedge in a per-file group carries the internal - ``_partial`` truncation marker set by the adaptive-retry give-up sites. - - The marker rides the item dicts up through every chunk merge, so it reaches - ``save_semantic_cache`` on BOTH the incremental checkpoint path (llm.py) and - the final authoritative save (cli.py) without either caller having to thread - an extra argument — the final save would otherwise overwrite a checkpoint's - ``partial`` flag with a clean-looking entry. - """ - for bucket in ("nodes", "edges", "hyperedges"): - for item in group.get(bucket, []): - if isinstance(item, dict) and item.get("_partial"): - return True - return False + return nodes, edges, hyperedges, uncached def save_semantic_cache( @@ -810,231 +279,138 @@ def save_semantic_cache( edges: list[dict], hyperedges: list[dict] | None = None, root: Path = Path("."), + cache_root: Path | None = None, merge_existing: bool = False, allowed_source_files: Iterable[str | Path] | None = None, mode: str | None = None, - prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, partial_source_files: Iterable[str | Path] | None = None, - cache_root: "Path | None" = None, + *, + cache: dict[str, dict[str, Any]] | None = None, ) -> int: - """Save semantic extraction results to cache, keyed by source_file. - - Groups nodes and edges by source_file, then saves one cache entry per file - under cache/semantic/ (separate from AST entries in cache/ast/) to prevent - hash-key collisions (#582). - - ``mode`` selects the cache namespace, mirroring - :func:`check_semantic_cache`: ``None`` (the default) writes - ``cache/semantic/`` — byte-identical to the historical behavior for - existing callers that omit it — while a non-None mode (e.g. ``"deep"``) - writes ``cache/semantic-{mode}/`` so richer deep-mode results never - overwrite standard-mode entries and vice versa (#1894). - - When ``merge_existing`` is True, any already-cached entry for a file is - unioned with the new results before saving instead of being overwritten. - This lets callers checkpoint incrementally (e.g. once per chunk) without - dropping a prior slice of a large file that was split across chunks. - - When ``allowed_source_files`` is provided, only those files may be used as - cache-write keys. Semantic nodes can legitimately mention another corpus - file, but a model must not be able to replace that file's complete cache - entry unless the file was part of the current extraction batch (#1757). - - When ``partial_source_files`` is provided, entries for those files are - stamped ``partial: True`` — the extraction was truncated, so the entry is - incomplete and :func:`load_cached` must treat it as a miss. Partial-ness is - ALSO detected intrinsically from a ``_partial`` marker on any grouped item, - so the flag survives even when a caller (e.g. cli.py's final save) does not - pass ``partial_source_files``. - - ``prompt`` is the extraction prompt that produced these results — text, or - a Path to the prompt file. It stamps entries into the p{fingerprint}/ - namespace so a later run under a different prompt re-extracts rather than - replaying them (#1939). Pass the same prompt here as to - :func:`check_semantic_cache`, or the write lands in a namespace the next - read won't consult. - - ``cache_root`` decouples *where* the cache directory is written from the - source-key anchor ``root`` — mirroring the same split that :func:`load_cached` - and :func:`save_cached` already expose (#1774). When given, cache files land - under ``cache_root`` while ``source_file`` paths are still resolved and - relativized against ``root``. When omitted, ``root`` is used for both - purposes (unchanged behaviour for existing callers). This fixes checkpoints - and the final save going to the corpus tree instead of ``--out`` (#1990, - #1991). - - Returns the number of files cached. - """ - from collections import defaultdict - + del cache_root + if cache is None: + return 0 kind = "semantic" if mode is None else f"semantic-{mode}" - by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []}) - for n in nodes: - src = n.get("source_file", "") - if src: - by_file[src]["nodes"].append(n) - for e in edges: - src = e.get("source_file", "") - if src: - by_file[src]["edges"].append(e) - for h in (hyperedges or []): - src = h.get("source_file", "") - if src: - by_file[src]["hyperedges"].append(h) - - root_path = Path(root).resolve() - - def resolved_source_path(value: str | Path) -> Path: - path = Path(value) - if not path.is_absolute(): - path = root_path / path - try: - return path.resolve() - except (OSError, RuntimeError): - # Keep the cache write best-effort for inaccessible paths or a - # symlink loop emitted by an untrusted semantic result. - return Path(os.path.abspath(path)) - - allowed_paths = None - if allowed_source_files is not None: - allowed_paths = {resolved_source_path(path) for path in allowed_source_files} - - partial_paths = None - if partial_source_files is not None: - partial_paths = {resolved_source_path(path) for path in partial_source_files} - # A chunk that truncated to an EMPTY parse contributes no grouped items, - # so its file is absent from by_file and the write loop below would never - # stamp it partial — leaving a prior clean slice looking complete (#1950 - # empty-parse gap). Seed an empty group for each named partial file that - # isn't already present, so the loop merges its existing entry and stamps - # it partial. Keyed by the resolved path (deduped against present groups). - _present = {resolved_source_path(k) for k in by_file} - for _pp in partial_paths: - if _pp not in _present: - by_file[str(_pp)] # defaultdict: create an empty {nodes,edges,hyperedges} - - def group_skipped(fpath: str) -> bool: - """Mirror the write-loop skip condition for one source_file group.""" - p = resolved_source_path(fpath) - return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) - - # Dangling-reference pruning (#1916). A node group is skipped by the write - # loop below when its source_file is not a real file (ghost path) or is - # out-of-scope per the #1757 guard — but an edge/hyperedge in an ALLOWED - # group that references a node id from a skipped group used to be written - # verbatim, so on replay (check_semantic_cache) it dangled forever (the - # #1895 merged-result filter runs AFTER this checkpoint write and is - # bypassed entirely on replay). Compute the node ids that will be skipped - # and drop any to-be-written edge whose endpoint — or hyperedge whose - # member (whole-hyperedge drop, mirroring #1895) — references one. Gated - # on allowed_source_files so unscoped callers stay byte-identical. - if allowed_paths is not None: - skipped_ids: set = set() - written_ids: set = set() - for fpath, result in by_file.items(): - target = skipped_ids if group_skipped(fpath) else written_ids - for n in result["nodes"]: - nid = n.get("id") - if nid is None: - continue - try: - hash(nid) - except TypeError: - continue - target.add(nid) - # A duplicate-attribution node (defined in a skipped AND a written - # group) still reaches the cache — don't over-prune references to it. - skipped_ids -= written_ids - if skipped_ids: - - def edge_dangles(e: dict) -> bool: - try: - return e.get("source") in skipped_ids or e.get("target") in skipped_ids - except TypeError: - # Non-hashable endpoint from an untrusted result; leave it - # to build-time validation rather than fail the save. - return False - - def hyperedge_dangles(h: dict) -> bool: - try: - return bool(skipped_ids & set(h.get("nodes") or [])) - except TypeError: - return False - - for fpath, result in by_file.items(): - if group_skipped(fpath): - continue - result["edges"] = [e for e in result["edges"] if not edge_dangles(e)] - result["hyperedges"] = [ - h for h in result["hyperedges"] if not hyperedge_dangles(h) - ] - - saved = 0 - skipped_not_file = 0 - for fpath, result in by_file.items(): - p = resolved_source_path(fpath) - if p.is_file(): - if allowed_paths is not None and p not in allowed_paths: - warnings.warn( - "semantic cache skipped out-of-scope source_file " - f"{fpath!r}; the file was not dispatched for extraction", - RuntimeWarning, - stacklevel=2, - ) + allowed = { + _relative(Path(item) if Path(item).is_absolute() else Path(root) / item, Path(root)) + for item in allowed_source_files or [] + } + partial = { + _relative( + Path(item) if Path(item).is_absolute() else Path(root) / item, + Path(root), + ) + for item in partial_source_files or [] + } + grouped: dict[str, dict[str, list[dict]]] = {} + accepted_node_ids: set[Any] = set() + skipped_node_ids: set[Any] = set() + out_of_scope_sources: set[str] = set() + scoped = allowed_source_files is not None + + # Work out which node identities are actually eligible for a durable + # checkpoint before grouping edges. A duplicate ID remains eligible when at + # least one attribution belongs to an allowed file. + for item in nodes: + if not isinstance(item, dict) or not item.get("source_file"): + continue + source = str(item["source_file"]).replace("\\", "/") + absolute = Path(source) if Path(source).is_absolute() else Path(root) / source + relative = _relative(absolute, Path(root)) + accepted = absolute.is_file() and (not allowed or relative in allowed) + if accepted: + accepted_node_ids.add(item.get("id")) + else: + skipped_node_ids.add(item.get("id")) + if scoped and absolute.is_file() and allowed and relative not in allowed: + out_of_scope_sources.add(relative) + skipped_node_ids.difference_update(accepted_node_ids) + + for bucket, items in (("nodes", nodes), ("edges", edges), ("hyperedges", hyperedges or [])): + for item in items: + if not isinstance(item, dict) or not item.get("source_file"): continue - if merge_existing: - # allow_legacy=False: merging a pre-fingerprint entry into this - # write would fuse two prompt vintages inside a single entry and - # then stamp the result as current-vintage — the exact mixing - # #1939 is about, made unfixable because the entry now claims a - # prompt that only produced half of it. - # allow_partial=True: a file split into slices across chunks - # accumulates here; if an earlier slice truncated, keep its nodes - # in the union AND let the entry stay partial (the _partial - # markers ride through, so is_partial below re-detects it) rather - # than a later clean slice silently replacing it and promoting the - # half-file to complete. - prev = load_cached(p, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file, - allow_legacy=False, allow_partial=True) - _prev_partial = bool(prev.get("partial")) if prev else False - if prev: - result = { - "nodes": (prev.get("nodes", []) or []) + result["nodes"], - "edges": (prev.get("edges", []) or []) + result["edges"], - "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"], - } - else: - _prev_partial = False - # A file is partial if the caller named it, any of its grouped items - # carries the intrinsic ``_partial`` marker, OR the entry it merged - # onto was already partial (an empty-parse truncation leaves a - # ``partial: True`` entry with no item markers, so a later clean slice - # merging over it must NOT silently promote the half-file to complete - # — #1950). Copy so the caller's dict is never mutated. A genuine - # complete re-extraction (merge_existing=False) overwrites the - # content-hash key with a non-partial entry that then serves normally. - is_partial = ( - (partial_paths is not None and p in partial_paths) - or _group_has_partial_marker(result) - or _prev_partial + source = str(item["source_file"]).replace("\\", "/") + absolute = Path(source) if Path(source).is_absolute() else Path(root) / source + relative = _relative(absolute, Path(root)) + if allowed and relative not in allowed: + if scoped and absolute.is_file(): + out_of_scope_sources.add(relative) + continue + if scoped and bucket == "edges" and ( + item.get("source") in skipped_node_ids + or item.get("target") in skipped_node_ids + ): + continue + if scoped and bucket == "hyperedges" and any( + member in skipped_node_ids for member in item.get("nodes", []) + ): + continue + grouped.setdefault(relative, {"nodes": [], "edges": [], "hyperedges": []})[bucket].append(item) + if out_of_scope_sources: + warnings.warn( + "ignored semantic result with out-of-scope source_file " + + ", ".join(repr(source) for source in sorted(out_of_scope_sources)), + RuntimeWarning, + stacklevel=2, + ) + for relative in partial: + if not allowed or relative in allowed: + grouped.setdefault(relative, {"nodes": [], "edges": [], "hyperedges": []}) + saved = 0 + for relative, result in grouped.items(): + path = Path(root) / relative + if not path.is_file(): + continue + key = _cache_key(path, Path(root), kind, prompt, prompt_file) + previous_partial = bool(cache.get(key, {}).get("partial")) + if merge_existing: + previous = load_cached( + path, root, kind=kind, prompt=prompt, prompt_file=prompt_file, + allow_legacy=False, allow_partial=True, cache=cache, ) - if is_partial: - result = {**result, "partial": True} - save_cached(p, result, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file) - saved += 1 - else: - skipped_not_file += 1 - if skipped_not_file and skipped_not_file == len(by_file): + if previous: + for bucket in ("nodes", "edges", "hyperedges"): + result[bucket] = [*previous.get(bucket, []), *result[bucket]] + if relative in partial: + for bucket in result.values(): + for item in bucket: + item["_partial"] = True + save_cached( + path, result, root, kind=kind, prompt=prompt, + prompt_file=prompt_file, cache=cache, + partial=( + relative in partial + or _group_has_partial_marker(result) + or (merge_existing and previous_partial) + ), + ) + saved += 1 + if saved == 0 and any((nodes, edges, hyperedges or [], partial)): warnings.warn( - f"save_semantic_cache: all {skipped_not_file} source_file group(s) were " - "skipped because their paths do not resolve to real files. This usually " - "means ``root`` is anchored to the wrong directory (e.g. the --out " - "directory instead of the corpus root). Pass the corpus directory as " - "``root`` and the output directory as ``cache_root`` (#1991).", + "semantic cache did not persist any source-file entries; verify the corpus root", RuntimeWarning, stacklevel=2, ) return saved + + +def prune_semantic_cache( + cache: dict[str, dict[str, Any]], live_hashes: set[str] +) -> int: + doomed = [ + key for key, entry in cache.items() + if key.startswith("semantic") and entry.get("content_hash") not in live_hashes + ] + for key in doomed: + del cache[key] + return len(doomed) + + +__all__ = [ + "_body_content", "_group_has_partial_marker", "cached_files", "cached_word_count", "check_semantic_cache", + "clear_cache", "file_hash", "load_cached", "prompt_fingerprint", + "prune_semantic_cache", "save_cached", "save_semantic_cache", +] diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 181e7493b..ec1f2eeb8 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -2,7 +2,7 @@ """ callflow_html.py — Generate call-flow architecture HTML from graphify knowledge graph outputs. -Reads graph.json plus optional GRAPH_REPORT.md, .graphify_labels.json, and sections JSON, +Reads an embedded Helix generation plus optional report, labels, and sections, then produces a self-contained HTML file with: - Dark-themed CSS (fixed template) - Navigation bar from section list @@ -13,8 +13,8 @@ Usage: python3 -m graphify export callflow-html - python3 -m graphify export callflow-html /path/to/project/graphify-out/graph.json - python3 -m graphify export callflow-html --graph /path/to/graph.json --output docs/architecture.html + python3 -m graphify export callflow-html /path/to/project + python3 -m graphify export callflow-html --graph /path/to/graph.helix --output docs/architecture.html """ from __future__ import annotations @@ -136,7 +136,7 @@ def endpoint_id(value) -> str: def normalize_node(raw: dict, index: int) -> dict: - """Normalize a graphify node across common graph.json schema variants.""" + """Normalize a Graphify node for the presentation model.""" node = dict(raw) node_id = first_present( node, @@ -219,61 +219,47 @@ def normalize_edge(raw: dict, index: int) -> dict | None: return edge -def _node_link_payload(data: dict) -> tuple[list, list] | None: - """Read current graphify graph.json via NetworkX's node-link parser.""" - if not isinstance(data.get("nodes"), list): - return None - if not isinstance(data.get("links"), list) and not isinstance(data.get("edges"), list): - return None - - try: - from networkx.readwrite import json_graph - - try: - graph = json_graph.node_link_graph(data, edges="links") - except TypeError: - graph = json_graph.node_link_graph(data) - except Exception: - return None - - nodes = [] - for node_id, attrs in graph.nodes(data=True): - node = dict(attrs) - node["id"] = node_id - nodes.append(node) - - edges = [] - for index, (source, target, attrs) in enumerate(graph.edges(data=True), 1): - edge = dict(attrs) - edge["source"] = edge.get("_src", edge.get("source", source)) - edge["target"] = edge.get("_tgt", edge.get("target", target)) - edge.setdefault("id", f"edge_{index}") - edges.append(edge) - return nodes, edges - - def load_graph(path: str | Path) -> tuple: - """Load graph.json. Returns normalized (nodes, edges, hyperedges, metadata).""" - if path: - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(Path(path)) - except ValueError as exc: - raise SystemExit(f"ERROR: {exc}") from exc - data = read_json(path) - if not isinstance(data, dict): - raise SystemExit(f"ERROR: graph file must contain a JSON object: {path}") - - graph_block = data.get("graph") if isinstance(data.get("graph"), dict) else {} - meta_block = data.get("metadata") if isinstance(data.get("metadata"), dict) else {} - - node_link = _node_link_payload(data) - if node_link: - raw_nodes, raw_edges = node_link - else: - raw_nodes = first_list(data.get("nodes"), data.get("vertices"), graph_block.get("nodes"), graph_block.get("vertices")) - raw_edges = first_list(data.get("links"), data.get("edges"), graph_block.get("links"), graph_block.get("edges")) - hyperedges = first_list(data.get("hyperedges"), graph_block.get("hyperedges"), data.get("groups"), graph_block.get("groups")) + """Load one native generation into the call-flow presentation records.""" + from graphify.helix.model import edge_attributes, graphify_attributes + from graphify.helix.persistence import load_graph as load_helix_graph + from graphify.security import validate_store_path + + loaded = load_helix_graph(validate_store_path(path)) + graph = loaded.graph + community_records = [ + record for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + ] + membership = { + member: record["id"] + for record in community_records + for member in record.get("members", []) + } + community_names = {record["id"]: record.get("name") for record in community_records} + raw_nodes = [] + for node in graph.nodes(): + attrs = graphify_attributes(node.attributes) + cid = membership.get(node.id) + raw_nodes.append({ + "id": node.id, + **attrs, + "community": cid, + "community_name": community_names.get(cid), + }) + raw_edges = [ + { + "id": str(edge.id), + "source": edge.source, + "target": edge.target, + **edge_attributes(edge), + } + for edge in graph.edges() + ] + graph_block = dict(graph.attributes).get("graph", {}) + graph_block = graph_block if isinstance(graph_block, dict) else {} + hyperedges = list(graph_block.get("hyperedges", [])) + meta_block = dict(loaded.metadata) nodes = [normalize_node(n, i) for i, n in enumerate(raw_nodes) if isinstance(n, dict)] edges = [] @@ -286,33 +272,23 @@ def load_graph(path: str | Path) -> tuple: meta = dict(graph_block) meta.update(meta_block) + build_meta = loaded.state.get("build", {}) + build_meta = build_meta if isinstance(build_meta, dict) else {} for key in ("built_at_commit", "commit", "project_name", "repo", "repository", "language_breakdown"): - if data.get(key) and not meta.get(key): - meta[key] = data.get(key) + if build_meta.get(key) and not meta.get(key): + meta[key] = build_meta.get(key) if meta.get("commit") and not meta.get("built_at_commit"): meta["built_at_commit"] = meta["commit"] + meta["community_labels"] = { + str(record["id"]): str(record.get("name") or f"Community {record['id']}") + for record in community_records + } + return nodes, edges, hyperedges, meta -def load_labels(path: str | Path | None) -> dict: - """Load community labels from .graphify_labels.json, tolerating wrapper keys.""" - data = read_json(path, default={}) - if not isinstance(data, dict): - return {} - if isinstance(data.get("labels"), dict): - data = data["labels"] - if isinstance(data.get("communities"), dict): - data = data["communities"] - labels = {} - for key, value in data.items(): - if isinstance(value, dict): - value = first_present(value, "label", "name", "title", default=key) - labels[str(key)] = str(value) - return labels - - -def load_sections(path: str | Path | None) -> list: +def load_sections(path: str | Path) -> list: """Load section definitions from JSON file.""" data = read_json(path, default=[]) if isinstance(data, dict) and isinstance(data.get("sections"), list): @@ -418,22 +394,20 @@ def resolve_graphify_paths(args) -> dict: graphify_out = Path(args.graphify_out).expanduser() elif args.graph: graphify_out = Path(args.graph).expanduser().parent - elif (base / "graph.json").exists(): + elif (base / "graph.helix").is_dir(): graphify_out = base else: graphify_out = base / GRAPHIFY_OUT project_root = graphify_out.parent if graphify_out.name == GRAPHIFY_OUT_NAME else base - graph = Path(args.graph).expanduser() if args.graph else graphify_out / "graph.json" + graph = Path(args.graph).expanduser() if args.graph else graphify_out / "graph.helix" report = Path(args.report).expanduser() if args.report else graphify_out / "GRAPH_REPORT.md" - labels = Path(args.labels).expanduser() if args.labels else graphify_out / ".graphify_labels.json" sections = Path(args.sections).expanduser() if args.sections else None return { "base": project_root, "graphify_out": graphify_out, "graph": graph, "report": report, - "labels": labels, "sections": sections, } @@ -1518,7 +1492,6 @@ def __init__( graphify_out: str | Path | None = None, graph: str | Path | None = None, report: str | Path | None = None, - labels: str | Path | None = None, sections: str | Path | None = None, output: str | Path | None = None, lang: str = "auto", @@ -1531,7 +1504,6 @@ def __init__( self.graphify_out = str(graphify_out) if graphify_out is not None else None self.graph = str(graph) if graph is not None else None self.report = str(report) if report is not None else None - self.labels = str(labels) if labels is not None else None self.sections = str(sections) if sections is not None else None self.output = str(output) if output is not None else None self.lang = lang @@ -1582,7 +1554,6 @@ def write_callflow_html( graphify_out: str | Path | None = None, graph: str | Path | None = None, report: str | Path | None = None, - labels: str | Path | None = None, sections: str | Path | None = None, output: str | Path | None = None, lang: str = "auto", @@ -1598,7 +1569,6 @@ def write_callflow_html( graphify_out=graphify_out, graph=graph, report=report, - labels=labels, sections=sections, output=output, lang=lang, @@ -1612,12 +1582,12 @@ def write_callflow_html( if not paths["graph"].exists(): raise FileNotFoundError( f"graphify output not found: {paths['graph']}. " - "Run graphify first or pass --graph /path/to/graph.json." + "Run graphify first or pass --graph /path/to/graph.helix." ) # Load data nodes, edges, hyperedges, meta = load_graph(paths["graph"]) - labels = load_labels(paths["labels"]) + labels = dict(meta.get("community_labels", {})) lang = detect_lang(args.lang, nodes, labels) if paths["sections"]: sections = load_sections(paths["sections"]) @@ -1627,7 +1597,7 @@ def write_callflow_html( report_text = load_report(paths["report"]) if not nodes: - raise ValueError("graph.json contains 0 nodes") + raise ValueError("the active Helix generation contains 0 nodes") if len(sections) <= 1: raise ValueError("no sections defined") @@ -1637,7 +1607,7 @@ def write_callflow_html( node_ids = {node.get("id") for node in nodes} missing_endpoint_edges = [edge for edge in edges if edge.get("source") not in node_ids or edge.get("target") not in node_ids] if verbose and missing_endpoint_edges: - print(f"WARNING: {len(missing_endpoint_edges)} edges reference nodes not present in graph.json.", file=sys.stderr) + print(f"WARNING: {len(missing_endpoint_edges)} edges reference missing nodes.", file=sys.stderr) meta["project_name"] = infer_project_name(str(paths["graph"]), meta) meta["node_count"] = len(nodes) @@ -1985,9 +1955,8 @@ def main(): ) parser.add_argument("project", nargs="?", default=None, help="Project root or graphify output directory") parser.add_argument("--graphify-out", default=None, help="Path to graphify output directory") - parser.add_argument("--graph", default=None, help="Path to graph.json") + parser.add_argument("--graph", default=None, help="Path to graph.helix store") parser.add_argument("--report", default=None, help="Path to GRAPH_REPORT.md") - parser.add_argument("--labels", default=None, help="Path to .graphify_labels.json") parser.add_argument("--sections", default=None, help="Path to sections JSON file; auto-derived when omitted") parser.add_argument("--output", default=None, help="Output HTML path") parser.add_argument("--lang", default="auto", help="HTML language: auto, zh-CN, en, etc. (default: auto)") @@ -2003,7 +1972,6 @@ def main(): graphify_out=args.graphify_out, graph=args.graph, report=args.report, - labels=args.labels, sections=args.sections, output=args.output, lang=args.lang, diff --git a/graphify/cli.py b/graphify/cli.py index 91df09672..26a0e5b90 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1,27 +1,24 @@ -"""graphify command dispatch — every non-install subcommand. +"""Helix-only command dispatch for every non-install Graphify command.""" -Extracted verbatim from __main__.main(); __main__ now calls dispatch_command(cmd) -after the install/platform dispatch. Kept out of __main__ to shrink the CLI entry -module. The path-redirect (`graphify ` -> extract) re-enters via a lazy -import of main to avoid a cli<->__main__ import cycle. -""" from __future__ import annotations + import json import os import re import sys import time -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT from pathlib import Path +from typing import Any + +from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT _SEARCH_NUDGE = json.dumps({ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run ' - '`graphify query ""` before grepping raw files. Only grep ' - 'after graphify has oriented you, or to modify/debug specific lines.' + "MANDATORY: graphify-out/graph.helix exists. Run " + "`graphify query \"\"` before broad source searches." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" @@ -29,13 +26,8 @@ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' - 'before reading source files. Use: `graphify query ""` ' - '(scoped subgraph), `graphify explain ""`, or ' - '`graphify path "" ""`. Only read raw files after graphify has ' - 'oriented you, or to modify/debug specific lines. This rule applies to ' - 'subagents too — include it in every subagent prompt involving code ' - 'exploration.' + "MANDATORY: orient with graphify-out/graph.helix before broad source reads. " + "Use `graphify query`, `graphify explain`, or `graphify path`." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" @@ -43,267 +35,39 @@ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'graphify-out/graph.json exists but may be STALE for this file (the file ' - 'changed after the last build). Prefer `graphify query ""` for ' - 'orientation, and run `graphify update` to refresh the graph. Reading the ' - 'file directly is fine.' + "graphify-out/graph.helix may be stale for this file. Use `graphify query` " + "for orientation and run `graphify update`; reading the file is allowed." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" -# Strict-mode block (opt-in). Claude Code PreToolUse honors -# hookSpecificOutput.permissionDecision == "deny" and shows permissionDecisionReason -# to the model. Fires at most once per session (see _mark_session_denied) so it can -# never strand an agent: the very next read proceeds with the soft nudge. _READ_DENY = json.dumps({ "hookSpecificOutput": { "hookEventName": "PreToolUse", "permissionDecision": "deny", "permissionDecisionReason": ( - 'graphify strict mode: this project has a fresh knowledge graph that covers ' - 'this file. Run `graphify query ""` (or `graphify explain` / ' - '`graphify path`) FIRST to orient yourself, then re-issue this Read — it ' - 'will be allowed. This block fires at most once per session; reading raw ' - 'files to modify or debug specific lines is fine after one query. Apply the ' - 'same rule in any subagent prompt that explores code.' + "graphify strict mode: run `graphify query `, `graphify explain`, " + "or `graphify path` first, then retry this read. This blocks at most once " + "per session." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" _HOOK_SOURCE_EXTS = ( - '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', - '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', - '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', + ".py", ".js", ".cjs", ".ts", ".tsx", ".jsx", ".astro", ".vue", ".svelte", + ".go", ".rs", ".java", ".rb", ".c", ".h", ".cpp", ".hpp", ".cc", + ".cs", ".kt", ".swift", ".php", ".scala", ".lua", ".sh", ".md", + ".rst", ".txt", ".mdx", ) _GEMINI_NUDGE_TEXT = ( - 'graphify: knowledge graph at graphify-out/. For focused questions, run ' - '`graphify query ""` (scoped subgraph, usually much smaller than ' - 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' - 'for broad architecture context.' + "graphify: native knowledge graph at graphify-out/graph.helix. " + "Use `graphify query \"\"` before broad source searches." ) -def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") - - -def _stamped_manifest_files( - files_by_type: dict[str, list[str]], - sem_result: dict, - root: Path, - partial_source_files: "set[str] | None" = None, -) -> dict[str, list[str]]: - """Manifest-safe files dict: only stamp semantic files that actually - produced output (cache hit or fresh extraction). Files whose chunk failed - have no source_file entry in sem_result — leaving their semantic_hash - empty so detect_incremental re-queues them (#933). - - A file in ``partial_source_files`` DID produce output this run, but only a - truncated fragment of it, so it is excluded from stamping too — otherwise - detect_incremental would see it "done" and never re-dispatch it, leaving the - incomplete node set live forever on the warm-incremental path. Same #933 - mechanism: leave it unstamped and it is re-queued next run. - - Both sides of the membership test are resolved against the scan ``root`` - before comparing (#1897): node/edge/hyperedge ``source_file`` values are - root-relative on a fresh extraction while ``files_by_type`` entries are - absolute (from detect()), so a raw string comparison never matched and - every freshly-extracted semantic doc was dropped from the manifest. - Mirrors the #1890 path normalization in graphify.llm. - - Hyperedges are counted as output (#1920): a chunk whose only result for a - document is a hyperedge (3+ nodes sharing a concept) is valid output that - the semantic cache persists per-``source_file`` — omitting it here left the - doc unstamped, so detect_incremental re-queued it on every run. The stamping - condition mirrors the cache-write keying (a hyperedge carries its own - ``source_file``); do not derive it from member nodes. - """ - root = Path(root) - - def _resolve(value: str) -> Path: - p = Path(value) - if not p.is_absolute(): - p = root / p - try: - return p.resolve() - except (OSError, RuntimeError): - return p - - sem_extracted: set[Path] = set() - for coll in ("nodes", "edges", "hyperedges"): - for item in sem_result.get(coll, []): - sf = item.get("source_file", "") - if sf: - sem_extracted.add(_resolve(sf)) - partial_resolved = {_resolve(p) for p in (partial_source_files or set())} - sem_types = {"document", "paper", "image"} - return { - ftype: [ - f for f in flist - if ftype not in sem_types - or (_resolve(f) in sem_extracted and _resolve(f) not in partial_resolved) - ] - for ftype, flist in files_by_type.items() - } - - -def _stale_graph_sources( - graph_path: Path, - scan_root: Path, - seen_files: set[str], -) -> list[str]: - """Source files graph.json still references but the current scan no longer - contains (#1909). - - Incremental extract's prune set was historically derived from the manifest - alone (``manifest - corpus``), so a file that became EXCLUDED - (.graphifyignore/.gitignore/--exclude changed) without being listed in the - manifest kept its stale nodes in graph.json forever. Derive prune - candidates from the graph's own node ``source_file``s instead: anything - the graph references that the post-exclude detect corpus no longer - contains is stale, whether the file was deleted or newly excluded. - - Only IN-ROOT paths are candidates: out-of-root/absolute entries - (--include sources, symlinked external corpora) are never walked by - detect, so their absence from the corpus is not staleness evidence. - Relative entries are re-anchored against both the scan root and the - graph's own output root; only anchors that land inside the scan root - count. Since #1941 extracts always store source_file relative to the SCAN - root, so the scan-root anchor is the live one; the out-root anchor stays - for graphs written by <=0.9.16, which stored them relative to the OUT root - (e.g. ``../project/x.py``, #555/#1899). - ``seen_files`` must be the FULL detect output including unclassified - files, so nodes from walked-but-unsupported sources (e.g. introspected - Cargo.toml manifests) are not misread as stale. - """ - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except Exception: - return [] - if not isinstance(data, dict): - return [] - try: - root_res = scan_root.resolve() - except (OSError, RuntimeError): - root_res = scan_root - # /graphify-out/graph.json — relative source_files may be anchored here. - out_base = graph_path.parent.parent - try: - out_base = out_base.resolve() - except (OSError, RuntimeError): - pass - - def _within_root(p: Path) -> bool: - try: - p.relative_to(root_res) - return True - except ValueError: - pass - try: - p.resolve().relative_to(root_res) - return True - except (ValueError, OSError, RuntimeError): - return False - - def _in_seen(p: Path) -> bool: - if str(p) in seen_files: - return True - try: - return str(p.resolve()) in seen_files - except (OSError, RuntimeError): - return False - - stale: list[str] = [] - checked: set[str] = set() - for n in data.get("nodes", []): - if not isinstance(n, dict): - continue - sf = n.get("source_file") - if not sf or not isinstance(sf, str) or sf in checked: - continue - checked.add(sf) - if "://" in sf: - continue # remote/virtual source (e.g. Google Workspace), not a scanned path - p = Path(sf) - if p.is_absolute(): - candidates = [p] - else: - rel = sf.replace("\\", "/") - bases = [root_res] - if out_base != root_res: - bases.append(out_base) - candidates = [ - Path(os.path.normpath(str(base / rel))) for base in bases - ] - in_root = [c for c in candidates if _within_root(c)] - if not in_root: - continue # out-of-root under every anchor: never prune - if any(_in_seen(c) for c in in_root): - continue # still part of the scan corpus - stale.append(sf) - return stale - - -def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int: - """Drop nodes/edges/hyperedges owned by ``stale_sources`` from graph.json - in place. Returns the number of nodes removed. - - Used by the ``--no-cluster`` incremental early-exit: that path never runs - ``build_merge`` (it would raw-dump only the new chunks), so an - exclusion-only change must prune the existing raw graph directly or the - newly-excluded file's nodes survive forever (#1909). - ``stale_sources`` comes from :func:`_stale_graph_sources`, i.e. the - graph's own ``source_file`` spellings, so exact string matching is enough. - """ - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except Exception: - return 0 - if not isinstance(data, dict): - return 0 - stale = set(stale_sources) - links_key = "links" if "links" in data else "edges" - nodes = [n for n in data.get("nodes", []) if isinstance(n, dict)] - kept_nodes = [n for n in nodes if n.get("source_file") not in stale] - removed_ids = { - n.get("id") for n in nodes if n.get("source_file") in stale - } - n_removed = len(nodes) - len(kept_nodes) - kept_edges = [ - e for e in data.get(links_key, []) - if isinstance(e, dict) - and e.get("source_file") not in stale - and e.get("source") not in removed_ids - and e.get("target") not in removed_ids - ] - kept_hyper = [ - h for h in data.get("hyperedges", []) - if isinstance(h, dict) and h.get("source_file") not in stale - ] - if n_removed == 0 and len(kept_edges) == len(data.get(links_key, [])) and ( - len(kept_hyper) == len(data.get("hyperedges", [])) - ): - return 0 - data["nodes"] = kept_nodes - data[links_key] = kept_edges - if "hyperedges" in data: - data["hyperedges"] = kept_hyper - from graphify.export import backup_if_protected as _backup - _backup(graph_path.parent) - from graphify.paths import write_json_atomic - write_json_atomic(graph_path, data, indent=2) - return n_removed - - class _StageTimer: - """Print per-stage wall-clock timings to stderr when --timing is set (#1490). - - Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : - N.Ns`` after each stage and a final total. Off by default, so normal output is - byte-identical and machine-read stdout is untouched. - """ - def __init__(self, enabled: bool) -> None: - import time as _time - self._now = _time.perf_counter + import time + + self._now = time.perf_counter self.enabled = enabled self.start = self._now() self._last = self.start @@ -317,41 +81,81 @@ def mark(self, stage: str) -> None: def total(self) -> None: if self.enabled: print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) -def _enforce_graph_size_cap_or_exit(gp: Path) -> None: - """Reject oversized graph files before parsing (CLI exit-on-fail flavor). - - Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the - raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. - Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, - ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, - ``global_graph``, ``watch``, ``export``) call the security helper directly - and let the ``ValueError`` propagate. - """ - from graphify.security import check_graph_file_size_cap + + +def _default_graph_path() -> str: + return str(Path(_GRAPHIFY_OUT) / "graph.helix") + + +def _option(args: list[str], *names: str) -> str | None: + for index, value in enumerate(args): + for name in names: + if value == name and index + 1 < len(args): + return args[index + 1] + if value.startswith(name + "="): + return value.split("=", 1)[1] + return None + + +def _options(args: list[str], *names: str) -> list[str]: + values: list[str] = [] + index = 0 + while index < len(args): + value = args[index] + matched = False + for name in names: + if value == name and index + 1 < len(args): + values.append(args[index + 1]) + index += 2 + matched = True + break + if value.startswith(name + "="): + values.append(value.split("=", 1)[1]) + index += 1 + matched = True + break + if not matched: + index += 1 + return values + + +def _store_arg(args: list[str], *, default: str | Path | None = None) -> Path: + value = _option(args, "--store", "--graph") or str( + default or Path(_GRAPHIFY_OUT) / "graph.helix" + ) + path = Path(value).expanduser() + if path.suffix.lower() == ".json" or path.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; pass a graph.helix store and rebuild from source" + ) + return path + + +def _validate_store_or_exit(gp: Path) -> None: + """Validate a native store path for command-line callers.""" + from graphify.security import validate_store_path + try: - check_graph_file_size_cap(gp) - except ValueError as exc: + validate_store_path(gp) + except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) - sys.exit(1) + raise SystemExit(1) from exc + + def _hook_strict_enabled(flag: bool) -> bool: - """Resolve strict mode: GRAPHIFY_HOOK_STRICT env overrides the baked-in flag - (truthy forces on without a reinstall, falsy is the kill switch); unset defers - to the flag the installed hook command carried.""" - v = os.environ.get("GRAPHIFY_HOOK_STRICT", "").strip().lower() - if v in ("1", "true", "yes", "on"): + value = os.environ.get("GRAPHIFY_HOOK_STRICT", "").strip().lower() + if value in {"1", "true", "yes", "on"}: return True - if v in ("0", "false", "no", "off"): + if value in {"0", "false", "no", "off"}: return False return flag -def _touch_query_stamp(graph_path: "Path") -> None: - """Record that graphify oriented the agent recently, next to the queried graph. - The strict guard suppresses its block while this stamp is fresh. Fail-silent.""" +def _touch_query_stamp(graph_path: Path) -> None: try: from graphify.paths import write_text_atomic - stamp = Path(graph_path).parent / "cache" / "last_query_stamp" + + stamp = graph_path.parent / "cache" / "last_query_stamp" stamp.parent.mkdir(parents=True, exist_ok=True) write_text_atomic(stamp, str(time.time())) except Exception: @@ -359,3360 +163,1102 @@ def _touch_query_stamp(graph_path: "Path") -> None: def _query_stamp_fresh() -> bool: - """True if a query/explain/path ran within GRAPHIFY_HOOK_STRICT_TTL (default - 1800s) — recent orientation, so strict mode does not block this read.""" from graphify.paths import out_path + try: ttl = float(os.environ.get("GRAPHIFY_HOOK_STRICT_TTL", "1800")) - return (time.time() - out_path("cache", "last_query_stamp").stat().st_mtime) < ttl + return time.time() - out_path("cache", "last_query_stamp").stat().st_mtime < ttl except Exception: return False def _mark_session_denied(session_id: str) -> bool: - """Atomically claim a one-time strict block for this session. Returns True only - on the FIRST call for a given session id (O_EXCL create wins once); every later - call — or any error — returns False, so a session is blocked at most once and an - agent can never be stranded. Best-effort GC of markers older than 24h.""" from graphify.paths import out_path - sid = re.sub(r"[^A-Za-z0-9_-]", "_", str(session_id))[:64] - if not sid: + + safe_id = re.sub(r"[^A-Za-z0-9_-]", "_", str(session_id))[:64] + if not safe_id: return False try: - d = out_path("cache", "hook_sessions") - d.mkdir(parents=True, exist_ok=True) - fd = os.open(str(d / f"{sid}.denied"), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + directory = out_path("cache", "hook_sessions") + directory.mkdir(parents=True, exist_ok=True) + fd = os.open( + str(directory / f"{safe_id}.denied"), + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o644, + ) os.close(fd) - try: - cutoff = time.time() - 86400 - for entry in os.scandir(d): - try: - if entry.stat().st_mtime < cutoff: - os.unlink(entry.path) - except OSError: - pass - except OSError: - pass return True - except FileExistsError: + except (FileExistsError, OSError): return False + + +def _target_is_indexed(file_path: str, root: Path) -> bool: + from graphify.paths import out_path + + if not file_path: + return True + try: + manifest_path = out_path("manifest.json") + if manifest_path.stat().st_size > 2_000_000: + return True + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or not manifest: + return True + path = Path(file_path) + relative: set[str] = {path.name} + try: + relative.add(path.resolve().relative_to(root).as_posix()) + except (ValueError, OSError, RuntimeError): + pass + keys = {str(key).replace("\\", "/") for key in manifest} + absolute = str(path).replace("\\", "/") + return absolute in keys or any( + value and any(key == value or key.endswith("/" + value) for key in keys) + for value in relative + ) except Exception: - return False + return True def _run_hook_guard(kind: str, strict: bool = False) -> None: - """Shell-agnostic PreToolUse guard (#522). - - Reads the tool-call JSON from stdin and, when a fresh in-project knowledge graph - exists, nudges the agent to use graphify instead of grepping/reading raw files. - Replaces the old inline bash hooks that failed to parse on Windows. - - Fails open everywhere: any error, or a non-matching tool call, prints nothing - and the caller exits 0, so a legitimate tool call is never blocked by a bug. - - In strict mode (opt-in, Claude Code Read only) the FIRST raw read of indexed, - in-project, fresh code per session is DENIED with a redirect to `graphify query` - (permissionDecision), then downgrades to the soft nudge — it fires at most once - per session and can never strand the agent. Search (Bash) and Glob stay - nudge-only: a compound shell command has no single parseable target and blocking - file listing would strand navigation. #1840: reads of out-of-project files are - ignored, and a graph that is stale for the target file softens to a non-mandatory - nudge instead of blocking or demanding. - """ - from graphify.paths import out_path, GRAPHIFY_OUT_NAME - # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so - # the tool is never blocked; the graph nudge is appended only when a graph - # exists. Handled before the stdin read below (which the search/read guards need). + from graphify.paths import GRAPHIFY_OUT_NAME, out_path + if kind == "gemini": - payload = {"decision": "allow"} + payload: dict[str, Any] = {"decision": "allow"} try: - if out_path("graph.json").is_file(): + if out_path("graph.helix").is_dir(): payload["additionalContext"] = _GEMINI_NUDGE_TEXT except Exception: pass sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) return try: - d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) - except Exception: - return - if not isinstance(d, dict): - return - t = d.get("tool_input", d) - if not isinstance(t, dict): - return - try: + data = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) + if not isinstance(data, dict): + return + tool_input = data.get("tool_input", data) + if not isinstance(tool_input, dict): + return + store_path = out_path("graph.helix") + if not store_path.is_dir(): + return if kind == "search": - cmd_str = str(t.get("command", "") or "") - # Two input shapes reach this guard (matcher "Bash|Grep", #1986): - # the Bash tool carries `command`, while Claude Code's dedicated - # Grep tool carries `pattern` (plus optional path/glob) and no - # command — a Grep call IS a content search by definition, so it - # nudges whenever a graph exists. For Bash, keep matching the same - # set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ - # ack/ag as a token (name followed by a space). Nudge-only, even in - # strict mode — see the docstring. - is_grep_tool = not cmd_str and bool(t.get("pattern")) - is_bash_search = any(tok in cmd_str for tok in ( - "grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) - if (is_grep_tool or is_bash_search) and out_path("graph.json").is_file(): + command = str(tool_input.get("command", "")) + is_grep_tool = not command and bool(tool_input.get("pattern")) + is_bash_search = any( + token in command + for token in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ") + ) + if is_grep_tool or is_bash_search: sys.stdout.write(_SEARCH_NUDGE) - elif kind == "read": - vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] - j = " ".join(vals).lower().replace("\\", "/") - tails = [ - "." + seg.rsplit(".", 1)[-1] - for v in vals if v - for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] - if "." in seg - ] - under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j - if under_out or not any(tl in _HOOK_SOURCE_EXTS for tl in tails): + return + if kind != "read": + return + + values = [ + str(tool_input.get("file_path") or ""), + str(tool_input.get("pattern") or ""), + str(tool_input.get("path") or ""), + ] + joined = " ".join(values).lower().replace("\\", "/") + tails = [Path(value).suffix.lower() for value in values if value] + if GRAPHIFY_OUT_NAME.lower() + "/" in joined or not any( + extension in _HOOK_SOURCE_EXTS for extension in tails + ): + return + + root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) + try: + root = root.resolve() + except (OSError, RuntimeError): + pass + explicit = [ + str(tool_input.get(name) or "") + for name in ("file_path", "path") + if tool_input.get(name) + ] + if explicit: + in_project = False + for value in explicit: + path = Path(value) + if not path.is_absolute(): + in_project = True + break + try: + path.resolve().relative_to(root) + in_project = True + break + except (ValueError, OSError, RuntimeError): + continue + if not in_project: return - # #1840 (a): skip files outside the graph's project. cwd (or - # CLAUDE_PROJECT_DIR, which Claude Code sets) is the project root, since - # the guard only triggers when graph.json exists relative to cwd. A path - # candidate that resolves outside that root is out-of-project. - root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) - try: - root = root.resolve() - except (OSError, RuntimeError): - pass - path_vals = [str(t.get("file_path") or ""), str(t.get("path") or "")] - explicit = [v for v in path_vals if v] - if explicit: - in_project = False - for v in explicit: - p = Path(v) - if not p.is_absolute(): - in_project = True # relative -> anchored at cwd == in project - break - try: - p.resolve().relative_to(root) - in_project = True - break - except (ValueError, OSError, RuntimeError): - continue - if not in_project: - return - # One stat for existence + mtime of the graph. + + graph_mtime = store_path.stat().st_mtime + file_path = str(tool_input.get("file_path") or "") + stale = False + if file_path: try: - gmtime = os.stat(str(out_path("graph.json"))).st_mtime + stale = Path(file_path).stat().st_mtime > graph_mtime except OSError: - return - # #1840 (b): stale-for-target -> soften, never block. The target file - # changed after the last build, or watch flagged the tree. - stale = False - fp = str(t.get("file_path") or "") - if fp: - try: - stale = os.stat(fp).st_mtime > gmtime - except OSError: - stale = False - try: - if out_path("needs_update").exists(): - stale = True - except Exception: pass - if stale: - sys.stdout.write(_READ_NUDGE_STALE) - return - # Strict block: Read tool only, first time per session, not recently - # oriented, and the file is demonstrably indexed. - tool_name = d.get("tool_name") - if _hook_strict_enabled(strict) and tool_name in (None, "Read") \ - and not _query_stamp_fresh() \ - and _target_is_indexed(fp, root) \ - and _mark_session_denied(str(d.get("session_id") or "")): - sys.stdout.write(_READ_DENY) - return - sys.stdout.write(_READ_NUDGE) - except Exception: - pass - - -def _target_is_indexed(file_path: str, root: "Path") -> bool: - """Guard the strict deny: only block a read of a file the graph actually indexes. - Reads manifest.json (cheap, capped); on any doubt (missing/corrupt/oversized - manifest, unresolvable path) returns True so the once-per-session deny still - applies — that block is self-limiting, so erring toward it is safe.""" - from graphify.paths import out_path - if not file_path: - return True - try: - mp = out_path("manifest.json") - st = mp.stat() - if st.st_size > 2_000_000: - return True - manifest = json.loads(mp.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or not manifest: - return True - p = Path(file_path) - rels = set() - try: - rels.add(p.resolve().relative_to(root).as_posix()) - except (ValueError, OSError, RuntimeError): - pass - rels.add(p.name) - keys = {str(k).replace("\\", "/") for k in manifest} - abskey = str(p).replace("\\", "/") - return abskey in keys or any(r and (r in keys or any(k.endswith("/" + r) or k == r for k in keys)) for r in rels) + if out_path("needs_update").exists(): + stale = True + if stale: + sys.stdout.write(_READ_NUDGE_STALE) + return + if ( + _hook_strict_enabled(strict) + and data.get("tool_name") in (None, "Read") + and not _query_stamp_fresh() + and _target_is_indexed(file_path, root) + and _mark_session_denied(str(data.get("session_id") or "")) + ): + sys.stdout.write(_READ_DENY) + return + sys.stdout.write(_READ_NUDGE) except Exception: - return True -def _clone_repo( - url: str, branch: str | None = None, out_dir: Path | None = None -) -> Path: - """Clone a GitHub repo to a local cache dir and return the path. + return - Clones into ~/.graphify/repos// by default so repeated - runs on the same URL reuse the existing clone (git pull instead of clone). - """ - import subprocess as _sp - import re as _re - # Normalise URL — strip trailing .git if present - url = url.rstrip("/") - if not url.endswith(".git"): - git_url = url + ".git" - else: - git_url = url - url = url[:-4] - - # Extract owner/repo from URL - m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) - if not m: - print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) - sys.exit(1) - owner, repo = m.group(1), m.group(2) - - if out_dir: - dest = out_dir - else: - dest = Path.home() / ".graphify" / "repos" / owner / repo +def _clone_repo(url: str, branch: str | None = None, out_dir: Path | None = None) -> Path: + import re + import subprocess + clean = url.rstrip("/") + git_url = clean if clean.endswith(".git") else clean + ".git" + match = re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", clean) + if not match: + raise ValueError(f"not a recognized GitHub URL: {url}") + owner, repo = match.group(1), match.group(2) + destination = out_dir or Path.home() / ".graphify" / "repos" / owner / repo if branch and branch.startswith("-"): - print(f"error: invalid branch name: {branch!r}", file=sys.stderr) - sys.exit(1) - - if dest.exists(): - print(f"Repo already cloned at {dest} - pulling latest...", flush=True) - cmd = ["git", "-C", str(dest), "pull"] + raise ValueError(f"invalid branch name: {branch!r}") + if destination.exists(): + command = ["git", "-C", str(destination), "pull"] if branch: - cmd += ["origin", "--", branch] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + command += ["origin", "--", branch] else: - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"Cloning {url} -> {dest} ...", flush=True) - cmd = ["git", "clone", "--depth", "1"] + destination.parent.mkdir(parents=True, exist_ok=True) + command = ["git", "clone", "--depth", "1"] if branch: - cmd += ["--branch", branch] - cmd += ["--", git_url, str(dest)] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) - - print(f"Ready at: {dest}", flush=True) - return dest + command += ["--branch", branch] + command += ["--", git_url, str(destination)] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr.strip() or "git command failed") + print(f"Ready at: {destination}") + return destination -def _reenter_main() -> None: - from graphify.__main__ import main - main() +def _loaded(args: list[str]): + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path + return load_graph(validate_store_path(_store_arg(args))) -def dispatch_command(cmd: str) -> None: - if cmd == "provider": - from graphify.llm import _custom_providers_path, BACKENDS - import json as _json - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - global_path = _custom_providers_path(global_=True) - - if subcmd == "list": - global_path.parent.mkdir(parents=True, exist_ok=True) - existing: dict = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if not existing: - print("No custom providers registered.") - else: - for name in existing: - print(f" {name} ({existing[name].get('base_url', '')})") - - elif subcmd == "show": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider show ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - print(_json.dumps({name: existing[name]}, indent=2)) - - elif subcmd == "add": - args = sys.argv[3:] - name = args[0] if args and not args[0].startswith("-") else "" - if not name: - print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) - sys.exit(1) - if name in BACKENDS: - print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) - sys.exit(1) - base_url = "" - default_model = "" - env_key = "" - pricing_input = 0.0 - pricing_output = 0.0 - i = 1 - while i < len(args): - a = args[i] - if a == "--base-url" and i + 1 < len(args): - base_url = args[i + 1]; i += 2 - elif a.startswith("--base-url="): - base_url = a.split("=", 1)[1]; i += 1 - elif a == "--default-model" and i + 1 < len(args): - default_model = args[i + 1]; i += 2 - elif a.startswith("--default-model="): - default_model = a.split("=", 1)[1]; i += 1 - elif a == "--env-key" and i + 1 < len(args): - env_key = args[i + 1]; i += 2 - elif a.startswith("--env-key="): - env_key = a.split("=", 1)[1]; i += 1 - elif a == "--pricing-input" and i + 1 < len(args): - pricing_input = float(args[i + 1]); i += 2 - elif a == "--pricing-output" and i + 1 < len(args): - pricing_output = float(args[i + 1]); i += 2 - else: - i += 1 - if not base_url or not default_model or not env_key: - print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) - sys.exit(1) - from graphify.llm import provider_base_url_ok - if not provider_base_url_ok(base_url, name): - print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) - sys.exit(1) - global_path.parent.mkdir(parents=True, exist_ok=True) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - existing[name] = { - "base_url": base_url, - "default_model": default_model, - "env_key": env_key, - "pricing": {"input": pricing_input, "output": pricing_output}, - "temperature": 0, - } - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") - - elif subcmd == "remove": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider remove ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - del existing[name] - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' removed.") - - else: - print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) - if subcmd: - sys.exit(1) - elif cmd == "prs": - from graphify.prs import cmd_prs - cmd_prs(sys.argv[2:]) - elif cmd == "hook": - from graphify.hooks import ( - install as hook_install, - uninstall as hook_uninstall, - status as hook_status, - ) - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - print(hook_install(Path("."))) - elif subcmd == "uninstall": - print(hook_uninstall(Path("."))) - elif subcmd == "status": - print(hook_status(Path("."))) +def _communities(loaded) -> tuple[dict[int, list[Any]], dict[int, str], dict[int, float]]: + communities: dict[int, list[Any]] = {} + labels: dict[int, str] = {} + cohesion: dict[int, float] = {} + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): + continue + cid = record["id"] + communities[cid] = list(record.get("members", [])) + labels[cid] = str(record.get("name") or f"Community {cid}") + if isinstance(record.get("cohesion"), (int, float)): + cohesion[cid] = float(record["cohesion"]) + return communities, labels, cohesion + + +def _query(args: list[str]) -> None: + from graphify.serve import _query_graph_text + + question = args[0] if args and not args[0].startswith("-") else "" + if not question: + raise ValueError("query requires a question") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + budget = int(_option(args, "--budget") or 2000) + mode = "dfs" if "--dfs" in args else "bfs" + filters: list[str] = [] + for index, value in enumerate(args): + if value == "--context" and index + 1 < len(args): + filters.append(args[index + 1]) + elif value.startswith("--context="): + filters.append(value.split("=", 1)[1]) + print(_query_graph_text( + loaded.graph, + question, + native_query=loaded.query, + mode=mode, + depth=2, + token_budget=budget, + context_filters=filters, + learning_overlay=( + dict(learning.get("nodes", {})) + if isinstance((learning := loaded.state.get("learning", {})), dict) + else {} + ), + )) + + +def _path(args: list[str]) -> None: + from graphify.helix.model import edge_attributes, node_attributes + from graphify.serve import _pick_scored_endpoint, _query_terms, _score_nodes + + positional = [value for value in args if not value.startswith("-")] + if len(positional) < 2: + raise ValueError("path requires source and target") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + graph = loaded.graph + source_scores = _score_nodes( + graph, _query_terms(positional[0]), native_query=loaded.query + ) + target_scores = _score_nodes( + graph, _query_terms(positional[1]), native_query=loaded.query + ) + if not source_scores or not target_scores: + raise ValueError("source or target did not resolve uniquely") + source = _pick_scored_endpoint(graph, source_scores, positional[0]) + target = _pick_scored_endpoint(graph, target_scores, positional[1]) + result = graph.shortest_path(source, target, direction="both") + node_ids = getattr(result, "node_ids", ()) + if not node_ids: + print("No path found.") + raise SystemExit(0) + path = list(node_ids) + segments = [] + for index, (left, right) in enumerate(zip(path, path[1:])): + records = [ + graph.edge(edge_id) + for edge_id in graph.incident_edge_ids(left) + ] + hop_edges = [ + record + for record in records + if record is not None + and {record.source, record.target} == {left, right} + ] + if not hop_edges: + raise RuntimeError("native shortest path returned an edge-less hop") + forward_edges = [ + edge for edge in hop_edges + if edge.source == left and edge.target == right + ] + selected_edges = forward_edges or hop_edges + attributes = [edge_attributes(edge) for edge in selected_edges] + relations = sorted({ + str(value) + for attrs in attributes + if (value := attrs.get("relation")) + }) + relation = "/".join(relations) if relations else "related" + confidences = sorted({ + str(value) + for attrs in attributes + if (value := attrs.get("confidence")) + }) + confidence_text = f" [{'/'.join(confidences)}]" if confidences else "" + if index == 0: + segments.append(str(node_attributes(graph, left).get("label", left))) + label = str(node_attributes(graph, right).get("label", right)) + if forward_edges: + segments.append(f"--{relation}{confidence_text}--> {label}") else: - print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) - sys.exit(1) - elif cmd == "query": - if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.serve import _query_graph_text - from graphify.security import sanitize_label - from networkx.readwrite import json_graph - from graphify import querylog - - question = sys.argv[2] - use_dfs = "--dfs" in sys.argv - budget = 2000 - graph_path = _default_graph_path() - context_filters: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--budget" and i + 1 < len(args): - try: - budget = int(args[i + 1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--budget="): - try: - budget = int(args[i].split("=", 1)[1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--context" and i + 1 < len(args): - context_filters.append(args[i + 1]) - i += 2 - elif args[i].startswith("--context="): - context_filters.append(args[i].split("=", 1)[1]) - i += 1 - elif args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # `query` deliberately keeps the graph undirected (unlike `path` / - # `explain`, which force directed=True): BFS/DFS here must explore - # both callers and callees of the seed node to build useful - # context, and forcing a DiGraph would make G.neighbors() return - # successors only, silently dropping every caller-side result for - # a seed with no outgoing edges. Direction is instead preserved - # per-edge below (mirrors graphify/build.py's _src/_tgt pattern) - # so the *rendering* stays correct without narrowing traversal. - _raw = dict( - _raw, - links=[ - {**link, "_src": link.get("source"), "_tgt": link.get("target")} - for link in _raw.get("links", []) - ], + segments.append(f"<--{relation}{confidence_text}-- {label}") + print(f"Shortest path ({len(path) - 1} hops):\n " + " ".join(segments)) + + +def _explain(args: list[str]) -> None: + from graphify.helix.model import edge_attributes, node_attributes + from graphify.serve import _find_node + + query = next((value for value in args if not value.startswith("-")), "") + if not query: + raise ValueError("explain requires a node label or ID") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + graph = loaded.graph + matches = _find_node(graph, query, native_query=loaded.query) + if len(matches) > 1: + query_path = query.replace("\\", "/") + file_matches = [ + node_id for node_id in matches + if str(node_attributes(graph, node_id).get("source_file", "")).replace("\\", "/") == query_path + and str(node_attributes(graph, node_id).get("label", "")) == Path(query_path).name + ] + if len(file_matches) == 1: + matches = file_matches + if len(matches) != 1: + print(f"No unique node match for {query}") + return + node_id = matches[0] + attrs = node_attributes(graph, node_id) + print(f"Node: {attrs.get('label', node_id)}") + print(f"ID: {node_id}") + print(f"Source: {attrs.get('source_file', '-')} {attrs.get('source_location', '')}".rstrip()) + print(f"Type: {attrs.get('file_type', '-')}" ) + print(f"Degree: {graph.degree(node_id).degree}") + from graphify.reflect import load_learning_overlay + + entry = load_learning_overlay(loaded.store_path).get(str(node_id)) + if isinstance(entry, dict) and entry.get("status"): + status = entry["status"] + if status == "preferred": + lesson = ( + f"Lesson: preferred source (start here) — {entry.get('uses', 0)} useful, " + f"score={entry.get('score', 0)}" ) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - import time as _time - _t0 = _time.perf_counter() - _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - ) - querylog.log_query( - kind="query", - question=question, - corpus=str(gp), - result=_result, - mode=_mode, - depth=2, - token_budget=budget, - duration_ms=(_time.perf_counter() - _t0) * 1000, - ) - _touch_query_stamp(gp) - print(_result) - elif cmd == "affected": - if len(sys.argv) < 3: - print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph - query = sys.argv[2] - graph_path = _default_graph_path() - depth = 2 - relations: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--depth" and i + 1 < len(args): - try: - depth = int(args[i + 1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--depth="): - try: - depth = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--relation" and i + 1 < len(args): - relations.append(args[i + 1]) - i += 2 - elif args[i].startswith("--relation="): - relations.append(args[i].split("=", 1)[1]) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - graph = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - print( - format_affected( - graph, - query, - relations=relations or DEFAULT_AFFECTED_RELATIONS, - depth=depth, + elif status == "contested": + lesson = ( + f"Lesson: contested (useful {entry.get('uses', 0)} / " + f"dead-end {entry.get('neg', 0)})" ) - ) - elif cmd in ("god-nodes", "god_nodes"): - # god_nodes has long been an analyzer (analyze.py), an MCP tool, and a - # README-advertised capability, but never a CLI subcommand — `graphify - # god_nodes` fell through to "unknown command" (#2004). Wire it as a - # read-only graph query, mirroring `affected`. - from graphify.affected import load_graph - from graphify.analyze import god_nodes as _god_nodes - from graphify.security import sanitize_label as _sanitize_label - graph_path = _default_graph_path() - top_n = 10 - as_json = "--json" in sys.argv - args = sys.argv[2:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--top" and i + 1 < len(args): - try: - top_n = int(args[i + 1]) - except ValueError: - print("error: --top must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--top="): - try: - top_n = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --top must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - G = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - gods = _god_nodes(G, top_n=top_n) - if as_json: - print(json.dumps(gods, indent=2)) else: - print("God nodes (most connected):") - for rank, n in enumerate(gods, 1): - print(f" {rank}. {_sanitize_label(str(n['label']))} - {n['degree']} edges") - elif cmd == "save-result": - # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] - # [--outcome useful|dead_end|corrected] [--correction TEXT] - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify save-result") - p.add_argument("--question", required=True) - p.add_argument("--answer", default=None) - p.add_argument("--answer-file", dest="answer_file", default=None) - p.add_argument("--type", dest="query_type", default="query") - p.add_argument("--nodes", nargs="*", default=[]) - p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) - p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - opts = p.parse_args(sys.argv[2:]) - if opts.answer_file: - opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() - elif not opts.answer: - p.error("--answer or --answer-file is required") - from graphify.ingest import save_query_result as _sqr - - out = _sqr( - question=opts.question, - answer=opts.answer, - memory_dir=Path(opts.memory_dir), - query_type=opts.query_type, - source_nodes=opts.nodes or None, - outcome=opts.outcome, - correction=opts.correction, - ) - print(f"Saved to {out}") - elif cmd == "reflect": - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - p.add_argument( - "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), + lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" + if entry.get("stale"): + lesson += " [code changed since — re-verify]" + print(lesson) + connections: list[tuple[str, Any, dict[str, Any]]] = [] + for edge_id in graph.incident_edge_ids(node_id): + edge = graph.edge(edge_id) + if edge is None: + continue + neighbor = edge.target if edge.source == node_id else edge.source + connections.append(( + "out" if edge.source == node_id else "in", + neighbor, + edge_attributes(edge), + )) + if not connections: + return + + print(f"\nConnections ({len(connections)}):") + # Native incident-edge order is deterministic. Keep it as the tie-break so + # equal-degree callers retain source order, matching the established CLI. + connections.sort(key=lambda connection: -graph.degree(connection[1]).degree) + for direction, neighbor, edge_data in connections[:20]: + relation = edge_data.get("relation", "") + confidence = edge_data.get("confidence", "") + location = edge_data.get("source_location") or "" + source_file = edge_data.get("source_file") or "" + at = f" {source_file}:{location}" if location else "" + arrow = "-->" if direction == "out" else "<--" + print( + f" {arrow} {node_attributes(graph, neighbor).get('label', neighbor)} " + f"[{relation}] [{confidence}]{at}" ) - p.add_argument("--graph", default=None) - p.add_argument("--analysis", default=None) - p.add_argument("--labels", default=None) - p.add_argument("--half-life-days", type=float, default=30.0, - help="signal weight halves every N days (default 30)") - p.add_argument("--min-corroboration", type=int, default=2, - help="distinct useful results to promote a node to preferred (default 2)") - p.add_argument("--if-stale", action="store_true", - help="skip when LESSONS.md is already newer than every input " - "(e.g. the git hook just refreshed it)") - opts = p.parse_args(sys.argv[2:]) - from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh - - graph_arg = opts.graph - if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" - if default_graph.exists(): - graph_arg = str(default_graph) - - _gp = Path(graph_arg) if graph_arg else None - _analysis_path = None - _labels_path = None - if _gp is not None: - _analysis_path = Path(opts.analysis) if opts.analysis else ( - _gp.parent / ".graphify_analysis.json") - _labels_path = Path(opts.labels) if opts.labels else ( - _gp.parent / ".graphify_labels.json") - - if opts.if_stale and _lessons_fresh( - Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path - ): - print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") + if len(connections) <= 20: + return + + remainder = connections[20:] + print(f" ... and {len(remainder)} more") + by_file: dict[tuple[str, str], int] = {} + for direction, _neighbor, edge_data in remainder: + source_file = edge_data.get("source_file") or "(unknown file)" + key = (direction, str(source_file)) + by_file[key] = by_file.get(key, 0) + 1 + grouped = sorted(by_file.items(), key=lambda item: (-item[1], item[0])) + print(" Grouped by file:") + for (direction, source_file), count in grouped[:20]: + arrow = "-->" if direction == "out" else "<--" + noun = "connection" if count == 1 else "connections" + print(f" {arrow} {source_file}: {count} {noun}") + if len(grouped) > 20: + print(f" ... and {len(grouped) - 20} more files") + + +def _export(args: list[str]) -> None: + if not args: + raise ValueError("export requires a format") + kind = args[0] + tail = args[1:] + positional_store = tail[0] if tail and not tail[0].startswith("-") else None + store_args = ["--store", positional_store, *tail[1:]] if positional_store else tail + store_path = _store_arg(store_args) + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path + + loaded = load_graph(validate_store_path(store_path)) + graph = loaded.graph + communities, labels, cohesion = _communities(loaded) + output = _option(args, "--out", "--output") + from graphify import export + + if kind == "html": + html_output = output or str(Path(_GRAPHIFY_OUT) / "graph.html") + if "--no-viz" in args: + Path(html_output).unlink(missing_ok=True) else: - out_path, agg = _reflect( - memory_dir=Path(opts.memory_dir), - out_path=Path(opts.out), - graph_path=_gp, - analysis_path=_analysis_path, - labels_path=_labels_path, - half_life_days=opts.half_life_days, - min_corroboration=opts.min_corroboration, - ) - c = agg["counts"] - print( - f"Reflected {agg['total']} memories " - f"({c['useful']} useful, {c['dead_end']} dead ends, " - f"{c['corrected']} corrected) -> {out_path}" - ) - elif cmd == "path": - if len(sys.argv) < 4: - print( - 'Usage: graphify path "" "" [--graph path]', - file=sys.stderr, - ) - sys.exit(1) - from graphify.serve import _pick_scored_endpoint, _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx - - source_label = sys.argv[2] - target_label = sys.argv[3] - graph_path = _default_graph_path() - args = sys.argv[4:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee - # direction, and multigraph so exact-pair parallel links (e.g. a - # `references` and a `calls` edge between the same two nodes) survive load - # instead of being silently collapsed last-writer-wins — otherwise the - # printed relation could be one the traversed pair doesn't actually - # carry (#2074). Local to this read; serve's shared graph is untouched. - _raw = {**_raw, "directed": True, "multigraph": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) - if not src_scored: - print(f"No node matching '{source_label}' found.", file=sys.stderr) - sys.exit(1) - if not tgt_scored: - print(f"No node matching '{target_label}' found.", file=sys.stderr) - sys.exit(1) - src_nid = _pick_scored_endpoint(G, src_scored, source_label) - tgt_nid = _pick_scored_endpoint(G, tgt_scored, target_label) - # Ambiguity guard: when both queries resolve to the same node, the - # shortest path is trivially zero hops, which is almost never what the - # caller wanted (see bug #828). - if src_nid == tgt_nid: - print( - f"'{source_label}' and '{target_label}' both resolved to the same " - f"node '{src_nid}'. Use a more specific label or the exact node ID.", - file=sys.stderr, - ) - sys.exit(1) - for _name, _scored, _nid in ( - ("source", src_scored, src_nid), - ("target", tgt_scored, tgt_nid), - ): - # A close runner-up only made the resolution ambiguous when the raw - # score head is what got picked; a full-token override was chosen on - # token coverage, not score, so the head's margin is irrelevant. - if len(_scored) >= 2 and _nid == _scored[0][1]: - _top, _runner = _scored[0][0], _scored[1][0] - if _top > 0 and (_top - _runner) / _top < 0.10: - print( - f"warning: {_name} match was ambiguous " - f"(top score {_top:g}, runner-up {_runner:g})", - file=sys.stderr, - ) - # Deterministic shortest path (#2074): to_undirected(as_view=True) - # iterates neighbors via a hash-seeded set union, so among equal-length - # paths BFS returned an arbitrary route that varied per process. Build a - # sorted, materialized undirected graph so neighbor order — and thus the - # chosen path — is canonical for a given graph.json. - _und = _nx.Graph() - _und.add_nodes_from(sorted(G.nodes)) - _und.add_edges_from(sorted((min(u, v), max(u, v)) for u, v in G.edges())) - try: - path_nodes = _nx.shortest_path(_und, src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): - print(f"No path found between '{source_label}' and '{target_label}'.") - sys.exit(0) - hops = len(path_nodes) - 1 - segments = [] - from graphify.build import edge_datas - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - # Report the ACTUAL stored relation(s) of the traversed pair and - # direction — never a fabricated `calls` (#2074). A pair may carry - # several parallel relations; show all, and fall back to an honest - # "related" when the stored edge has no relation. - if G.has_edge(u, v): - datas = edge_datas(G, u, v) - forward = True - else: - datas = edge_datas(G, v, u) - forward = False - rels = sorted({d.get("relation") for d in datas if d.get("relation")}) - rel = "/".join(rels) if rels else "related" - confs = sorted({d.get("confidence") for d in datas if d.get("confidence")}) - conf_str = f" [{'/'.join(confs)}]" if confs else "" - if i == 0: - segments.append(G.nodes[u].get("label", u)) - if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") - print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) - from graphify import querylog - querylog.log_query( - kind="path", - question=f"{sys.argv[2]} -> {sys.argv[3]}", - corpus=str(gp), - nodes_returned=hops, + export.to_html(graph, communities, html_output, community_labels=labels) + elif kind == "graphml": + export.to_graphml(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.graphml")) + elif kind in {"cypher", "neo4j", "falkordb"}: + export.to_cypher(graph, output or str(Path(_GRAPHIFY_OUT) / "cypher.txt")) + elif kind == "svg": + export.to_svg(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.svg"), community_labels=labels) + elif kind == "obsidian": + count = export.to_obsidian( + graph, communities, output or str(Path(_GRAPHIFY_OUT) / "obsidian"), + community_labels=labels, cohesion=cohesion, ) - _touch_query_stamp(gp) - - elif cmd == "explain": - if len(sys.argv) < 3: - print('Usage: graphify explain "" [--graph path]', file=sys.stderr) - sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph - - label = sys.argv[2] - graph_path = _default_graph_path() - args = sys.argv[3:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - matches = _find_node(G, label) - if not matches: - print(f"No node matching '{label}' found.") - sys.exit(0) - nid = matches[0] - d = G.nodes[nid] - print(f"Node: {d.get('label', nid)}") - print(f" ID: {nid}") - print( - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() + print(f"Wrote {count} Obsidian notes.") + elif kind == "canvas": + export.to_canvas(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.canvas"), community_labels=labels) + elif kind == "wiki": + from graphify.wiki import to_wiki + + count = to_wiki( + graph, communities, output or str(Path(_GRAPHIFY_OUT) / "wiki"), + community_labels=labels, cohesion=cohesion, + god_nodes_data=list(loaded.state.get("analysis", {}).get("god_nodes", [])), ) - print(f" Type: {d.get('file_type', '')}") - print(f" Community: {d.get('community_name') or d.get('community', '')}") - # Work-memory overlay: a derived experiential hint from `graphify reflect`, - # merged in display-only from the .graphify_learning.json sidecar next to - # graph.json. No line when the node has no overlay entry. - try: - from graphify.reflect import load_learning_overlay as _llo - from graphify.security import sanitize_label as _sl - _overlay = _llo(gp) - _entry = _overlay.get(str(nid)) - if _entry: - _status = _sl(str(_entry.get("status", ""))) - if _status == "contested": - _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " - f"dead-end {_entry.get('neg', 0)})") - elif _status == "preferred": - _line = (f" Lesson: preferred source (start here) — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - else: - _line = (f" Lesson: {_status or 'tentative'} — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - if _entry.get("stale"): - _line += " [code changed since — re-verify]" - print(_line) - except Exception: - pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) - if connections: - print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - arrow = "-->" if direction == "out" else "<--" - # Append the edge's location — the actual call/import/reference - # SITE (in the caller's file for an incoming call), not a def - # line (#BUG1). Labeled by [rel] so the meaning is unambiguous. - loc = edata.get("source_location") or "" - sfile = edata.get("source_file") or "" - at = f" {sfile}:{loc}" if loc else "" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]{at}") - if len(connections) > 20: - remainder = connections[20:] - print(f" ... and {len(remainder)} more") - # #2009: a bare count silently hides the answer on high-degree - # nodes ("who calls this, what's the impact?"). Group the cut - # connections by direction + file so their shape is visible - # without falling back to a repo-wide grep. - by_file: dict[tuple[str, str], int] = {} - for direction, _nb, edata in remainder: - sfile = edata.get("source_file") or "(unknown file)" - key = (direction, sfile) - by_file[key] = by_file.get(key, 0) + 1 - # Count desc, then (direction, file) so equal-count groups have a - # byte-stable order (not the degree-derived insertion order). - grouped = sorted(by_file.items(), key=lambda kv: (-kv[1], kv[0])) - print(" Grouped by file:") - for (direction, sfile), count in grouped[:20]: - arrow = "-->" if direction == "out" else "<--" - noun = "connection" if count == 1 else "connections" - print(f" {arrow} {sfile}: {count} {noun}") - if len(grouped) > 20: - print(f" ... and {len(grouped) - 20} more files") - from graphify import querylog - querylog.log_query( - kind="explain", - question=sys.argv[2], - corpus=str(gp), - nodes_returned=len(connections), - ) - _touch_query_stamp(gp) + print(f"Wrote {count} wiki articles.") + elif kind == "callflow-html": + from graphify.callflow_html import write_callflow_html - elif cmd == "diagnose": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd != "multigraph": - print( - "Usage: graphify diagnose multigraph " - "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", - file=sys.stderr, - ) - sys.exit(1) - - graph_path = Path(_default_graph_path()) - max_examples = 5 - directed: bool | None = None - direction_flag: str | None = None - json_output = False - extract_path: Path | None = None - - i = 3 - while i < len(sys.argv): - arg = sys.argv[i] - if arg == "--graph": - i += 1 - if i >= len(sys.argv): - print("error: --graph requires a path", file=sys.stderr) - sys.exit(1) - graph_path = Path(sys.argv[i]) - elif arg == "--json": - json_output = True - elif arg == "--max-examples": - i += 1 - if i >= len(sys.argv): - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - try: - max_examples = int(sys.argv[i]) - except ValueError: - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - if max_examples < 0: - print("error: --max-examples must be >= 0", file=sys.stderr) - sys.exit(1) - elif arg == "--directed": - if direction_flag == "undirected": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "directed" - directed = True - elif arg == "--undirected": - if direction_flag == "directed": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "undirected" - directed = False - elif arg == "--extract-path": - i += 1 - if i >= len(sys.argv): - print("error: --extract-path requires a path", file=sys.stderr) - sys.exit(1) - extract_path = Path(sys.argv[i]) - else: - print(f"error: unknown diagnose option {arg}", file=sys.stderr) - sys.exit(1) - i += 1 - - from graphify.diagnostics import ( - diagnose_file, - format_diagnostic_json, - format_diagnostic_report, - ) + result = write_callflow_html(graph=store_path, output=output, verbose=True) + print(result) + else: + raise ValueError(f"unknown export format: {kind}") - try: - summary = diagnose_file( - graph_path, - directed=directed, - root=Path(".").resolve(), - max_examples=max_examples, - extract_path=extract_path, - ) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - if json_output: - print(json.dumps(format_diagnostic_json(summary), indent=2)) - else: - print(format_diagnostic_report(summary)) +def _global(args: list[str]) -> None: + from graphify.global_graph import global_add, global_list, global_path, global_remove - elif cmd == "add": - if len(sys.argv) < 3: - print( - "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", - file=sys.stderr, - ) - sys.exit(1) - from graphify.ingest import ingest as _ingest - - url = sys.argv[2] - author: str | None = None - contributor: str | None = None - target_dir = Path("raw") - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--author" and i + 1 < len(args): - author = args[i + 1] - i += 2 - elif args[i] == "--contributor" and i + 1 < len(args): - contributor = args[i + 1] - i += 2 - elif args[i] == "--dir" and i + 1 < len(args): - target_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - try: - saved = _ingest(url, target_dir, author=author, contributor=contributor) - print(f"Saved to {saved}") - print("Run /graphify --update in your AI assistant to update the graph.") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import watch as _watch + if not args or args[0] == "list": + repos = global_list() + if not repos: + print("No projects in the global graph.") + for tag, record in sorted(repos.items()): + print(f"{tag}: {record.get('node_count', 0)} nodes ({record.get('source_path', '')})") + return + if args[0] == "add" and len(args) >= 2: + source = Path(args[1]) + tag = _option(args[2:], "--as") or source.resolve().name + print(global_add(source, tag, retain_rollback="--retain-rollback" in args)) + return + if args[0] == "remove" and len(args) >= 2: + print( + f"Removed {global_remove(args[1], retain_rollback='--retain-rollback' in args)} " + "nodes." + ) + return + if args[0] == "path": + print(global_path()) + return + raise ValueError("usage: graphify global add [--as TAG] | remove TAG | list | path") - try: - _watch(watch_path) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd in ("cluster-only", "label"): - # `label` is `cluster-only` that always (re)generates community names with - # the configured backend, even when a .graphify_labels.json already exists. - force_relabel = cmd == "label" - # Mirror the tree/export arg-parsing pattern: walk argv so flags and - # the optional positional path can appear in any order (#724). - no_viz = "--no-viz" in sys.argv - no_label = "--no-label" in sys.argv - missing_only = "--missing-only" in sys.argv - co_timing = "--timing" in sys.argv - _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) - label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None - _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) - label_model = _model_arg.split("=", 1)[1] if _model_arg else None - _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) - min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 - args = sys.argv[2:] - watch_path: Path | None = None - graph_override: Path | None = None - co_resolution: float = 1.0 - co_exclude_hubs: float | None = None - label_max_concurrency: int = 4 - label_batch_size: int = 100 - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_override = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--backend" and i_arg + 1 < len(args): - label_backend = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--backend="): - label_backend = a.split("=", 1)[1]; i_arg += 1 - elif a == "--model" and i_arg + 1 < len(args): - label_model = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--model="): - label_model = a.split("=", 1)[1]; i_arg += 1 - elif a == "--resolution" and i_arg + 1 < len(args): - co_resolution = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--resolution="): - co_resolution = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--exclude-hubs" and i_arg + 1 < len(args): - co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--exclude-hubs="): - co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--max-concurrency" and i_arg + 1 < len(args): - label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--max-concurrency="): - label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 - elif a == "--batch-size" and i_arg + 1 < len(args): - label_batch_size = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--batch-size="): - label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): - i_arg += 1 - elif a.startswith("--"): - i_arg += 1 - elif watch_path is None: - watch_path = Path(a); i_arg += 1 - else: - i_arg += 1 - if watch_path is None: - watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all, remap_communities_to_previous - from graphify.analyze import ( - god_nodes, - surprising_connections, - suggest_questions, + +def _first_positional(args: list[str], *, value_options: set[str]) -> str | None: + """Return the first positional argument without mistaking option values for it.""" + index = 0 + while index < len(args): + value = args[index] + if value in value_options: + index += 2 + continue + if value.startswith("-"): + index += 1 + continue + return value + return None + + +def _activate_external_only( + result: dict, + *, + output_root: Path, + no_cluster: bool, + retain_rollback: bool, + source_name: str, +) -> None: + """Replace the active store with one transient external extraction DTO.""" + from graphify.analyze import god_nodes, suggest_questions, surprising_connections + from graphify.build import build_from_extraction, build_unclustered_extraction + from graphify.cluster import cluster, score_all + from graphify.helix.native import HELIX_PYTHON_VERSION + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.helix.state import community_records, community_summaries, new_state + from graphify.report import generate + from graphify.watch import _community_labels, _topology_sources, _warn_obsolete_store + + out = output_root.resolve() / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + _warn_obsolete_store(out) + store_path = out / "graph.helix" + build_data = ( + build_unclustered_extraction(result, root=output_root) + if no_cluster + else build_from_extraction(result, root=output_root) + ) + + def durable_state(graph, communities, cohesion, labels, analysis): + state = new_state() + state["build"] = { + "helix_python_version": HELIX_PYTHON_VERSION, + "node_count": graph.node_count, + "edge_count": graph.edge_count, + "directed": graph.directed, + "multigraph": graph.multigraph, + "semantic": False, + "source_root": source_name, + "source_commit": None, + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source="generated", ) - from graphify.report import generate - from graphify.export import to_json, to_html - - stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) - print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - stages.mark("load") - print("Re-clustering...") - communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) - # Mirror the watch/update path (#822): map new cids to prior ones by - # node-overlap so the existing .graphify_labels.json keeps attaching - # to the same conceptual community after re-clustering. Without this, - # labels follow raw cid index and become misaligned whenever the - # graph has changed between labeling and cluster-only (#1027). - previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None + state["analysis"] = analysis + state["incremental"] = { + "files": {}, + "extractor_state": {"mode": "postgres"}, + "topology_sources": _topology_sources(build_data), + "extraction_cache": {}, + # Database-only extraction replaces the corpus. It is intentionally + # not carried into a later filesystem extraction unless the caller + # explicitly supplies --postgres again. + "external_extractions": {}, + } + state["semantic"] = { + "used": False, + "backend": None, + "input_tokens": 0, + "output_tokens": 0, } - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - stages.mark("cluster") - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - stages.mark("analyze") - # Where outputs (GRAPH_REPORT.md, re-clustered graph.json, labels, - # analysis, html) land. When `--graph` points at a graph INSIDE a - # graphify-out/ dir (another project/tenant's output), write beside it, - # not into a stray graphify-out/ in the CWD (#1747). But when `--graph` - # points at an arbitrary path — e.g. a `backup/graph.json` archived - # before re-clustering (#934) — fall back to the CWD's graphify-out/, - # which is the restore-into-place workflow that test pins. The default - # (no --graph) case already has graph_json under watch_path/graphify-out. - _out_name = Path(_GRAPHIFY_OUT).name - if graph_override is not None and graph_json.parent.name == _out_name: - out = graph_json.parent + return state + + with HelixEmbeddedStore( + store_path, retain_rollback=retain_rollback + ) as store: + if no_cluster: + state = durable_state(build_data, {}, {}, {}, {}) + store.save_generation(build_data, state) + node_count = build_data.node_count + edge_count = build_data.edge_count else: - out = watch_path / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - labels_path = out / ".graphify_labels.json" - existing_labels: dict[int, str] = {} - if labels_path.exists(): - try: - existing_labels = { - int(k): v - for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) + with store.staged_graph(build_data) as staged: + graph = staged.graph + communities = cluster(graph) + cohesion = score_all(graph, communities) + labels = _community_labels(graph, communities) + gods = god_nodes(graph) + surprises = surprising_connections(graph, communities) + questions = suggest_questions(graph, communities, labels) + analysis = { + "god_nodes": gods, + "surprises": surprises, + "suggested_questions": questions, + "confidence_counts": { + confidence: sum( + 1 + for edge in build_data.edges + if str(edge.attributes.get("confidence", "EXTRACTED")) + == confidence + ) + for confidence in ("EXTRACTED", "INFERRED", "AMBIGUOUS") + }, + "community_summaries": community_summaries( + graph, communities, labels + ), + "report_inputs": { + "detection": { + "total_files": 0, + "total_words": 0, + "warning": "PostgreSQL schema introspection; no filesystem corpus.", + }, + "tokens": {"input": 0, "output": 0}, + "source": source_name, + }, } - except Exception: - existing_labels = {} - # Accumulate token usage from the labeling LLM calls so cluster-only mode - # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on - # the reuse / no-label paths, which make no LLM calls. - label_token_usage = {"input": 0, "output": 0} - # #2073: a --no-label run produces only "Community N" placeholders. - # Persisting them (plus a matching .sig) made the reuse branch treat them - # as fresh forever, permanently blocking real labeling on later runs. - placeholder_only = False - if labels_path.exists() and not force_relabel: - # Reuse saved labels, but don't blindly trust them: the graph may have - # been re-scoped/re-clustered since labeling, in which case a cid now - # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). - # Validate each community against the membership signature saved beside the - # labels; any community that changed (or has no saved label) is renamed by - # its current hub — deterministic and correct-by-construction — and the user - # is told to `graphify label` for fresh LLM names. Unchanged communities keep - # their saved label. When no signature sidecar exists (labels predate this), - # fall back to hub-filling only the communities missing a label. - from graphify.cluster import community_member_sigs, label_communities_by_hub - sig_path = labels_path.parent / (labels_path.name + ".sig") - saved_sigs: dict[int, str] = {} - if sig_path.exists(): - try: - saved_sigs = { - int(k): v for k, v in - json.loads(sig_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - saved_sigs = {} - cur_sigs = community_member_sigs(communities) - count_mismatch = len(existing_labels) != len(communities) - labels = {} - hub_labels: dict[int, str] | None = None - changed = 0 - for cid in communities: - # A persisted "Community {cid}" is a placeholder, not an earned - # label — treat it as absent so the hub labeler replaces it and an - # already-polluted sidecar (e.g. from a prior --no-label run) heals - # instead of suppressing real labels forever (#2073). - have_label = ( - cid in existing_labels - and existing_labels[cid] != f"Community {cid}" + state = durable_state( + graph, communities, cohesion, labels, analysis ) - if saved_sigs: - # Precise: the membership signature tells us if this exact - # community changed since it was labeled. - fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) - else: - # No signature sidecar (labels predate it). A differing community - # COUNT means the labels describe a different clustering, so a cid's - # old label can't be trusted; equal count is the best "same" signal. - fresh = have_label and not count_mismatch - if fresh: - labels[cid] = existing_labels[cid] - else: - if hub_labels is None: - hub_labels = label_communities_by_hub(G, communities) - labels[cid] = hub_labels[cid] - if have_label: - changed += 1 - if changed: - print( - f"[graphify] community set changed since labeling " - f"({len(existing_labels)} saved labels, {len(communities)} communities now; " - f"renamed {changed} community(ies) by their hub). " - f"Run `graphify label` to refresh names with the LLM.", - file=sys.stderr, + del graph + activated = store.activate_staged(staged, state) + graph = activated.graph + report = generate( + graph, + communities, + cohesion, + labels, + gods, + surprises, + analysis["report_inputs"]["detection"], + {"input": 0, "output": 0}, + source_name, + suggested_questions=questions, ) - elif no_label and not force_relabel: - labels = {cid: f"Community {cid}" for cid in communities} - placeholder_only = True - else: - # No labels file yet (or `graphify label` forced a refresh). When run - # standalone there is no orchestrating agent to do skill.md Step 5, so - # auto-name communities rather than leave "Community N" (#1097). - from graphify.cluster import label_communities_by_hub - from graphify.llm import generate_community_labels - print("Labeling communities...") - # Deterministic, LLM-free base labels: name each community after its - # highest-degree hub, so the report is readable even with no backend - # (previously bare "Community N"). A configured LLM backend overrides these - # with richer names below; its no-backend placeholder fallback does NOT. - hub_labels = label_communities_by_hub(G, communities) - label_communities_input = communities - labels = dict(hub_labels) - if missing_only: - labels = { - cid: existing_labels.get(cid, hub_labels[cid]) - for cid in communities - } - label_communities_input = { - cid: members - for cid, members in communities.items() - if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" - } - generated_labels, _ = generate_community_labels( - G, label_communities_input, backend=label_backend, model=label_model, gods=gods, - max_concurrency=label_max_concurrency, batch_size=label_batch_size, - usage_out=label_token_usage, - ) - # Only let the LLM OVERRIDE where it produced a real name — its no-backend - # fallback returns "Community {cid}" placeholders, which must not clobber - # the deterministic hub labels. - labels.update({ - cid: v for cid, v in generated_labels.items() - if v and v != f"Community {cid}" - }) - stages.mark("label") - questions = suggest_questions(G, communities, labels) - tokens = label_token_usage - from graphify.export import _git_head as _gh - _commit = _gh() - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, - {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions, - min_community_size=min_community_size, built_at_commit=_commit, - learning=_llfr(out / "graph.json")) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - stages.mark("report") - from graphify.export import backup_if_protected as _backup - _backup(out) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "questions": questions, - } - (out / ".graphify_analysis.json").write_text( - json.dumps(analysis, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - to_json(G, communities, str(out / "graph.json"), community_labels=labels) - # Don't persist placeholder-only labels (or their .sig): leaving the - # sidecar absent lets a later run generate real labels instead of reading - # back "Community N" as authoritative (#2073). - if not placeholder_only: - from graphify.paths import write_json_atomic as _wja - _wja(labels_path, {str(k): v for k, v in labels.items()}, ensure_ascii=False) - # Membership signatures beside the labels so a later cluster-only can - # detect which communities changed and avoid reusing a stale label - # (see reuse above). - from graphify.cluster import community_member_sigs as _cms - (labels_path.parent / (labels_path.name + ".sig")).write_text( - json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") - - # Mirror watch.py pattern: gate to_html so core outputs (graph.json + - # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise - # fall back to ValueError handling so an oversized graph doesn't crash - # the CLI mid-write and leave a stale graph.html on disk. - html_target = out / "graph.html" - if no_viz: - if html_target.exists(): - html_target.unlink() - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") - else: - try: - # Over-cap fallback (#1019): force the community-aggregation - # path so an oversized graph still renders a usable graph.html. - _node_limit = 5000 if _over_cap else None - to_html(G, communities, str(html_target), community_labels=labels or None, - node_limit=_node_limit) - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") - except ValueError as viz_err: - if html_target.exists(): - html_target.unlink() - print(f"Skipped graph.html: {viz_err}") - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") - - elif cmd == "update": - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - no_cluster = False - args = sys.argv[2:] - watch_arg: str | None = None - for a in args: - if a == "--force": - force = True - continue - if a == "--no-cluster": - no_cluster = True - continue - if a.startswith("-"): - print(f"error: unknown update option: {a}", file=sys.stderr) - sys.exit(2) - if watch_arg is not None: - print("error: update accepts at most one path argument", file=sys.stderr) - sys.exit(2) - watch_arg = a - - if watch_arg is not None: - watch_path = Path(watch_arg) - else: - # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import _rebuild_code - - print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - # Interactive CLI: block on the per-repo lock rather than skip, so the - # user sees their explicit `graphify update` complete instead of - # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) - if ok: - print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") - else: - print( - "Nothing to update or rebuild failed — check output above.", - file=sys.stderr, - ) - sys.exit(1) - - elif cmd == "hook-check": - # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. - # Keep this as a cross-platform no-op so installed hooks never break Bash - # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. - sys.exit(0) - elif cmd == "hook-guard": - # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old - # inline-bash hooks that failed on Windows. Prints an additionalContext nudge - # toward graphify when a fresh in-project graph exists; always exits 0. In - # strict mode (opt-in, `hook-guard read --strict`) it blocks the first raw - # read per session via the JSON permissionDecision payload — never via exit - # code — and downgrades to the nudge thereafter. - _run_hook_guard( - sys.argv[2] if len(sys.argv) > 2 else "", - strict="--strict" in sys.argv[3:], + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") + node_count = graph.node_count + edge_count = graph.edge_count + + with HelixEmbeddedStore(store_path, read_only=True) as reopened: + counts = reopened.verify_counts() + if counts["nodes"] != node_count or counts["edges"] != edge_count: + raise RuntimeError("activated PostgreSQL Helix generation failed verification") + # A database-only graph has no filesystem extraction root. Leaving a root + # marker here would make the next explicit source extraction resolve its + # cached paths against the command's old working directory. + (out / ".graphify_root").unlink(missing_ok=True) + print( + f"[graphify extract] PostgreSQL: {node_count} nodes, {edge_count} edges -> " + f"{store_path}" + ) + + +def _update_or_extract(cmd: str, args: list[str]) -> None: + from graphify.watch import _rebuild_code, _write_build_config + + if cmd == "extract" and not args: + raise ValueError( + "Usage: graphify extract [--code-only] " + "[--out DIR|--output DIR]" ) - sys.exit(0) - elif cmd == "check-update": - if len(sys.argv) < 3: - print("Usage: graphify check-update ", file=sys.stderr) - sys.exit(1) - from graphify.watch import check_update - - check_update(Path(sys.argv[2]).resolve()) - sys.exit(0) - elif cmd == "tree": - # Emit a D3 v7 collapsible-tree HTML view of graph.json: - # expand-all / collapse-all / reset-view buttons, multi-line - # wrapText labels with separately-coloured name + count, - # depth-based palette, click-to-toggle subtree, hover inspector - # showing top-K outbound edges per symbol. - from typing import Optional as _Opt - from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - output_path: "_Opt[Path]" = None - root: "_Opt[str]" = None - max_children = DEFAULT_MAX_CHILDREN - top_k_edges = 0 - project_label: "_Opt[str]" = None - args = sys.argv[2:] - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--output" and i_arg + 1 < len(args): - output_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--root" and i_arg + 1 < len(args): - root = args[i_arg + 1]; i_arg += 2 - elif a == "--max-children" and i_arg + 1 < len(args): - max_children = int(args[i_arg + 1]); i_arg += 2 - elif a == "--top-k-edges" and i_arg + 1 < len(args): - top_k_edges = int(args[i_arg + 1]); i_arg += 2 - elif a == "--label" and i_arg + 1 < len(args): - project_label = args[i_arg + 1]; i_arg += 2 - elif a in ("-h", "--help"): - print("Usage: graphify tree [--graph PATH] [--output HTML]") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") - print(" --root PATH filesystem root (default: longest common dir of all source_files)") - print(" --max-children N cap visible children per node (default 200)") - print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") - print(" --label NAME project label shown in the page header") - return - else: - i_arg += 1 - if not graph_path.is_file(): - print(f"error: graph.json not found at {graph_path}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(graph_path) - if output_path is None: - output_path = graph_path.parent / "GRAPH_TREE.html" - out = write_tree_html( - graph_path=graph_path, output_path=output_path, - root=root, max_children=max_children, - top_k_edges=top_k_edges, project_label=project_label, + timer = _StageTimer("--timing" in args) + postgres_dsn = _option(args, "--postgres") + positional = _first_positional( + args, + value_options={ + "--backend", "--model", "--mode", "--out", "--output", + "--exclude", "--token-budget", "--max-concurrency", "--changed", + "--postgres", "--max-workers", "--api-timeout", "--resolution", + "--exclude-hubs", "--as", + }, + ) + target = Path(positional or ".") + force = "--force" in args + no_cluster = "--no-cluster" in args + changed: list[Path] | None = None + if cmd == "update" and _option(args, "--changed"): + changed = [Path(value) for value in str(_option(args, "--changed")).split(",")] + include_semantic = cmd == "extract" and "--code-only" not in args + output_value = _option(args, "--out", "--output") + output = Path(output_value) if output_value else None + explicit_excludes = _options(args, "--exclude") + if "--no-gitignore" in args or explicit_excludes: + config_root = output if output is not None else target + _write_build_config( + config_root / _GRAPHIFY_OUT, + excludes=explicit_excludes or None, + gitignore=False if "--no-gitignore" in args else None, ) - size_kb = out.stat().st_size / 1024 - print(f"wrote {out} ({size_kb:.1f} KB)") - print(f"open with: xdg-open {out} (or file://{out.resolve()})") - sys.exit(0) - - elif cmd == "merge-driver": - # git merge driver for graph.json — takes (base, current, other) and writes - # the union of current+other nodes/edges back to current. Exits 1 on - # corrupt input so git surfaces the conflict instead of silently - # accepting a poisoned merge (see F-005). - # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) - if len(sys.argv) < 5: - print("Usage: graphify merge-driver ", file=sys.stderr) - sys.exit(1) - _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] - # Hard caps so a malicious or corrupted graph.json cannot exhaust memory - # at parse time. 50 MB / 100k nodes are well above any realistic graph - # (typical graphs are <5 MB / <50k nodes); anything larger should fail - # the merge so a human can investigate. - _MERGE_MAX_BYTES = 50 * 1024 * 1024 - _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data - try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) - except Exception as exc: - print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) - sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: - print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " - f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", - file=sys.stderr, - ) - sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - from graphify.paths import write_json_atomic - write_json_atomic(_current_path, out_data, indent=2) - sys.exit(0) - - elif cmd == "merge-graphs": - # graphify merge-graphs graph1.json graph2.json ... --out merged.json - args = sys.argv[2:] - graph_paths: list[Path] = [] - out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" - i = 0 - while i < len(args): - if args[i] == "--out" and i + 1 < len(args): - out_path = Path(args[i + 1]) - i += 2 - else: - graph_paths.append(Path(args[i])) - i += 1 - if len(graph_paths) < 2: - print( - "Usage: graphify merge-graphs [...] [--out merged.json]", - file=sys.stderr, - ) - sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags - graphs = [] - for gp in graph_paths: - if not gp.exists(): - print(f"error: not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) + try: + external_extraction = None + if postgres_dsn is not None: + if cmd != "extract": + raise ValueError("--postgres is supported by graphify extract only") + from graphify.pg_introspect import introspect_postgres + + print("[graphify extract] introspecting PostgreSQL schema...") try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - # Unique repo tag per graph. The bare `graphify-out/..` dir name is not - # unique across inputs (src/graphify-out and frontend/src/graphify-out both - # → "src"), which collides same-stem node ids and silently merges unrelated - # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. - repo_tags = _repo_tags(graph_paths) - naive_tags = [gp.parent.parent.name for gp in graph_paths] - if len(set(naive_tags)) != len(naive_tags): - print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = _nx.Graph() - for G, repo_tag in zip(graphs, repo_tags): - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - out_path.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path, out_data, indent=2) - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") - print(f"Written to: {out_path}") - - elif cmd == "clone": - if len(sys.argv) < 3: - print( - "Usage: graphify clone [--branch ] [--out ]", - file=sys.stderr, - ) - sys.exit(1) - url = sys.argv[2] - branch: str | None = None - out_dir: Path | None = None - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--branch" and i + 1 < len(args): - branch = args[i + 1] - i += 2 - elif args[i] == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - local_path = _clone_repo(url, branch=branch, out_dir=out_dir) - print(local_path) - - elif cmd == "export": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): - print("Usage: graphify export ", file=sys.stderr) - print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) - print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) - print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) - print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - sys.exit(1) - - # Parse shared args - args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" - labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" - report_path_explicit = False - sections_path: Path | None = None - callflow_output: Path | None = None - callflow_lang = "auto" - callflow_max_sections = 15 - callflow_diagram_scale = 1.0 - callflow_max_diagram_nodes = 18 - callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" - node_limit = 5000 - no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" - # Shared push-connection settings for the graph-database sinks (neo4j, - # falkordb), parsed from the generic --push/--user/--password flags below. - push_uri: str | None = None - push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it - # F-031: prefer an env var so the password never appears on argv (visible - # in `ps` output / shell history). The explicit --password flag still - # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, - # NEO4J_PASSWORD otherwise. - push_password: str | None = ( - os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" - else os.environ.get("NEO4J_PASSWORD") - ) or None - i = 0 - while i < len(args): - a = args[i] - if a == "--graph" and i + 1 < len(args): - graph_path = Path(args[i + 1]) - graph_path_explicit = True - i += 2 - elif a == "--labels" and i + 1 < len(args): - labels_path = Path(args[i + 1]) - labels_path_explicit = True - i += 2 - elif a == "--report" and i + 1 < len(args): - report_path = Path(args[i + 1]) - report_path_explicit = True - i += 2 - elif a == "--sections" and i + 1 < len(args): - sections_path = Path(args[i + 1]); i += 2 - elif a == "--output" and i + 1 < len(args): - callflow_output = Path(args[i + 1]).expanduser() - if not callflow_output.is_absolute(): - callflow_output = Path.cwd() / callflow_output - i += 2 - elif a == "--lang" and i + 1 < len(args): - callflow_lang = args[i + 1]; i += 2 - elif a == "--max-sections" and i + 1 < len(args): - callflow_max_sections = int(args[i + 1]); i += 2 - elif a == "--diagram-scale" and i + 1 < len(args): - callflow_diagram_scale = float(args[i + 1]); i += 2 - elif a == "--max-diagram-nodes" and i + 1 < len(args): - callflow_max_diagram_nodes = int(args[i + 1]); i += 2 - elif a == "--max-diagram-edges" and i + 1 < len(args): - callflow_max_diagram_edges = int(args[i + 1]); i += 2 - elif a in ("-h", "--help") and subcmd == "callflow-html": - print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") - print(" --report PATH path to GRAPH_REPORT.md") - print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") - print(" --lang LANG auto, zh-CN, en, etc. (default auto)") - print(" --max-sections N maximum auto-derived sections (default 15)") - print(" --diagram-scale N Mermaid diagram scale (default 1.0)") - print(" --max-diagram-nodes N representative nodes per section (default 18)") - print(" --max-diagram-edges N representative edges per section (default 24)") - sys.exit(0) - elif a == "--node-limit" and i + 1 < len(args): - node_limit = int(args[i + 1]); i += 2 - elif a == "--no-viz": - no_viz = True; i += 1 - elif a == "--dir" and i + 1 < len(args): - obsidian_dir = Path(args[i + 1]); i += 2 - elif a == "--push" and i + 1 < len(args): - push_uri = args[i + 1]; i += 2 - elif a == "--user" and i + 1 < len(args): - push_user = args[i + 1]; i += 2 - elif a == "--password" and i + 1 < len(args): - push_password = args[i + 1]; i += 2 - elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: - candidate = Path(a) - if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": - graph_path = candidate - elif (candidate / "graph.json").exists(): - graph_path = candidate / "graph.json" - else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" - graph_path_explicit = True - i += 1 - else: - i += 1 - - graph_path = graph_path.expanduser() - if graph_path_explicit: - graph_out_dir = graph_path.parent - if not labels_path_explicit: - labels_path = graph_out_dir / ".graphify_labels.json" - if not report_path_explicit: - report_path = graph_out_dir / "GRAPH_REPORT.md" - labels_path = labels_path.expanduser() - report_path = report_path.expanduser() - - if not graph_path.exists(): - print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) - sys.exit(1) - - if subcmd == "callflow-html": - from graphify.callflow_html import write_callflow_html as _write_callflow_html - out = _write_callflow_html( - graph=graph_path, - report=report_path, - labels=labels_path, - sections=sections_path, - output=callflow_output, - lang=callflow_lang, - max_sections=callflow_max_sections, - diagram_scale=callflow_diagram_scale, - max_diagram_nodes=callflow_max_diagram_nodes, - max_diagram_edges=callflow_max_diagram_edges, - verbose=True, - ) - print(f"callflow HTML written - open in any browser: {out}") - sys.exit(0) - - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap - - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) - - # Load optional analysis/labels - communities: dict[int, list[str]] = {} - if analysis_path.exists(): - _an = json.loads(analysis_path.read_text(encoding="utf-8")) - communities = {int(k): v for k, v in _an.get("communities", {}).items()} - cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} - gods_data = _an.get("gods", []) - else: - cohesion = {} - gods_data = [] - - # Fallback: graph.json carries the per-node community as a node attribute - # (`to_json` writes it on every node). The analysis sidecar is the - # canonical source — but the post-commit / watch rebuild path doesn't - # regenerate it, and `extract` may have its temp files cleaned up. When - # that happens, `graphify export html` previously bailed with - # "Single community - aggregated view not useful." even though the - # per-node attribute had the right data all along. Reconstruct from - # the graph itself so downstream subcommands (html, obsidian, wiki, - # svg, graphml, neo4j) don't silently produce a degraded artifact. - if not communities: - reconstructed: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid_raw = data.get("community") - if cid_raw is None: - continue - try: - cid = int(cid_raw) - except (TypeError, ValueError): - continue - reconstructed.setdefault(cid, []).append(str(node_id)) - if reconstructed: - communities = reconstructed - - labels: dict[int, str] = {} - if labels_path.exists(): - labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - - out_dir = graph_path.parent - - if subcmd == "html": - from graphify.export import to_html as _to_html - if no_viz: - html_target = out_dir / "graph.html" - if html_target.exists(): - html_target.unlink() - print("--no-viz: skipped graph.html") - else: - # Over-cap fallback (#1019): force the community-aggregation - # path so the oversized graph still renders a usable artifact. - _effective_node_limit = 5000 if _over_cap else node_limit - _to_html(G, communities, str(out_dir / "graph.html"), - community_labels=labels or None, node_limit=_effective_node_limit) - if G.number_of_nodes() <= _effective_node_limit: - print(f"graph.html written - open in any browser, no server needed") - if _over_cap: - sys.exit(0) - - elif subcmd == "obsidian": - from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas - n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) - print(f"Obsidian vault: {n} notes in {obsidian_dir}/") - _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), - community_labels=labels or None) - print(f"Canvas: {obsidian_dir}/graph.canvas") - print(f"Open {obsidian_dir}/ as a vault in Obsidian.") - - elif subcmd == "wiki": - from graphify.wiki import to_wiki as _to_wiki - from graphify.analyze import god_nodes as _god_nodes - if not communities: - print( - "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" - "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", - file=sys.stderr, + external_extraction = introspect_postgres(postgres_dsn) + except (ConnectionError, ImportError) as exc: + raise RuntimeError(str(exc)) from exc + if positional is None: + _activate_external_only( + external_extraction, + output_root=output or Path.cwd(), + no_cluster=no_cluster, + retain_rollback="--retain-rollback" in args, + source_name="PostgreSQL schema", ) - sys.exit(1) - if not gods_data: - gods_data = _god_nodes(G) - n = _to_wiki(G, communities, str(out_dir / "wiki"), - community_labels=labels or None, cohesion=cohesion or None, - god_nodes_data=gods_data) - print(f"Wiki: {n} articles written to {out_dir}/wiki/") - print(f" {out_dir}/wiki/index.md -> agent entry point") - - elif subcmd == "svg": - from graphify.export import to_svg as _to_svg - _to_svg(G, communities, str(out_dir / "graph.svg"), - community_labels=labels or None) - print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") - - elif subcmd == "graphml": - from graphify.export import to_graphml as _to_graphml - _to_graphml(G, communities, str(out_dir / "graph.graphml")) - print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") - - elif subcmd == "neo4j": - if push_uri: - from graphify.export import push_to_neo4j as _push - if push_password is None: - print("error: --password required for --push", file=sys.stderr) - sys.exit(1) - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") - - elif subcmd == "falkordb": - if push_uri: - from graphify.export import push_to_falkordb as _push - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " - f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " - f"import), so load a graph with: graphify export falkordb --push " - f"falkordb://localhost:6379") - - elif cmd == "benchmark": - from graphify.benchmark import run_benchmark, print_benchmark - - graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() - _enforce_graph_size_cap_or_exit(Path(graph_path)) - # Try to load corpus_words from detect output - corpus_words = None - detect_path = Path(".graphify_detect.json") - if detect_path.exists(): - try: - detect_data = json.loads(detect_path.read_text(encoding="utf-8")) - corpus_words = detect_data.get("total_words") - except Exception: - pass - result = run_benchmark(graph_path, corpus_words=corpus_words) - print_benchmark(result) - - elif cmd == "global": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - from graphify.global_graph import ( - global_add as _global_add, - global_remove as _global_remove, - global_list as _global_list, - global_path as _global_path, + return + if not _rebuild_code( + target, output_root=output, changed_paths=changed, force=force, no_cluster=no_cluster, + block_on_lock=True, + include_semantic=include_semantic, + code_only="--code-only" in args, + backend=_option(args, "--backend"), + model=_option(args, "--model"), + deep_mode=_option(args, "--mode") == "deep" or "--deep" in args, + token_budget=int(_option(args, "--token-budget") or 60_000), + max_concurrency=int(_option(args, "--max-concurrency") or 4), + retain_rollback="--retain-rollback" in args, + external_extraction=external_extraction, + external_kind="postgres" if external_extraction is not None else None, + prune_excluded=cmd == "extract", + raise_on_error=True, + _timer=timer, + ): + raise RuntimeError("graph rebuild failed") + finally: + timer.total() + + +def _save_result(args: list[str]) -> None: + from graphify.ingest import save_query_result + + question = _option(args, "--question") + answer = _option(args, "--answer") + answer_file = _option(args, "--answer-file") + if answer_file: + answer = Path(answer_file).read_text(encoding="utf-8").strip() + if not question or not answer: + raise ValueError("save-result requires --question and --answer or --answer-file") + nodes: list[str] = [] + if "--nodes" in args: + index = args.index("--nodes") + 1 + while index < len(args) and not args[index].startswith("--"): + nodes.append(args[index]) + index += 1 + output = save_query_result( + question=question, + answer=answer, + memory_dir=Path(_option(args, "--memory-dir") or Path(_GRAPHIFY_OUT) / "memory"), + query_type=_option(args, "--type") or "query", + source_nodes=nodes or None, + outcome=_option(args, "--outcome"), + correction=_option(args, "--correction"), + ) + print(f"Saved to {output}") + + +def _reflect(args: list[str]) -> None: + from graphify.reflect import lessons_fresh, reflect + + memory_dir = Path(_option(args, "--memory-dir") or Path(_GRAPHIFY_OUT) / "memory") + output = Path(_option(args, "--out") or Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md") + store_value = _option(args, "--store", "--graph") + candidate = _store_arg(args) + if store_value: + _validate_store_or_exit(candidate) + store = candidate + else: + # Preserve v8's automatic graph-aware reflection when the project's + # default graph exists, while still allowing a graph-less cold start. + store = candidate if candidate.is_dir() else None + if "--if-stale" in args and lessons_fresh(output, memory_dir, store): + print(f"Lessons already up to date -> {output}") + return + result, aggregate = reflect( + memory_dir=memory_dir, + out_path=output, + graph_path=store, + half_life_days=float(_option(args, "--half-life-days") or 30), + min_corroboration=int(_option(args, "--min-corroboration") or 2), + ) + print(f"Reflected {aggregate['total']} memories -> {result}") + + +def _tree(args: list[str]) -> None: + from graphify.tree_html import write_tree_html + + store = _store_arg(args) + output = Path(_option(args, "--output", "--out") or Path(_GRAPHIFY_OUT) / "GRAPH_TREE.html") + result = write_tree_html( + store, + output, + root=_option(args, "--root"), + max_children=int(_option(args, "--max-children") or 200), + project_label=_option(args, "--label"), + top_k_edges=int(_option(args, "--top-k-edges") or 12), + ) + print(result) + + +def _cache_check(args: list[str]) -> None: + from graphify.cache import check_semantic_cache + from graphify.helix.persistence import load_graph + from graphify.llm import _extraction_system + + if not args or args[0].startswith("-"): + raise ValueError("cache-check requires a newline-delimited file list") + root = Path(_option(args, "--root") or ".") + files = [line for line in Path(args[0]).read_text(encoding="utf-8").splitlines() if line.strip()] + store = root / _GRAPHIFY_OUT / "graph.helix" + cache: dict[str, dict] = {} + if store.is_dir(): + loaded = load_graph(store) + value = loaded.state.get("incremental", {}).get("extraction_cache", {}) + if isinstance(value, dict): + cache = value + mode = "deep" if "--deep" in args else _option(args, "--mode") + prompt_file = _option(args, "--prompt-file") + nodes, edges, hyperedges, uncached = check_semantic_cache( + files, + cache, + root=root, + mode=mode, + prompt=None if prompt_file else _extraction_system(deep=mode == "deep"), + prompt_file=prompt_file, + ) + # These are transient extraction interchange files, never graph stores. + out = root / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + if nodes or edges or hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), + encoding="utf-8", ) - if subcmd == "add": - # graphify global add [--as ] - args = sys.argv[3:] - source = None - tag = None - i = 0 - while i < len(args): - if args[i] == "--as" and i + 1 < len(args): - tag = args[i + 1]; i += 2 - elif not source: - source = Path(args[i]); i += 1 - else: - i += 1 - if not source: - print("Usage: graphify global add [--as ]", file=sys.stderr) - sys.exit(1) - tag = tag or source.parent.parent.name - try: - result = _global_add(source, tag) - if result["skipped"]: - print(f"'{tag}' unchanged since last add - global graph not modified.") - else: - print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " - f"-{result['nodes_removed']} pruned. Global: {_global_path()}") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "remove": - tag = sys.argv[3] if len(sys.argv) > 3 else "" - if not tag: - print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) - try: - removed = _global_remove(tag) - print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") - except KeyError as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "list": - repos = _global_list() - if not repos: - print("Global graph is empty. Use 'graphify global add' to add a project.") - else: - print(f"Global graph: {_global_path()}") - for tag, info in repos.items(): - print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") - elif subcmd == "path": - print(_global_path()) - else: - print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) - - elif cmd == "extract": - # Headless full-pipeline extraction for CI / scripts (#698). - # Runs detect -> AST extraction on code -> semantic LLM extraction on - # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. - if len(sys.argv) < 3: - print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " - "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] " - "[--no-gitignore] [--code-only] " - "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", - file=sys.stderr, - ) - sys.exit(1) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - has_path = True - if sys.argv[2].startswith("-"): - has_path = False - target = Path(".").resolve() - else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): - print(f"error: path not found: {target}", file=sys.stderr) - sys.exit(1) - - backend: str | None = None - model: str | None = None - extract_mode: str | None = None - out_dir: Path | None = None - cli_postgres_dsn: str | None = None - cli_cargo: bool = False - cli_allow_partial: bool = False - no_cluster = False - dedup_llm = False - google_workspace = False - global_merge = False - code_only = False - no_gitignore = False - global_repo_tag: str | None = None - # Performance/tuning knobs (issue #792). None means "use library default". - cli_max_workers: int | None = None - cli_token_budget: int | None = None - cli_max_concurrency: int | None = None - cli_api_timeout: float | None = None - # Clustering tuning knobs - cli_resolution: float = 1.0 - cli_exclude_hubs: float | None = None - cli_excludes: list[str] = [] - cli_timing: bool = False - # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 - # disables the incremental gate and skips semantic-cache reads (#1894). - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - - def _parse_int(name: str, raw: str) -> int: - try: - v = int(raw) - except ValueError: - print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - def _parse_float(name: str, raw: str) -> float: - try: - v = float(raw) - except ValueError: - print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - args = sys.argv[3:] if has_path else sys.argv[2:] - i = 0 - while i < len(args): - a = args[i] - if a == "--backend" and i + 1 < len(args): - backend = args[i + 1]; i += 2 - elif a.startswith("--backend="): - backend = a.split("=", 1)[1]; i += 1 - elif a == "--model" and i + 1 < len(args): - model = args[i + 1]; i += 2 - elif a.startswith("--model="): - model = a.split("=", 1)[1]; i += 1 - elif a == "--mode" and i + 1 < len(args): - extract_mode = args[i + 1]; i += 2 - elif a.startswith("--mode="): - extract_mode = a.split("=", 1)[1]; i += 1 - elif a in ("--out", "--output") and i + 1 < len(args): - # --output is an alias of --out (#2004): it was silently dropped - # before, and `graphify tree` already documents --output, so the - # mistake is natural. (--output= does not startswith --out=.) - out_dir = Path(args[i + 1]); i += 2 - elif a.startswith(("--out=", "--output=")): - out_dir = Path(a.split("=", 1)[1]); i += 1 - elif a == "--no-cluster": - no_cluster = True; i += 1 - elif a == "--dedup-llm": - dedup_llm = True; i += 1 - elif a == "--code-only": - code_only = True; i += 1 - elif a == "--google-workspace": - google_workspace = True; i += 1 - elif a == "--no-gitignore": - no_gitignore = True; i += 1 - elif a == "--global": - global_merge = True; i += 1 - elif a == "--as" and i + 1 < len(args): - global_repo_tag = args[i + 1]; i += 2 - elif a == "--max-workers" and i + 1 < len(args): - cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 - elif a.startswith("--max-workers="): - cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 - elif a == "--token-budget" and i + 1 < len(args): - cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 - elif a.startswith("--token-budget="): - cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 - elif a == "--max-concurrency" and i + 1 < len(args): - cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 - elif a.startswith("--max-concurrency="): - cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 - elif a == "--api-timeout" and i + 1 < len(args): - cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 - elif a.startswith("--api-timeout="): - cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 - elif a == "--resolution" and i + 1 < len(args): - cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 - elif a.startswith("--resolution="): - cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 - elif a == "--exclude-hubs" and i + 1 < len(args): - cli_exclude_hubs = float(args[i + 1]); i += 2 - elif a.startswith("--exclude-hubs="): - cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 - elif a == "--exclude" and i + 1 < len(args): - cli_excludes.append(args[i + 1]); i += 2 - elif a.startswith("--exclude="): - cli_excludes.append(a.split("=", 1)[1]); i += 1 - elif a == "--postgres" and i + 1 < len(args): - cli_postgres_dsn = args[i + 1]; i += 2 - elif a.startswith("--postgres="): - cli_postgres_dsn = a.split("=", 1)[1]; i += 1 - elif a == "--cargo": - cli_cargo = True - i += 1 - elif a == "--force": - force = True; i += 1 - elif a == "--allow-partial": - cli_allow_partial = True; i += 1 - elif a == "--timing": - cli_timing = True; i += 1 - else: - i += 1 - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) - sys.exit(1) +def _merge_chunks(args: list[str]) -> None: + """Merge validated transient semantic DTO fragments. - _VALID_MODES = {"deep"} - if extract_mode is not None and extract_mode not in _VALID_MODES: - print( - f"error: unknown --mode '{extract_mode}'. " - f"Available: {', '.join(sorted(_VALID_MODES))}", - file=sys.stderr, - ) - sys.exit(2) - deep_mode = extract_mode == "deep" - if deep_mode: - print("[graphify extract] deep mode enabled: richer semantic extraction") - - # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so - # _call_openai_compat picks it up without needing a new kwarg path. - if cli_api_timeout is not None: - os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) - if cli_max_workers is not None: - os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) - - # Resolve output dir. The user-facing contract is "/graphify-out/" - # so a fresh checkout writes graphify-out/ at the project root, matching - # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) - graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) - # Persist corpus-shaping options so later update/watch/hook rebuilds - # use the same file set as the initial extraction (#1886). - from graphify.watch import ( - _write_build_config as _write_build_cfg, - _read_build_excludes as _read_build_ex, - _read_build_gitignore as _read_build_gi, - ) - # #1971 persistence: an explicit --no-gitignore persists False; a later - # flag-less `graphify extract` must NOT clobber it back to True, which - # would make the git-ignored code silently disappear again (the exact - # complaint #1971 is about). Honor the persisted value for THIS run when - # the flag is absent (read before the write below), and write False only - # when the flag is set — None leaves the setting as-is, mirroring how - # #1886 persists --exclude. - _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) - # An explicit list replaces the persisted one; omission reuses it. - _effective_excludes = cli_excludes or _read_build_ex(graphify_out) - _write_build_cfg( - graphify_out, - excludes=cli_excludes or None, - gitignore=False if no_gitignore else None, - ) + These files are untrusted extraction results, not graph stores. They are + validated before merging and the output is written atomically for the skill + workflow to consume before constructing a native generation. + """ + import glob - stages = _StageTimer(cli_timing) + from graphify.paths import write_json_atomic + from graphify.semantic_cleanup import load_validated_semantic_fragment + + output_value = _option(args, "--out") + if output_value is None: + raise ValueError("merge-chunks requires --out ") + output = Path(output_value) + chunk_args: list[str] = [] + index = 0 + while index < len(args): + if args[index] == "--out": + index += 2 + continue + if args[index].startswith("--out="): + index += 1 + continue + chunk_args.append(args[index]) + index += 1 - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - # #1925: a missing manifest.json must not degrade to a full scan that - # discards the existing graph's semantic layer. An existing graph.json - # is a sufficient incremental baseline: detect_incremental treats an - # absent manifest as "everything is new" (re-extract all, nothing - # deleted), and build_merge + _stale_graph_sources reconcile replaced - # and genuinely-deleted sources against the current corpus, so doc/ - # paper/image nodes survive a --code-only rebuild instead of being - # dropped with the rest of the committed graph. - incremental_mode = existing_graph_path.exists() if has_path else False - # --force: full scan, not the manifest-gated incremental diff — a warm - # unchanged tree would otherwise dispatch zero files (#1894). - incremental_mode = incremental_mode and not force - if force: - print("[graphify extract] --force: full re-scan, semantic cache reads skipped") - elif incremental_mode and not manifest_path.exists(): - print( - "[graphify extract] manifest.json missing; using existing " - "graph.json as the incremental baseline (all files re-checked; " - "nodes for files outside this run's scope are preserved)" - ) + chunk_files: list[str] = [] + for value in chunk_args: + expanded = glob.glob(value) + chunk_files.extend(sorted(expanded) if expanded else [value]) - if not has_path: - detection = {} - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=_effective_excludes or None, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - excluded_files = list(detection.get("excluded_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - # #1909: derive the prune set from the existing graph itself, not - # just the manifest. A file that became excluded without ever - # being manifest-listed (every pre-#1897 graph is in this state) - # still has stale nodes carried forward by build_merge unless the - # graph's own sources are reconciled against the current corpus. - _seen_files = {f for _fl in files_by_type.values() for f in _fl} - _seen_files.update(detection.get("unclassified", [])) - graph_stale_sources = _stale_graph_sources( - existing_graph_path, target, _seen_files - ) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect( - target, - google_workspace=google_workspace or None, - extra_excludes=_effective_excludes or None, - cache_root=out_root, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - # --code-only: index code (pure local AST, no key) and skip the semantic - # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no - # LLM backend is configured (#1734). Report what was skipped rather than - # silently dropping it. - if code_only and semantic_files: - print( - f"[graphify extract] --code-only: skipping {len(semantic_files)} " - f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images) — no LLM extraction" - ) - semantic_files = [] - doc_files = [] - paper_files = [] - image_files = [] - if deep_mode and incremental_mode and not code_only: - # Deep mode reads/writes its own cache namespace - # (cache/semantic-deep/), so the manifest's changed-file gate is - # not a valid proxy for deep coverage: over a warm unchanged tree - # it dispatches zero files and `--mode deep` silently no-ops - # (#1894). Widen the semantic pass to the FULL live - # doc/paper/image set (``files_by_type`` from detect_incremental, - # which already excludes excluded files) and let the - # mode-namespaced cache decide hits/misses — the first deep run - # re-dispatches everything (deep namespace cold), later deep runs - # hit the deep cache. - _deep_all = [ - Path(p) - for _ftype in ("document", "paper", "image") - for p in files_by_type.get(_ftype, []) - ] - if len(_deep_all) != len(semantic_files): - print( - f"[graphify extract] deep mode: widening semantic pass from " - f"{len(semantic_files)} changed to {len(_deep_all)} live " - f"doc/paper/image file(s); the deep semantic cache decides " - f"what is re-extracted" - ) - semantic_files = _deep_all - if incremental_mode: - # Excluded-but-alive files are reported separately from deletions - # (#1908): they still exist on disk, the scan just stopped - # covering them (ignore rules / --exclude changed). - _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - f"{_excl_note}" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - # Surface files that were seen but not classified (extensionless non-shebang - # project files like Dockerfile/Makefile, or unsupported extensions), so they - # are no longer invisible in graphify's own output (#1692). - _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] - if _unclassified: - _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) - _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" - print( - f"[graphify extract] {len(_unclassified)} file(s) not classified " - f"(no supported extension or shebang), skipped: {_names}{_more}" - ) - # Name the files dropped by the sensitive-file filter so a wrongly-flagged - # source/doc is visible, not just a count (#2106). Operational skips - # (symlink/office/Workspace) carry a " [reason]" suffix; exclude those here - # so this line reports only the security-heuristic drops. - _sensitive = detection.get("skipped_sensitive", []) if isinstance(detection, dict) else [] - _sec = [s for s in _sensitive if " [" not in s] - if _sec: - _snames = ", ".join(sorted({Path(p).name for p in _sec})[:6]) - _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else "" - print( - f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive " - f"(rename or move if wrongly flagged): {_snames}{_smore}" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: + merged: dict[str, Any] = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + } + seen_ids: set[str] = set() + valid_chunks = 0 + for raw_path in chunk_files: + chunk, errors = load_validated_semantic_fragment(Path(raw_path)) + if errors: print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", + f"[graphify merge-chunks] warning: skipping invalid chunk " + f"{raw_path}: {'; '.join(errors[:3])}", file=sys.stderr, ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - hint = "" - if semantic_files: - hint = (" Or pass --code-only to index just the code " - "(local AST, no key) and skip the non-code files.") - print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key." + hint, - file=sys.stderr, - ) - sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), - ) - try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None - if not allow_no_key: - print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", - file=sys.stderr, - ) - sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - - # Track whether this run's extraction was incomplete (a whole extractor - # pass crashed, or some semantic chunks failed). A partial result must not - # be force-written over a good complete graph — the final write falls back - # to the #479 shrink guard unless --allow-partial is set. - _extraction_incomplete = False - # A walk that couldn't fully enumerate the corpus (permission-denied - # subtree, I/O error) yields a legitimately smaller graph that must not - # be force-written over a complete one — same failure class as a crashed - # pass. detect()/detect_incremental() already record these; consume them. - if detection.get("walk_errors"): - _extraction_incomplete = True - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - # `root` stays the scanned project so source_file/ids relativize - # against it; conflating the two basenamed every node (#1941). - ast_kwargs: dict = {"cache_root": out_root, "root": target} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") - try: - ast_result = _ast_extract(code_files, **ast_kwargs) - except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the whole AST pass was lost - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, + continue + assert chunk is not None + valid_chunks += 1 + for node in chunk.get("nodes", []): + node_id = str(node.get("id")) + if node_id not in seen_ids: + seen_ids.add(node_id) + merged["nodes"].append(node) + merged["edges"].extend(chunk.get("edges", [])) + merged["hyperedges"].extend(chunk.get("hyperedges", [])) + for token_key in ("input_tokens", "output_tokens"): + value = chunk.get(token_key, 0) + if isinstance(value, (int, float)) and not isinstance(value, bool): + merged[token_key] += value + if not valid_chunks: + raise ValueError( + f"no valid chunks to merge; refusing to write {output}" ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - # Semantic files whose extraction truncated this run. They are left - # unstamped in the manifest so detect_incremental re-queues them next run - # (mirrors the #933 failed-chunk handling); captured below before the - # _partial markers are stripped from the corpus. - _partial_semantic_files: set[str] = set() - sem_cache_hits = 0 - sem_cache_misses = 0 - # Deep mode uses its own namespace (cache/semantic-deep/) so deep and - # standard results for the same content never shadow each other (#1894). - sem_cache_mode = "deep" if deep_mode else None - # Entries are attributed to the extraction prompt that produced them, so - # a release that changes the prompt re-extracts rather than replaying the - # older vintage alongside the new one (#1939). Read and write must pass - # the same prompt, or the write lands where the next read won't look. - from graphify.llm import _extraction_system as _sem_prompt_for - sem_prompt = _sem_prompt_for(deep=deep_mode) - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - if force: - # --force: skip the cache READ so every semantic file is - # re-dispatched; the save below still runs so the fresh - # results replace the stale entries. - cached_nodes, cached_edges, cached_hyperedges = [], [], [] - uncached_paths = list(sem_paths_str) - else: - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=target, cache_root=out_root, - mode=sem_cache_mode, prompt=sem_prompt) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - "cache_root": out_root, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress + output.parent.mkdir(parents=True, exist_ok=True) + write_json_atomic(output, merged, ensure_ascii=False) + summary = ( + f"{valid_chunks} chunks" + if valid_chunks == len(chunk_files) + else f"{valid_chunks} of {len(chunk_files)} chunks" + ) + print( + f"Merged {summary}: {len(merged['nodes'])} nodes, " + f"{len(merged['edges'])} edges, {merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out tokens" + ) - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the semantic pass crashed - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - # Some (but not all) chunks failed — the graph is missing nodes - # from the failed chunks, so it must not clobber a larger complete - # graph without an explicit --allow-partial override. - if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: - _extraction_incomplete = True - # Which files truncated this run (item markers + the empty-parse - # _partial_files set). Computed BEFORE the save so it can be passed - # as partial_source_files: without it, a file whose only truncated - # chunk parsed empty (so it has no item markers here) would be - # written as a complete cache entry, re-promoting it (#1950). - from graphify.llm import ( - _partial_source_files as _partial_sf, - _strip_partial_markers as _strip_partial, - ) - _partial_semantic_files = set(_partial_sf(fresh)) - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=target, - cache_root=out_root, - allowed_source_files=uncached_paths, - mode=sem_cache_mode, - prompt=sem_prompt, - partial_source_files=_partial_semantic_files or None, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - # Strip the markers before the corpus feeds the graph so the - # internal flag never leaks into graph.json. - _strip_partial(fresh) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - # Hash keys are anchored to the corpus (``target``) — the same anchor - # the cache read/write above use — while the stat-index artifact - # follows the cache location (``out_root``). Anchoring these hashes to - # ``out_root`` instead would mismatch every key under ``--out`` and - # sweep the entire fresh cache as orphaned (#1990/#1991). - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(target) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) - except OSError: - pass - # A pathless database extraction has no filesystem corpus to sweep. - if has_path: - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } +def _rollback(args: list[str]) -> None: + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.security import validate_store_path - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - # Path normalization against the scan root happens inside the helper - # (#1897) so fresh root-relative source_files match detect()'s - # absolute file lists. - _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target, - partial_source_files=_partial_semantic_files) - - # Files dispatched this run but dropped by _stamped_manifest_files - # above (failed chunk, LLM omission, or any future exclusion) still - # carry a stale semantic_hash from a prior successful run in the - # on-disk manifest; save_manifest's seed loop would otherwise copy it - # verbatim and mask the omission (#1948). Derived from semantic_files - # — what was actually SENT to the backend this run (narrowed by the - # incremental gate and --code-only, widened by deep mode) — NOT from - # files_by_type: the full live corpus includes untouched files that - # were never dispatched, and clearing those would blank the whole - # manifest on every partial incremental run, forcing a full-corpus - # re-extraction on the next one. - _stamped_semantic = { - f for _flist in _manifest_files.values() for f in _flist - } - _cleared_semantic = {str(p) for p in semantic_files} - _stamped_semantic - - # Full-scan manifest saves prune rows for in-root files that left the - # scan corpus but still exist on disk (#1908). The corpus must be the - # RAW detect output (files_by_type), NOT the #933-stamp-filtered - # _manifest_files above — pruning to the filtered set would erase - # failed-chunk/omitted-doc rows and every doc row on --code-only runs. - _scan_corpus = ( - {f for _fl in files_by_type.values() for f in _fl} - if has_path else None - ) + store_path = validate_store_path(_store_arg(args)) + with HelixEmbeddedStore(store_path, retain_rollback=True) as store: + loaded = store.rollback() + print(f"Rolled back {store_path} to generation {loaded.generation}.") - def _invalidate_file_manifest_for_db_graph() -> None: - if has_path: - return - try: - manifest_path.unlink(missing_ok=True) - except OSError as exc: - print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) - sys.exit(1) - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import ( - backup_if_protected as _backup, - existing_graph_node_count as _existing_graph_node_count, +def _doctor(args: list[str]) -> None: + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.security import validate_store_path + + store_path = validate_store_path(_store_arg(args)) + with HelixEmbeddedStore(store_path, read_only=True) as store: + result = store.verify() if "--deep" in args else store.verify_counts() + mode = "deep" if "--deep" in args else "counts" + print(f"Helix store OK ({mode}): {json.dumps(result, sort_keys=True)}") + + +def dispatch_command(cmd: str) -> None: + args = sys.argv[2:] + try: + if cmd == "query": + _query(args) + elif cmd == "path": + _path(args) + elif cmd == "explain": + _explain(args) + elif cmd == "affected": + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected + + query = next((value for value in args if not value.startswith("-")), "") + depth = int(_option(args, "--depth") or 2) + relations = [ + value.split("=", 1)[1] + for value in args + if value.startswith("--relation=") + ] + for index, value in enumerate(args): + if value == "--relation" and index + 1 < len(args): + relations.append(args[index + 1]) + loaded = _loaded(args) + print(format_affected( + loaded.graph, query, + node_query=loaded.query, + relations=relations or DEFAULT_AFFECTED_RELATIONS, + depth=depth, + )) + elif cmd in {"god-nodes", "god_nodes"}: + from graphify.analyze import god_nodes + from graphify.security import sanitize_label + + loaded = _loaded(args) + nodes = god_nodes( + loaded.graph, + top_n=int(_option(args, "--top") or 10), ) - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - # An exclusion-only change reaches this gate (excluded files - # are deliberately NOT in deleted_files, #1908) but must still - # scrub the newly-excluded sources from the raw graph (#1909). - # This path never runs build_merge, so prune in place. - if graph_stale_sources: - _n_pruned = _prune_graph_json_sources( - existing_graph_path, graph_stale_sources - ) - if _n_pruned: - print( - f"[graphify extract] pruned {_n_pruned} node(s) from " - f"{len(graph_stale_sources)} source file(s) no longer " - "in the scan (deleted or excluded)." - ) - print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Disambiguate colliding-basename file-node labels (#2032). This raw - # --no-cluster path bypasses build_from_json (where the clustered path - # gets this), so apply it directly on the merged node list. - from graphify.build import disambiguate_file_labels_in_nodes as _disamb_labels - _disamb_labels(merged["nodes"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - # RT-parity for the raw path: an incomplete build must not force a - # partial graph over a larger complete one here either. The clustered - # path gets this from to_json's #479 guard; this path never calls - # to_json, so replicate the shrink check against the existing file and - # exit before the write/manifest unless --allow-partial is set. - if _extraction_incomplete and not cli_allow_partial: - from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH - _existing_n = _existing_graph_node_count(graph_json_path) - _malformed = _existing_n is _MALFORMED_GRAPH - _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n - if _malformed or _shrinks: - _detail = ( - f"the existing {graph_json_path} is present but unparseable " - "(corrupt or a mid-write), so a shrink cannot be ruled out" - if _malformed - else f"smaller than the existing {graph_json_path} " - f"({len(merged['nodes'])} < {_existing_n} nodes)" - ) + if "--json" in args: + print(json.dumps(nodes, indent=2)) + else: + print("God nodes (most connected):") + for rank, node in enumerate(nodes, 1): print( - "[graphify extract] error: extraction was incomplete (an AST/" - f"semantic pass failed) and the resulting --no-cluster graph is {_detail}. " - "Refusing to overwrite a complete graph with a partial one. Re-run after " - "fixing the failures, or pass --allow-partial to overwrite anyway.", - file=sys.stderr, + f" {rank}. {sanitize_label(str(node['label']))} - " + f"{node['degree']} edges" ) - sys.exit(1) - _backup(graphify_out) - _invalidate_file_manifest_for_db_graph() - from graphify.paths import write_json_atomic as _write_json_atomic - _write_json_atomic(graph_json_path, merged, indent=2) - try: - # Record the scan root so a later build_merge / update runbook can - # relativize deleted-file paths correctly even for a custom --out - # (its grandparent-of-graph.json fallback points at the wrong dir - # otherwise, and deleted files never prune — #2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" - ) - except OSError: - pass - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" - ) - try: - if has_path: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - # Prune everything the current scan no longer covers: genuinely - # deleted manifest rows, excluded-but-alive manifest rows (#1908), - # and the graph's own stale sources — which catches files that - # became excluded without ever being manifest-listed (#1909). - _prune_sources: list[str] = list(deleted_files) - for _src in list(excluded_files) + graph_stale_sources: - if _src not in _prune_sources: - _prune_sources.append(_src) - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=_prune_sources or None, - dedup=True, - dedup_llm_backend=dedup_backend, - root=target, - ) - else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) + elif cmd in {"extract", "update"}: + _update_or_extract(cmd, args) + elif cmd == "watch": + from graphify.watch import watch - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - _invalidate_file_manifest_for_db_graph() - # force=True bypasses the #479 shrink guard entirely. A full build - # legitimately shrinks (fuzzy dedup collapse, deleted code) so it keeps - # force=True — EXCEPT when this run's extraction was incomplete (an - # extractor pass crashed or some semantic chunks failed). Then a partial - # graph could silently overwrite a good complete one, so fall back to the - # shrink guard (force=False) unless the user opts in with --allow-partial. - # - # Both write paths are guarded: the clustered path here via to_json's - # #479 check, and the `--no-cluster` raw-dump path above via the same - # shrink check against the existing file (existing_graph_node_count). - # - # Trade-off: this reuses to_json's coarse node-count guard, not the - # source-aware _check_shrink that watch/update use. On an incremental run - # a legitimate deletion that coincides with an unrelated transient chunk - # failure can therefore be refused here — recoverable by re-running or - # passing --allow-partial (the good graph is preserved and the manifest - # is not stamped, so the retry re-extracts). - _force_write = cli_allow_partial or not _extraction_incomplete - _wrote = _to_json(G, communities, str(graph_json_path), force=_force_write) - if not _wrote: - # The shrink guard refused: this partial build is smaller than the - # existing graph. Exit before writing the manifest/marker below, which - # would otherwise stamp these files as done and make the next - # incremental run skip re-extracting them (poisoning the manifest - # against the graph we declined to write). Exit non-zero so a retry - # re-attempts. - print( - "[graphify extract] error: extraction was incomplete (an AST/semantic " - f"pass failed) and the resulting graph is smaller than the existing " - f"{graph_json_path}. Refusing to overwrite a complete graph with a " - "partial one. Re-run after fixing the failures, or pass --allow-partial " - "to overwrite anyway.", - file=sys.stderr, - ) - sys.exit(1) - try: - # See the --no-cluster path above: persist the scan root so build_merge - # can relativize deleted-file paths under a custom --out (#2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" - ) - except OSError: - pass - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + target = Path(next((value for value in args if not value.startswith("-")), ".")) + watch( + target, + debounce=float(_option(args, "--debounce") or 3.0), + retain_rollback="--retain-rollback" in args, ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - from graphify.paths import write_json_atomic as _wja - _wja(analysis_path, analysis, indent=2) - try: - if has_path: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted{_excl_note}" + elif cmd == "check-update": + from graphify.watch import check_update + + check_update(Path(args[0] if args else ".")) + elif cmd == "cluster-only": + from graphify.operations import recluster, reanalyze + + store = _store_arg(args, default=Path(args[0]) / _GRAPHIFY_OUT / "graph.helix" if args and not args[0].startswith("-") else None) + _validate_store_or_exit(store) + recluster(store) + reanalyze(store) + print(f"Reclustered {store}.") + elif cmd == "label": + from graphify.operations import relabel + + default = ( + Path(args[0]) / _GRAPHIFY_OUT / "graph.helix" + if args and not args[0].startswith("-") + else None ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" + store = _store_arg(args, default=default) + labels = relabel( + store, + backend=_option(args, "--backend"), + model=_option(args, "--model"), + missing_only="--missing-only" in args, + max_concurrency=int(_option(args, "--max-concurrency") or 4), + batch_size=int(_option(args, "--batch-size") or 100), ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() - - elif cmd == "cache-check": - # graphify cache-check [--root ] [--mode | --deep] - # [--prompt-file ] - # Reads file paths (one per line) from , checks semantic cache. - # --mode deep (or --deep) checks the cache/semantic-deep/ namespace - # written by `extract --mode deep` instead of cache/semantic/ (#1894). - # --prompt-file names the extraction prompt the caller will use (an agent's - # references/extraction-spec.md), restricting hits to entries produced by - # that same prompt (#1939). Omitting it reads the unattributed layout, which - # cannot see entries a fingerprinted run wrote. - # Writes: - # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges - # graphify-out/.graphify_uncached.txt — paths that need extraction - # Stdout: "Cache: N hit, M miss" - from graphify.cache import check_semantic_cache - if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ] " - "[--mode | --deep] [--prompt-file ]", file=sys.stderr) - sys.exit(1) - files_from = Path(sys.argv[2]) - root = Path(".") - cache_mode: str | None = None - prompt_file: str | None = None - i = 3 - while i < len(sys.argv): - if sys.argv[i] == "--root" and i + 1 < len(sys.argv): - root = Path(sys.argv[i + 1]) - i += 2 - elif sys.argv[i] == "--mode" and i + 1 < len(sys.argv): - cache_mode = sys.argv[i + 1] - i += 2 - elif sys.argv[i].startswith("--mode="): - cache_mode = sys.argv[i].split("=", 1)[1] - i += 1 - elif sys.argv[i] == "--deep": - cache_mode = "deep" - i += 1 - elif sys.argv[i] == "--prompt-file" and i + 1 < len(sys.argv): - prompt_file = sys.argv[i + 1] - i += 2 - elif sys.argv[i].startswith("--prompt-file="): - prompt_file = sys.argv[i].split("=", 1)[1] - i += 1 - else: - i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( - files, root, mode=cache_mode, prompt_file=prompt_file - ) - out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( - json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, - ensure_ascii=False), - encoding="utf-8", + print(f"Labeled {len(labels)} communities in {store}.") + elif cmd == "export": + _export(args) + elif cmd == "tree": + _tree(args) + elif cmd == "save-result": + _save_result(args) + elif cmd == "reflect": + _reflect(args) + elif cmd == "benchmark": + from graphify.benchmark import print_benchmark, run_benchmark + + print_benchmark(run_benchmark(str(_store_arg(args)))) + elif cmd == "global": + _global(args) + elif cmd == "rollback": + _rollback(args) + elif cmd == "doctor": + _doctor(args) + elif cmd == "prs": + from graphify.prs import cmd_prs + + cmd_prs(args) + elif cmd == "clone": + if not args: + raise ValueError("clone requires a GitHub URL") + clone_output = _option(args, "--out") + _clone_repo(args[0], _option(args, "--branch"), Path(clone_output) if clone_output else None) + elif cmd == "hook-check": + _run_hook_guard("search") + elif cmd == "hook-guard": + _run_hook_guard( + args[0] if args else "search", + strict="--strict" in args[1:], ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") - print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - - elif cmd == "merge-chunks": - # graphify merge-chunks --out - # Concatenates .graphify_chunk_*.json files written by semantic subagents. - # Deduplicates nodes by id (first writer wins). Sums token counts. - import glob as _glob - if len(sys.argv) < 3: - print("Usage: graphify merge-chunks --out ", file=sys.stderr) - sys.exit(1) - out_path: Path | None = None - chunk_args: list[str] = [] - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path = Path(sys.argv[i + 1]) - i += 2 + elif cmd == "hook": + from graphify import hooks + + action = args[0] if args else "install" + target = Path(args[1] if len(args) > 1 else ".") + if action == "install": + print(hooks.install(target)) + elif action == "uninstall": + print(hooks.uninstall(target)) + elif action == "status": + print(hooks.status(target)) else: - chunk_args.append(sys.argv[i]) - i += 1 - if not out_path: - print("error: --out required", file=sys.stderr) - sys.exit(1) - chunk_files: list[str] = [] - for arg in chunk_args: - expanded = _glob.glob(arg) - chunk_files.extend(sorted(expanded) if expanded else [arg]) - merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - seen_ids: set[str] = set() - valid_chunks = 0 - # These chunk files are untrusted subagent output. load_validated_... - # stats the file size BEFORE reading it (so a multi-GB chunk can't blow up - # memory), parses the JSON, and validates the security caps + the node/ - # edge id charset that blocks path traversal (#825) — the same enforcement - # the skill merge path applies. A bad chunk is skipped with a warning - # while valid siblings still merge; if every chunk is invalid, fail - # closed instead of reporting success and replacing --out with an empty - # semantic layer. Deliberately NOT wired into - # build_from_json/load_graph_json, which must keep loading valid - # pre-existing graphs. file_type is left to build's coercion (#840). - from graphify.semantic_cleanup import load_validated_semantic_fragment - for cf in chunk_files: - chunk, _chunk_errs = load_validated_semantic_fragment(Path(cf)) - if _chunk_errs: - print( - f"[graphify merge-chunks] warning: skipping invalid chunk {cf}: " - f"{'; '.join(_chunk_errs[:3])}", - file=sys.stderr, - ) - continue - valid_chunks += 1 - for n in chunk.get("nodes", []): - if n.get("id") not in seen_ids: - seen_ids.add(n["id"]) - merged["nodes"].append(n) - merged["edges"].extend(chunk.get("edges", [])) - merged["hyperedges"].extend(chunk.get("hyperedges", [])) - # Coerce token counts: a chunk is untrusted, so a non-numeric - # input_tokens/output_tokens must not abort the whole merge with a - # TypeError after other chunks already merged. - for _tok in ("input_tokens", "output_tokens"): - _v = chunk.get(_tok, 0) - merged[_tok] += _v if isinstance(_v, (int, float)) else 0 - if not valid_chunks: - print( - f"[graphify merge-chunks] error: no valid chunks to merge; " - f"refusing to write {out_path}", - file=sys.stderr, + raise ValueError("hook action must be install, uninstall, or status") + elif cmd == "cache-check": + _cache_check(args) + elif cmd == "merge-chunks": + _merge_chunks(args) + elif cmd == "add": + from graphify.ingest import ingest + + if not args or args[0].startswith("-"): + raise ValueError("add requires a URL") + target = Path(_option(args, "--dir") or "raw") + saved = ingest( + args[0], target, + author=_option(args, "--author"), + contributor=_option(args, "--contributor"), ) - sys.exit(1) - out_path.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path, merged, ensure_ascii=False) - chunk_summary = ( - f"{valid_chunks} chunks" - if valid_chunks == len(chunk_files) - else f"{valid_chunks} of {len(chunk_files)} chunks" - ) - print( - f"Merged {chunk_summary}: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " - f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" - ) - - elif cmd == "merge-semantic": - # graphify merge-semantic --cached --new --out - # Merges cached semantic results with freshly-extracted chunk results. - # Deduplicates nodes by id (cached entries take priority over new ones). - if len(sys.argv) < 3: - print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) - sys.exit(1) - cached_path: Path | None = None - new_path: Path | None = None - out_path2: Path | None = None - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): - cached_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): - new_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path2 = Path(sys.argv[i + 1]); i += 2 + print(f"Saved to {saved}") + elif cmd in {"diagnose", "merge-driver"}: + raise ValueError(f"{cmd} was removed with the obsolete JSON graph format") + else: + # A bare path remains shorthand for extract. + if Path(cmd).exists(): + _update_or_extract("extract", [cmd, *args]) else: - i += 1 - if not out_path2: - print("error: --out required", file=sys.stderr) - sys.exit(1) - empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty - seen_ids2: set[str] = set() - all_nodes: list[dict] = [] - for n in cached_data.get("nodes", []) + new_data.get("nodes", []): - if n.get("id") not in seen_ids2: - seen_ids2.add(n["id"]) - all_nodes.append(n) - merged2 = { - "nodes": all_nodes, - "edges": cached_data.get("edges", []) + new_data.get("edges", []), - "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), - } - out_path2.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path2, merged2, ensure_ascii=False) - print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): - # User ran `graphify ` directly — treat as `graphify extract `. - # Common when following the PowerShell note in README (`graphify .`) or - # copy-pasting skill invocations without the leading slash. - sys.argv.insert(2, sys.argv[1]) - sys.argv[1] = "extract" - _reenter_main() - else: - print(f"error: unknown command '{cmd}'", file=sys.stderr) - print("Run 'graphify --help' for usage.", file=sys.stderr) - sys.exit(1) + raise ValueError(f"unknown command: {cmd}") + except (ValueError, FileNotFoundError, RuntimeError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/graphify/cluster.py b/graphify/cluster.py index 682210700..fb2195fe9 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,80 +1,19 @@ -"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" +"""Community detection performed directly by native Helix.""" from __future__ import annotations -import contextlib -import inspect -import io -import json -import sys -import networkx as nx +from typing import Any +from graphify.helix.access import degree_map, node_attributes, node_ids -def _suppress_output(): - """Context manager to suppress stdout/stderr during library calls. - graspologic's leiden() emits ANSI escape sequences (progress bars, - colored warnings) that corrupt PowerShell 5.1's scroll buffer on - Windows (see issue #19). Redirecting stdout/stderr to devnull during - the call prevents this without losing any graphify output. - """ - return contextlib.redirect_stdout(io.StringIO()) - - -def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: - """Run community detection. Returns {node_id: community_id}. +def _partition(graph: Any, resolution: float = 1.0) -> dict[Any, int]: + from helixdb.graph import LeidenOptions - Tries Leiden (graspologic) first — best quality. - Falls back to Louvain (built into networkx) if graspologic is not installed. - - resolution > 1.0 → more, smaller communities. - resolution < 1.0 → fewer, larger communities. - - Output from graspologic is suppressed to prevent ANSI escape codes - from corrupting terminal scroll buffers on Windows PowerShell 5.1. - """ - stable = nx.Graph() - stable.add_nodes_from(sorted(G.nodes(), key=str)) - edge_rows = sorted( - G.edges(data=True), - key=lambda row: ( - str(row[0]), - str(row[1]), - json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), - ), - ) - for src, tgt, attrs in edge_rows: - stable.add_edge(src, tgt, **attrs) - - try: - from graspologic.partition import leiden - lsig = inspect.signature(leiden).parameters - kwargs: dict = {} - if "random_seed" in lsig: - kwargs["random_seed"] = 42 - if "trials" in lsig: - kwargs["trials"] = 1 - if "resolution" in lsig: - kwargs["resolution"] = resolution - # Suppress graspologic output to prevent ANSI escape codes from - # corrupting PowerShell 5.1 scroll buffer (issue #19) - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - with _suppress_output(): - result = leiden(stable, **kwargs) - finally: - sys.stderr = old_stderr - return result - except ImportError: - pass - - # Fallback: networkx louvain (available since networkx 2.7). - # Inspect kwargs to stay compatible across NetworkX versions — max_level - # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} - if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: - kwargs["max_level"] = 10 - communities = nx.community.louvain_communities(stable, **kwargs) - return {node: cid for cid, nodes in enumerate(communities) for node in nodes} + result = graph.to_undirected().leiden(LeidenOptions(resolution=resolution)) + return { + node_id: community_id + for community_id, record in enumerate(result.communities) + for node_id in record.node_ids + } _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split @@ -84,7 +23,7 @@ def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: def label_communities_by_hub( - G: nx.Graph, communities: dict[int, list[str]] + G: Any, communities: dict[int, list[str]] ) -> dict[int, str]: """Deterministic, LLM-free community labels: name each community after its highest-degree member — the structural hub — so a report reads ``auth`` / @@ -96,14 +35,15 @@ def label_communities_by_hub( overrides these with richer names. """ labels: dict[int, str] = {} + degrees = degree_map(G) for cid, members in communities.items(): - present = [n for n in members if n in G] + present = [n for n in members if G.contains_node(n)] if not present: labels[cid] = f"Community {cid}" continue # highest degree wins; ties broken by node id (ascending) for determinism - hub = min(present, key=lambda n: (-G.degree(n), str(n))) - name = str(G.nodes[hub].get("label") or hub).strip() + hub = min(present, key=lambda n: (-degrees.get(n, 0), str(n))) + name = str(node_attributes(G, hub).get("label") or hub).strip() if name.endswith("()"): name = name[:-2] labels[cid] = name or f"Community {cid}" @@ -113,7 +53,7 @@ def label_communities_by_hub( def community_member_sigs(communities: dict[int, list[str]]) -> dict[int, str]: """Per-community membership fingerprints: ``{cid: sha256(sorted member ids)}``. - Persisted next to ``.graphify_labels.json`` so a later ``cluster-only`` can tell + Persisted with native community state so a later ``cluster-only`` can tell which communities actually changed since labeling. A cid whose members no longer hash the same is a different community — reusing its old (LLM) label there is the "stale label after re-scoping" bug this guards against. Deterministic; independent @@ -132,7 +72,7 @@ def community_member_sigs(communities: dict[int, list[str]]) -> dict[int, str]: def cluster( - G: nx.Graph, + G: Any, resolution: float = 1.0, exclude_hubs_percentile: float | None = None, ) -> dict[int, list[str]]: @@ -152,32 +92,36 @@ def cluster( majority-vote neighbour community afterwards. Useful for staging/utility super-hubs that inflate god-node rankings (#919). """ - if G.number_of_nodes() == 0: + if G.node_count == 0: return {} - if G.is_directed(): + if G.directed: G = G.to_undirected() - if G.number_of_edges() == 0: - return {i: [n] for i, n in enumerate(sorted(G.nodes))} + if G.edge_count == 0: + return {i: [n] for i, n in enumerate(sorted(node_ids(G), key=repr))} + + all_degrees = degree_map(G) # Compute hub exclusion set before removing anything so degree is based on full graph hub_nodes: set[str] = set() if exclude_hubs_percentile is not None: - degrees = sorted(d for _, d in G.degree()) + degrees = sorted(all_degrees.values()) if degrees: idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) threshold = degrees[idx] - hub_nodes = {n for n, d in G.degree() if d > threshold} + hub_nodes = {n for n, d in all_degrees.items() if d > threshold} # Leiden warns and drops isolates - handle them separately # Also exclude hub nodes from partitioning so they don't pull unrelated # subsystems into the same community excluded = hub_nodes - isolates = [n for n in G.nodes() if G.degree(n) == 0 and n not in excluded] - connected_nodes = [n for n in G.nodes() if G.degree(n) > 0 and n not in excluded] - connected = G.subgraph(connected_nodes) + isolates = [n for n, value in all_degrees.items() if value == 0 and n not in excluded] + connected_nodes = [ + n for n, value in all_degrees.items() if value > 0 and n not in excluded + ] + connected = G.induced_subgraph(connected_nodes) raw: dict[int, list[str]] = {} - if connected.number_of_nodes() > 0: + if connected.node_count > 0: partition = _partition(connected, resolution=resolution) for node, cid in partition.items(): raw.setdefault(cid, []).append(node) @@ -207,7 +151,7 @@ def cluster( next_cid += 1 # Split oversized communities - max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION)) + max_size = max(_MIN_SPLIT_SIZE, int(G.node_count * _MAX_COMMUNITY_FRACTION)) final_communities: list[list[str]] = [] for nodes in raw.values(): if len(nodes) > max_size: @@ -236,10 +180,10 @@ def cluster( return {i: sorted(nodes) for i, nodes in enumerate(final_communities)} -def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]: +def _split_community(G: Any, nodes: list[str]) -> list[list[str]]: """Run a second Leiden pass on a community subgraph to split it further.""" - subgraph = G.subgraph(nodes) - if subgraph.number_of_edges() == 0: + subgraph = G.induced_subgraph(nodes) + if subgraph.edge_count == 0: # No edges - split into individual nodes return [[n] for n in sorted(nodes)] try: @@ -254,18 +198,21 @@ def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]: return [sorted(nodes)] -def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: - """Ratio of actual intra-community edges to maximum possible.""" - n = len(community_nodes) - if n <= 1: +def cohesion_score(graph: Any, community_nodes: list[Any]) -> float: + count = len(community_nodes) + if count <= 1: return 1.0 - subgraph = G.subgraph(community_nodes) - actual = subgraph.number_of_edges() - possible = n * (n - 1) / 2 - return actual / possible if possible > 0 else 0.0 + subgraph = graph.induced_subgraph(community_nodes) + actual = len({ + frozenset((edge.source, edge.target)) + for edge in subgraph.edges() + if edge.source != edge.target + }) + possible = count * (count - 1) / 2 + return round(actual / possible, 2) if possible else 0.0 -def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: +def score_all(G: Any, communities: dict[int, list[str]]) -> dict[int, float]: return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} diff --git a/graphify/dedup.py b/graphify/dedup.py index 0e1341e7c..a47d5d872 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -310,7 +310,7 @@ def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: f"from '{lose_file}'. An ID is derived from the source path plus the " f"entity name, so this one does not identify a single entity and the " f"dropped node is lost. To keep them distinct, run 'graphify extract' " - f"per subfolder and merge with 'graphify merge-graphs'.", + f"per subfolder and register each native store with 'graphify global add'.", file=sys.stderr, ) @@ -581,7 +581,7 @@ def deduplicate_entities( for edge in edges: e = dict(edge) # Tolerate "from"/"to" keys from LLM backends that don't follow the - # schema exactly — build_from_json normalises later but dedup runs + # schema exactly — build_from_extraction normalises later but dedup runs # first so bracket access would KeyError here (#803). # Use explicit key presence check (not `or`) so empty-string src/tgt # aren't silently replaced by the fallback key. @@ -591,7 +591,7 @@ def deduplicate_entities( continue e["source"] = remap.get(src, src) e["target"] = remap.get(tgt, tgt) - # Remove legacy keys so they don't leak into edge attrs in graph.json. + # Remove legacy keys so they don't leak into edge attrs in native graph. e.pop("from", None) e.pop("to", None) if e["source"] != e["target"]: diff --git a/graphify/detect.py b/graphify/detect.py index 62efad012..451b92f28 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1231,7 +1231,7 @@ def _wc(path: Path) -> int: # directory subtree is silently skipped). That turns a transient # PermissionError, or a directory created/deleted mid-walk (e.g. concurrent # writes racing the scan), into a partial file list and, downstream, a - # silently partial graph.json. Record and surface every skipped directory + # silently partial native graph. Record and surface every skipped directory # so an incomplete enumeration is visible rather than silent. walk_errors: list[str] = [] diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..26d2135ad 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -9,9 +9,6 @@ from pathlib import Path from typing import Any -import networkx as nx - - _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -161,8 +158,8 @@ def diagnose_extraction( max_examples: int = 5, extract_path: str | Path | None = None, ) -> dict[str, Any]: - """Summarize same-endpoint edge-collapse risk for one JSON graph/extraction dict.""" - from graphify.build import build_from_json + """Summarize same-endpoint edge-collapse risk for an extraction payload.""" + from graphify.build import build_from_extraction node_ids = _node_ids(extraction) raw_edges = _edge_list(extraction) @@ -170,7 +167,7 @@ def diagnose_extraction( # Code-typed semantic nodes the extractor could not verify against the source # it read (#1949): likely-inferred (or hallucinated) symbols surfaced from a - # document. Count them so the flag on graph.json nodes is actually surfaced. + # document. Count them so the verification flag is surfaced. unverified_node_count = sum( 1 for n in extraction.get("nodes", []) if isinstance(n, dict) and n.get("verification") == "unverified" @@ -234,10 +231,10 @@ def diagnose_extraction( post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) - graph_type = type(graph).__name__ - post_build_edge_count = graph.number_of_edges() - post_build_node_count = graph.number_of_nodes() + graph = build_from_extraction(graph_input, directed=directed, root=root) + graph_type = graph.kind + post_build_edge_count = graph.edge_count + post_build_node_count = graph.node_count except Exception as exc: build_error = f"{type(exc).__name__}: {exc}" @@ -277,56 +274,6 @@ def diagnose_extraction( } -def _read_json_file(path: str | Path) -> dict[str, Any]: - """Read a JSON graph after applying Graphify's graph-load size cap.""" - from graphify.security import check_graph_file_size_cap - - json_path = Path(path) - check_graph_file_size_cap(json_path) - try: - data = json.loads(json_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot parse {json_path}: {exc}. " - "The file may be corrupted — re-run 'graphify extract'." - ) from exc - if not isinstance(data, dict): - raise ValueError("diagnostic input must be a JSON object") - return data - - -def diagnose_file( - path: str | Path, - *, - directed: bool | None = None, - root: str | Path | None = None, - max_examples: int = 5, - extract_path: str | Path | None = None, -) -> dict[str, Any]: - """Diagnose a graph/extraction JSON file without mutating it. - - When `directed` is None, the JSON's "directed" flag is honored. Raw - extraction JSON that has no "directed" flag defaults to directed analysis. - """ - data = _read_json_file(path) - if directed is None: - raw_directed = data.get("directed") - effective_directed = raw_directed if isinstance(raw_directed, bool) else True - else: - effective_directed = directed - - summary = diagnose_extraction( - data, - directed=effective_directed, - root=root, - max_examples=max_examples, - extract_path=extract_path, - ) - summary["input_path"] = str(path) - summary["effective_directed"] = effective_directed - return summary - - def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: return { "schema_version": 1, @@ -339,7 +286,7 @@ def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: "producer_suppression": summary.get("producer_suppression", {}), "notes": [ "Diagnostics are read-only.", - "A normal graph.json is already post-build and cannot recover raw producer edges.", + "An active Helix generation is post-build and cannot recover raw producer edges.", "Producer suppression sites are heuristic source-code evidence.", ], } @@ -350,7 +297,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: lines = [ "[graphify] MultiDiGraph edge-collapse diagnostic", f"input: {summary.get('input_path', '')}", - "input_stage: provided JSON (normal graph.json is post-build)", + "input_stage: extraction payload", f"effective_directed: {summary.get('effective_directed', '')}", f"nodes: {summary['node_count']}", f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", @@ -400,7 +347,5 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"locations={example['source_locations']} " f"contexts={example['contexts']}" ) - lines.append( - "note: normal graph.json is post-build; raw producer loss must be measured earlier." - ) + lines.append("note: raw producer loss must be measured before native ingestion.") return "\n".join(lines) diff --git a/graphify/export.py b/graphify/export.py index e1f2caa99..7f0569233 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1,4 +1,4 @@ -# write graph to HTML, JSON, SVG, GraphML, Obsidian vault, and Neo4j Cypher +# write native graph snapshots to presentation and graph-database formats from __future__ import annotations import hashlib import html as _html @@ -6,13 +6,12 @@ import math import os import re -import shutil import sys from collections import Counter -from datetime import date from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph +from typing import Any + +from .helix.model import GraphBuildData, edge_attributes, graphify_attributes, node_attributes from graphify.security import sanitize_label from graphify.analyze import _node_community_map from graphify.build import edge_data @@ -20,81 +19,13 @@ from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 -# Artifacts worth preserving across rebuilds (non-regenerable without LLM or curation). -_BACKUP_ARTIFACTS = [ - "graph.json", - "GRAPH_REPORT.md", - ".graphify_labels.json", - ".graphify_analysis.json", - "manifest.json", - ".graphify_semantic_marker", - "cost.json", -] - - -def backup_if_protected(out_dir: Path) -> "Path | None": - """Snapshot graph artifacts to a dated subfolder before an overwrite. +def _portable_identity(value: Any) -> str: + """Lossless string identity for formats without Helix typed IDs.""" + from helixdb.graph import external_id_to_json - Triggers when graph.json exists AND either: - - .graphify_semantic_marker is present (graph cost real LLM tokens), or - - .graphify_labels.json contains at least one non-default community label - (graph has been curated by a human or skill). - - Returns the backup folder path, or None if no backup was taken. - Never raises — backup failure prints a warning but never blocks the write. - Set GRAPHIFY_NO_BACKUP=1 to disable. - """ - if os.environ.get("GRAPHIFY_NO_BACKUP"): - return None - out = Path(out_dir) - if not (out / "graph.json").exists(): - return None - - is_semantic = (out / ".graphify_semantic_marker").exists() - is_curated = False - labels_file = out / ".graphify_labels.json" - if labels_file.exists(): - try: - labels = json.loads(labels_file.read_text(encoding="utf-8")) - is_curated = any(v != f"Community {k}" for k, v in labels.items()) - except Exception: - pass - - if not is_semantic and not is_curated: - return None - - reason = "+".join(filter(None, ["semantic" if is_semantic else "", "curated" if is_curated else ""])) - today = date.today().isoformat() - backup_dir = out / today - graph_src = out / "graph.json" - - # Skip re-copying if today's backup already has identical graph.json content. - # If content differs (graph changed since the last backup today), overwrite - # the backup in place — one folder per day, always the latest pre-overwrite state. - if backup_dir.exists() and (backup_dir / "graph.json").exists(): - src_hash = hashlib.sha256(graph_src.read_bytes()).hexdigest() - bak_hash = hashlib.sha256((backup_dir / "graph.json").read_bytes()).hexdigest() - if src_hash == bak_hash: - return backup_dir # identical content, nothing to do - - try: - backup_dir.mkdir(parents=True, exist_ok=True) - copied = 0 - for name in _BACKUP_ARTIFACTS: - src = out / name - if src.exists(): - try: - shutil.copy2(src, backup_dir / name) - copied += 1 - except Exception: - pass - if copied: - print(f"[graphify] backed up {reason} graph ({copied} files) -> {backup_dir.name}/") - return backup_dir - except Exception as exc: - import sys - print(f"[graphify] warning: backup failed ({exc}) - continuing with overwrite", file=sys.stderr) - return None + return json.dumps( + external_id_to_json(value), sort_keys=True, separators=(",", ":") + ) def _obsidian_tag(name: str) -> str: """Sanitize a community name for use as an Obsidian tag. @@ -159,15 +90,36 @@ def _yaml_str(s: str) -> str: _CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} -def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: +def _edge_rows(graph: Any, node_id: Any | None = None): + records = graph.edges() if node_id is None else ( + graph.edge(edge_id) for edge_id in graph.incident_edge_ids(node_id) + ) + for edge in records: + if edge is not None: + yield edge.source, edge.target, edge_attributes(edge), edge + + +def _edge_attributes(graph: Any, source: Any, target: Any) -> dict: + edge_ids = graph.edges_between(source, target) + edge = graph.edge(edge_ids[0]) if edge_ids else None + return edge_attributes(edge) if edge is not None else {} + + +def _graph_attributes(graph: Any) -> dict: + metadata = dict(graph.attributes) + value = metadata.get("graph", {}) + return dict(value) if isinstance(value, dict) else {} + + +def attach_hyperedges(G: GraphBuildData, hyperedges: list) -> None: """Store hyperedges in the graph's metadata dict.""" - existing = G.graph.get("hyperedges", []) + existing = G.attributes.get("hyperedges", []) seen_ids = {h["id"] for h in existing} for h in hyperedges: if h.get("id") and h["id"] not in seen_ids: existing.append(h) seen_ids.add(h["id"]) - G.graph["hyperedges"] = existing + G.attributes["hyperedges"] = existing def _git_head() -> str | None: @@ -180,147 +132,6 @@ def _git_head() -> str | None: return None -# Sentinel: an existing graph.json is present and non-empty but cannot be parsed -# into a node count (corrupt, mid-write, or structurally wrong). The caller must -# fail CLOSED on this — the same way to_json's #479 guard refuses to overwrite -# such a file — because we cannot prove the new graph isn't a silent shrink. -MALFORMED_GRAPH = object() - - -def existing_graph_node_count(path: "str | Path"): - """Node count of an existing graph.json. - - Returns: - - an ``int`` node count when the file parses; - - ``None`` when there is verifiably nothing to protect — absent, empty, or - over the size cap (matching how :func:`to_json` lets the new graph - replace an empty/oversized file); - - :data:`MALFORMED_GRAPH` when the file is present and non-empty but - unparseable — the caller must treat this as fail-closed (refuse to - overwrite), mirroring to_json's #479 handling of a corrupt/mid-write file. - - The raw ``--no-cluster`` write path uses this to apply the same #479 shrink - guard that :func:`to_json` applies inline for the clustered path. - """ - p = Path(path) - if not p.exists(): - return None - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(p) - except Exception: - # Oversized: reading it to compare would be the DoS the cap guards against. - return None - try: - raw = p.read_text(encoding="utf-8") - except Exception: - # Present but unreadable: fail closed if it has bytes, else nothing to lose. - try: - return MALFORMED_GRAPH if p.stat().st_size > 0 else None - except Exception: - return None - if not raw.strip(): - return None - try: - data = json.loads(raw) - except Exception: - return MALFORMED_GRAPH - nodes = data.get("nodes") if isinstance(data, dict) else None - return len(nodes) if isinstance(nodes, list) else MALFORMED_GRAPH - - -def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False, built_at_commit: str | None = None, community_labels: dict[int, str] | None = None) -> bool: - # Safety check: refuse to silently shrink an existing graph (#479) - existing_path = Path(output_path) - if not force and existing_path.exists(): - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(existing_path) - except Exception: - # Existing graph.json trips the size cap; reading it to compare would - # be the very DoS the cap guards against. Can't verify — let the new - # graph replace the oversized file. - oversized = True - else: - oversized = False - if not oversized: - try: - raw = existing_path.read_text(encoding="utf-8") - except Exception: - raw = "" - if not raw.strip(): - # Empty/whitespace existing file (e.g. a freshly touched path): - # no nodes to lose, so any new graph is a growth — proceed. - existing_n = 0 - else: - try: - existing_data = json.loads(raw) - existing_n = len(existing_data.get("nodes", [])) - except Exception as exc: - # Non-empty but unparseable existing graph (corrupt or a - # mid-write): we cannot verify the new graph is not a silent - # shrink. Fail SAFE — refuse rather than overwrite. A - # fail-OPEN here (the prior behavior) is the silent data-loss - # path #479 exists to prevent: a transiently unreadable - # graph.json would let a partial rebuild clobber a good one. - import sys as _sys - print( - f"[graphify] WARNING: existing {existing_path} could not be " - f"read to verify the new graph is not smaller ({exc}). " - f"Refusing to overwrite; pass force=True to override.", - file=_sys.stderr, - ) - return False - new_n = G.number_of_nodes() - if new_n < existing_n: - import sys as _sys - print( - f"[graphify] WARNING: new graph has {new_n} nodes but existing " - f"graph.json has {existing_n} (net -{existing_n - new_n}). " - f"Refusing to overwrite. Possible causes: missing chunk files from " - f"a previous session, or fuzzy dedup collapsed same-named symbols " - f"across files during an --update on an already-current graph. " - f"Run a full rebuild (/graphify .) to be safe, or pass force=True " - f"only if you have verified the reduction is legitimate.", - file=_sys.stderr, - ) - return False - - node_community = _node_community_map(communities) - _labels: dict[int, str] = {int(k): v for k, v in (community_labels or {}).items()} - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - for node in data["nodes"]: - cid = node_community.get(node["id"]) - node["community"] = cid - if cid is not None and _labels: - node["community_name"] = _labels.get(cid, f"Community {cid}") - node["norm_label"] = _strip_diacritics(node.get("label", "")).lower() - for link in data["links"]: - if "confidence_score" not in link: - conf = link.get("confidence", "EXTRACTED") - link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) - # Restore original edge direction. Undirected NetworkX storage may - # canonicalize endpoint order, flipping `calls` and other directional - # edges in graph.json. The build path stashes the true endpoints in - # _src/_tgt for exactly this purpose (#563). - true_src = link.pop("_src", None) - true_tgt = link.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - link["source"] = true_src - link["target"] = true_tgt - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - commit = built_at_commit if built_at_commit is not None else _git_head() - if commit: - data["built_at_commit"] = commit - from graphify.paths import write_json_atomic - # Atomic write: a crash/ENOSPC mid-write must not truncate a good graph.json. - write_json_atomic(output_path, data, indent=2) - return True - - def prune_dangling_edges(graph_data: dict) -> tuple[dict, int]: """Remove edges whose source or target node is not in the node set. @@ -381,25 +192,27 @@ def _cypher_label(raw: str, fallback: str) -> str: return cleaned -def to_cypher(G: nx.Graph, output_path: str) -> None: +def to_cypher(G: Any, output_path: str) -> None: lines = ["// Neo4j Cypher import - generated by /graphify", ""] - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) label = _cypher_escape(data.get("label", node_id)) - node_id_esc = _cypher_escape(node_id) + node_id_esc = _cypher_escape(_portable_identity(node_id)) ftype = _cypher_label( (data.get("file_type", "unknown") or "unknown").capitalize(), "Entity", ) lines.append(f"MERGE (n:{ftype} {{id: '{node_id_esc}', label: '{label}'}});") lines.append("") - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _cypher_label( (data.get("relation", "RELATES_TO") or "RELATES_TO").upper(), "RELATES_TO", ) conf = _cypher_escape(data.get("confidence", "EXTRACTED")) - u_esc = _cypher_escape(u) - v_esc = _cypher_escape(v) + u_esc = _cypher_escape(_portable_identity(u)) + v_esc = _cypher_escape(_portable_identity(v)) lines.append( f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" @@ -429,7 +242,7 @@ def _cap_filename(s: str, limit: int = 200) -> str: return f"{truncated}_{digest}" -def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: +def _dedup_node_filenames(G: Any, safe_name) -> dict[Any, str]: """Map each node_id to a unique note filename, appending a numeric suffix on collision. The collision set is keyed on the lowercased name so two labels differing only by case (e.g. "References" vs "references") still get distinct @@ -439,7 +252,8 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: silently overwrites a node whose literal label is already "base_1".""" node_filenames: dict[str, str] = {} used: set[str] = set() - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) base = safe_name(data.get("label", node_id)) candidate = base n = 1 @@ -452,7 +266,7 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: def to_obsidian( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_dir: str, community_labels: dict[int, str] | None = None, @@ -515,7 +329,7 @@ def safe_name(label: str) -> str: # Helper: compute dominant confidence for a node across all its edges def _dominant_confidence(node_id: str) -> str: confs = [] - for u, v, edata in G.edges(node_id, data=True): + for u, v, edata, _ in _edge_rows(G, node_id): confs.append(edata.get("confidence", "EXTRACTED")) if not confs: return "EXTRACTED" @@ -531,7 +345,8 @@ def _dominant_confidence(node_id: str) -> str: # Write one .md file per node node_notes_written = 0 - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) label = data.get("label", node_id) cid = node_community.get(node_id) community_name = ( @@ -571,7 +386,10 @@ def _dominant_confidence(node_id: str) -> str: neighbors = list(G.neighbors(node_id)) if neighbors: lines.append("## Connections") - for neighbor in sorted(neighbors, key=lambda n: G.nodes[n].get("label", n)): + for neighbor in sorted( + neighbors, + key=lambda n: str(node_attributes(G, n).get("label", n)), + ): edata = edge_data(G, node_id, neighbor) neighbor_label = node_filename[neighbor] relation = edata.get("relation", "") @@ -592,7 +410,8 @@ def _dominant_confidence(node_id: str) -> str: inter_community_edges: dict[int, dict[int, int]] = {} for cid in communities: inter_community_edges[cid] = {} - for u, v in G.edges(): + for edge in G.edges(): + u, v = edge.source, edge.target cu = node_community.get(u) cv = node_community.get(v) if cu is not None and cv is not None and cu != cv: @@ -642,7 +461,7 @@ def _community_name(cid) -> str: # synthesized/merge-artifact ids). Dereferencing those via G.nodes[n] or # node_filename[n] raises KeyError and aborts the whole vault export, so # skip dangling members rather than crashing (issue #1236). - members = [m for m in all_members if m in G and m in node_filename] + members = [m for m in all_members if G.contains_node(m) and m in node_filename] n_members = len(members) coh_value = cohesion.get(cid) if cohesion else None @@ -672,8 +491,10 @@ def _community_name(cid) -> str: # Members section lines.append("## Members") - for node_id in sorted(members, key=lambda n: G.nodes[n].get("label", n)): - data = G.nodes[node_id] + for node_id in sorted( + members, key=lambda n: str(node_attributes(G, n).get("label", n)) + ): + data = node_attributes(G, node_id) node_label = node_filename[node_id] ftype = data.get("file_type", "") source = data.get("source_file", "") @@ -706,7 +527,7 @@ def _community_name(cid) -> str: # Top bridge nodes - highest degree nodes that connect to other communities bridge_nodes = [ - (node_id, G.degree(node_id), _community_reach(node_id)) + (node_id, G.degree(node_id).degree, _community_reach(node_id)) for node_id in members if _community_reach(node_id) > 0 ] @@ -783,7 +604,7 @@ def _community_name(cid) -> str: def to_canvas( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -818,8 +639,8 @@ def safe_name(label: str) -> str: # analysis sidecar) the grid below produces nothing and the canvas is written # as an empty 32-byte shell on an otherwise populated graph. Emit every node # into one synthetic community so the canvas always reflects the graph (#1324). - if not communities and G.number_of_nodes() > 0: - communities = {0: [str(n) for n in G.nodes()]} + if not communities and G.node_count > 0: + communities = {0: [node.id for node in G.nodes()]} num_communities = len(communities) cols = math.ceil(math.sqrt(num_communities)) if num_communities > 0 else 1 @@ -844,7 +665,7 @@ def safe_name(label: str) -> str: # Skip dangling community members with no backing node / filename, so box # sizing matches the cards actually laid out and `G.nodes[m]` never # KeyErrors below — mirrors the to_obsidian guard (#1236). - members = [m for m in communities[cid] if m in G and m in node_filenames] + members = [m for m in communities[cid] if G.contains_node(m) and m in node_filenames] n = len(members) inner_cols = max(1, math.ceil(math.sqrt(n))) w = max(600, 220 * inner_cols) @@ -919,16 +740,21 @@ def safe_name(label: str) -> str: inner_cols = group_cols[cid] # Same dangling-member guard as the sizing loop and to_obsidian (#1236): # a community id absent from G / node_filenames would KeyError the sort. - members = [m for m in members if m in G and m in node_filenames] - sorted_members = sorted(members, key=lambda n: G.nodes[n].get("label", n)) + members = [m for m in members if G.contains_node(m) and m in node_filenames] + sorted_members = sorted( + members, key=lambda n: str(node_attributes(G, n).get("label", n)) + ) for m_idx, node_id in enumerate(sorted_members): col = m_idx % inner_cols row = m_idx // inner_cols nx_x = gx + 20 + col * (180 + 20) nx_y = gy + 80 + row * (60 + 20) - fname = node_filenames.get(node_id, safe_name(G.nodes[node_id].get("label", node_id))) + fallback_name = safe_name( + str(node_attributes(G, node_id).get("label", node_id)) + ) + fname = node_filenames.get(node_id, fallback_name) canvas_nodes.append({ - "id": f"n_{node_id}", + "id": f"n_{_portable_identity(node_id)}", "type": "file", "file": f"{fname}.md", "x": nx_x, @@ -939,7 +765,7 @@ def safe_name(label: str) -> str: # Generate edges - only between nodes both in canvas, cap at 200 highest-weight all_edges_weighted: list[tuple[float, str, str, str]] = [] - for u, v, edata in G.edges(data=True): + for u, v, edata, _ in _edge_rows(G): if u in all_canvas_nodes and v in all_canvas_nodes: weight = edata.get("weight", 1.0) relation = edata.get("relation", "") @@ -950,9 +776,9 @@ def safe_name(label: str) -> str: all_edges_weighted.sort(key=lambda x: -x[0]) for weight, u, v, label in all_edges_weighted[:200]: canvas_edges.append({ - "id": f"e_{u}_{v}", - "fromNode": f"n_{u}", - "toNode": f"n_{v}", + "id": f"e_{_portable_identity(u)}_{_portable_identity(v)}", + "fromNode": f"n_{_portable_identity(u)}", + "toNode": f"n_{_portable_identity(v)}", "label": label, }) @@ -961,7 +787,7 @@ def safe_name(label: str) -> str: def to_graphml( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, ) -> None: @@ -970,61 +796,66 @@ def to_graphml( Community IDs are written as a node attribute so Gephi can colour by community. Edge confidence (EXTRACTED/INFERRED/AMBIGUOUS) is preserved as an edge attribute. """ - H = G.copy() + import xml.etree.ElementTree as ET + node_community = _node_community_map(communities) - for node_id in H.nodes(): - H.nodes[node_id]["community"] = node_community.get(node_id, -1) - # Drop internal markers (e.g. the AST-provenance "_origin" tag, #1116, and - # the "_src"/"_tgt" direction markers) — they are persistence/runtime details, - # not graph data, and should not leak into the exported file. - for _, attrs in H.nodes(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - for _, _, attrs in H.edges(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - # nx.write_graphml only accepts scalar attribute values: None raises, and a - # dict/list value (e.g. a per-node `metadata` dict, or the graph-level - # `hyperedges` list set by attach_hyperedges()) raises - # "GraphML does not support type as data values" (#1831). - # Coerce None -> "" and non-scalars -> a JSON string, across all three scopes. - def _graphml_safe(val): - if val is None: - return "" - if isinstance(val, bool) or isinstance(val, (int, float, str)): - return val # GraphML-native scalars pass through unchanged - try: - return json.dumps(val, default=str, sort_keys=True) - except (TypeError, ValueError): - return str(val) - - for key, val in list(H.graph.items()): - H.graph[key] = _graphml_safe(val) - for node_id in H.nodes(): - for key, val in list(H.nodes[node_id].items()): - H.nodes[node_id][key] = _graphml_safe(val) - for u, v in H.edges(): - for key, val in list(H.edges[u, v].items()): - H.edges[u, v][key] = _graphml_safe(val) - - # Write atomically: a mid-serialization error otherwise leaves a 0-byte - # .graphml on disk that downstream tooling mistakes for a completed export - # (#1831). Write to a sibling temp file, then replace on success. + root = ET.Element("graphml", xmlns="http://graphml.graphdrawing.org/xmlns") + nodes = [ + ( + node.id, + { + **{k: v for k, v in graphify_attributes(node.attributes).items() if not k.startswith("_")}, + "community": node_community.get(node.id, -1), + }, + ) + for node in G.nodes() + ] + edges = [ + ( + edge.source, + edge.target, + {k: v for k, v in edge_attributes(edge).items() if not k.startswith("_")}, + ) + for edge in G.edges() + ] + keys = sorted( + {key for _, values in nodes for key in values} + | {key for _, _, values in edges for key in values} + ) + for key in keys: + ET.SubElement(root, "key", attrib={ + "id": str(key), "for": "all", "attr.name": str(key), "attr.type": "string" + }) + graph_element = ET.SubElement( + root, "graph", edgedefault="directed" if G.directed else "undirected" + ) + for node_id, values in nodes: + element = ET.SubElement(graph_element, "node", id=_portable_identity(node_id)) + for key, value in sorted(values.items()): + data = ET.SubElement(element, "data", key=str(key)) + data.text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True) + for index, (source, target, values) in enumerate(edges): + element = ET.SubElement( + graph_element, + "edge", + id=f"e{index}", + source=_portable_identity(source), + target=_portable_identity(target), + ) + for key, value in sorted(values.items()): + data = ET.SubElement(element, "data", key=str(key)) + data.text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True) out = Path(output_path) tmp = out.with_name(out.name + ".tmp") try: - nx.write_graphml(H, str(tmp)) - os.replace(str(tmp), str(out)) + ET.ElementTree(root).write(tmp, encoding="utf-8", xml_declaration=True) + os.replace(tmp, out) finally: - if tmp.exists(): - try: - tmp.unlink() - except OSError: - pass + tmp.unlink(missing_ok=True) def to_svg( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -1051,16 +882,23 @@ def to_svg( ax.set_facecolor("#1a1a2e") ax.axis("off") - pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1)) + from helixdb.graph import LayoutOptions - degree = dict(G.degree()) + pos = { + point.node_id: (point.x, point.y) + for point in G.spring_layout(LayoutOptions( + seed=42, k=2.0 / (G.node_count ** 0.5 + 1) + )) + } + degree = {row.node_id: int(row.degree) for row in G.degrees()} max_deg = max(degree.values(), default=1) or 1 - node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()] - node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()] + node_ids = [node.id for node in G.nodes()] + node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in node_ids] + node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in node_ids] # Draw edges - dashed for non-EXTRACTED - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edge_rows(G): conf = data.get("confidence", "EXTRACTED") style = "solid" if conf == "EXTRACTED" else "dashed" alpha = 0.6 if conf == "EXTRACTED" else 0.3 @@ -1069,11 +907,25 @@ def to_svg( ax.plot([x0, x1], [y0, y1], color="#aaaaaa", linewidth=0.8, linestyle=style, alpha=alpha, zorder=1) - nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors, - node_size=node_sizes, alpha=0.9) - nx.draw_networkx_labels(G, pos, ax=ax, - labels={n: G.nodes[n].get("label", n) for n in G.nodes()}, - font_size=7, font_color="white") + ax.scatter( + [pos[node][0] for node in node_ids], + [pos[node][1] for node in node_ids], + c=node_colors, + s=node_sizes, + alpha=0.9, + zorder=2, + ) + for node in node_ids: + ax.text( + pos[node][0], + pos[node][1], + str(node_attributes(G, node).get("label", node)), + fontsize=7, + color="white", + ha="center", + va="center", + zorder=3, + ) # Legend if community_labels: diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..0b78f8c93 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -2,12 +2,13 @@ from __future__ import annotations from graphify.analyze import _node_community_map -import networkx as nx +from graphify.helix.model import edge_attributes, graphify_attributes import re +from typing import Any def push_to_neo4j( - G: nx.Graph, + G: Any, uri: str, user: str, password: str, @@ -42,7 +43,8 @@ def _safe_label(label: str) -> str: edges_pushed = 0 with driver.session() as session: - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") @@ -59,7 +61,8 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() @@ -78,7 +81,7 @@ def _safe_label(label: str) -> str: return {"nodes": nodes_pushed, "edges": edges_pushed} def push_to_falkordb( - G: nx.Graph, + G: Any, uri: str, user: str | None = None, password: str | None = None, @@ -141,7 +144,8 @@ def _safe_label(label: str) -> str: nodes_pushed = 0 edges_pushed = 0 - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") @@ -157,7 +161,8 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 59c0e52e3..1d7df09c6 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -6,7 +6,9 @@ import html as _html from graphify.analyze import _node_community_map import json -import networkx as nx +from typing import Any + +from graphify.helix.model import edge_attributes, graphify_attributes from graphify.security import sanitize_label @@ -323,7 +325,7 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: """ def to_html( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -340,91 +342,34 @@ def to_html( If member_counts is provided (aggregated community view), node sizes are based on community member counts rather than graph degree. - If node_limit is set and the graph exceeds it, automatically builds an - aggregated community-level meta-graph instead of raising ValueError. + Large graphs must be exported with a non-browser format. """ limit = node_limit if node_limit is not None else _viz_node_limit() - if G.number_of_nodes() > limit: - if node_limit is not None: - # Build aggregated community meta-graph - from collections import Counter as _Counter - import networkx as _nx - print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) - edge_counts = _Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") - if meta.number_of_nodes() <= 1: - print("Single community - aggregated view not useful. Skipping graph.html.") - return - meta_communities = {cid: [str(cid)] for cid in communities} - mc = {cid: len(members) for cid, members in communities.items()} - # Remap hyperedges from semantic node IDs to community IDs - raw_hyperedges = G.graph.get("hyperedges", []) - if raw_hyperedges: - remapped = [] - for he in raw_hyperedges: - he_members = he.get("nodes", []) - comm_ids, seen = [], set() - for nid in he_members: - c = node_to_community.get(nid) - if c is None: - continue - s = str(c) - if s in seen: - continue - seen.add(s) - comm_ids.append(s) - if len(comm_ids) < 2: - continue - remapped.append({ - "id": he.get("id", ""), - "label": he.get("label") or he.get("relation", "").replace("_", " "), - "nodes": comm_ids, - }) - meta.graph["hyperedges"] = remapped - to_html(meta, meta_communities, output_path, - community_labels=community_labels, member_counts=mc) - print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") - print("Tip: run with --obsidian for full node-level detail.") - return + if G.node_count > limit: raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"Graph has {G.node_count} nodes - too large for HTML viz " f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " f"or reduce input size." ) node_community = _node_community_map(communities) - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} max_deg = max(degree.values(), default=1) or 1 max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 - # Work-memory overlay (derived sidecar). When not passed explicitly, load it - # best-effort from the sibling .graphify_learning.json next to the output - # graph.html (which lives beside graph.json). Empty/missing => no learning + # Work-memory overlay comes from the active native generation. When it is not + # passed explicitly, leave it empty; graph.html remains a presentation export. # fields, so the un-annotated render is byte-identical to pre-feature. if learning_overlay is None: learning_overlay = {} - try: - from graphify.reflect import load_learning_overlay as _llo - learning_overlay = _llo(Path(output_path)) - except Exception: - learning_overlay = {} # Status -> ring color. preferred=green, contested=amber. Tentative gets no # ring (it's not yet trustworthy enough to highlight in the map). _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} # Build nodes list for vis.js vis_nodes = [] - for node_id, data in G.nodes(data=True): + for record in G.nodes(): + node_id, data = record.id, graphify_attributes(record.attributes) cid = node_community.get(node_id, 0) color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] label = sanitize_label(data.get("label", node_id)) @@ -482,19 +427,15 @@ def to_html( node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) vis_nodes.append(node) - # Build edges list. Restore original edge direction from _src/_tgt - # (stashed by build.py for exactly this reason): undirected NetworkX - # canonicalizes endpoint order, which would otherwise flip the arrow - # for `calls` and `rationale_for` in the rendered graph (#563). + # Build edges directly from native records, whose endpoints retain direction. vis_edges = [] - for u, v, data in G.edges(data=True): + for record in G.edges(): + u, v, data = record.source, record.target, edge_attributes(record) confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") - true_src = data.get("_src", u) - true_tgt = data.get("_tgt", v) vis_edges.append({ - "from": true_src, - "to": true_tgt, + "from": u, + "to": v, "label": relation, "title": _html.escape(f"{relation} [{confidence}]"), "dashes": confidence != "EXTRACTED", @@ -518,9 +459,10 @@ def _js_safe(obj) -> str: nodes_json = _js_safe(vis_nodes) edges_json = _js_safe(vis_edges) legend_json = _js_safe(legend_data) - hyperedges_json = _js_safe(getattr(G, "graph", {}).get("hyperedges", [])) + metadata = dict(G.attributes).get("graph", {}) + hyperedges_json = _js_safe(metadata.get("hyperedges", []) if isinstance(metadata, dict) else []) title = _html.escape(sanitize_label(str(output_path))) - stats = f"{G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities" + stats = f"{G.node_count} nodes · {G.edge_count} edges · {len(communities)} communities" html = f""" diff --git a/graphify/extract.py b/graphify/extract.py index e5a48dc04..fb2c235c9 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -958,7 +958,7 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st A Swift ``import CoreKit`` names a module, not a file path, so — unlike the file-resolving JS/TS handlers — there is no existing node for the edge to point at. The returned ``(id, label)`` pairs let the extractor materialize a - ``type=module`` anchor node so the edge survives; without it ``build_from_json`` + ``type=module`` anchor node so the edge survives; without it ``build_from_extraction`` prunes every Swift import edge as a dangling/external reference (#1327). """ modules: list[tuple[str, str]] = [] @@ -1280,7 +1280,7 @@ def extract_svelte(path: Path) -> dict: existing_ids = {n["id"] for n in result.get("nodes", [])} # Source file node ID must match the one _extract_generic creates: # _make_id(str(path)) - single arg, no stem prefix. Otherwise the source - # endpoint is a phantom node and build_from_json drops the edge (#701). + # endpoint is a phantom node and build_from_extraction drops the edge (#701). file_node_id = _make_id(str(path)) aliases = _load_tsconfig_aliases(path.parent) for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src): @@ -1307,7 +1307,7 @@ def extract_svelte(path: Path) -> dict: stub_source_file = str(resolved_alias) else: # Bare/scoped import (node_modules) - use last segment; - # build_from_json drops as external if no matching node exists. + # build_from_extraction drops as external if no matching node exists. module_name = raw.split("/")[-1] if not module_name: continue @@ -2969,7 +2969,7 @@ def _key(label: str) -> str: # ── Pascal / Delphi extractor ───────────────────────────────────────────────── -# Size cap for project XML files we parse with stdlib ElementTree. +# Size cap for project XML files parsed with defusedxml. # Real .csproj/.fsproj/.vbproj/.lpk files are well under 2 MiB; anything # larger is either malformed or hostile. _PROJECT_XML_MAX_BYTES = 2 * 1024 * 1024 @@ -2978,7 +2978,7 @@ def _key(label: str) -> str: def _project_xml_is_safe(src: bytes) -> bool: """Reject XML that declares DTDs or entities. - Stdlib ``xml.etree.ElementTree`` does not cap entity expansion, so a + XML parsing also has an explicit input cap, so a crafted project file could trigger a billion-laughs style DoS. External entity resolution is already disabled by pyexpat defaults, but rejecting `` dict: - package --contains--> listed unit """ try: - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET src = path.read_bytes() except OSError as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -3116,7 +3116,7 @@ def extract_slnx(path: Path) -> dict: Project="..."/>`` children. Unlike .sln there are no GUIDs -- projects are identified by their path. """ - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -3195,7 +3195,7 @@ def _resolve(proj_path: str) -> str: def extract_csproj(path: Path) -> dict: """Extract packages, project refs, and target framework from a .csproj/.fsproj/.vbproj.""" - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -3706,7 +3706,7 @@ def add_member(label: str, line_no: int, context: str) -> None: def extract_xaml(path: Path) -> dict: """Extract WPF/XAML structure, bindings, x:Class, and event handler references.""" - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -4235,43 +4235,22 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: ProcessPoolExecutor. Args: - args: (index, path_str, root_str, cache_location_str) tuple. ``root`` - anchors hash keys / node ids / the XAML boundary; ``cache_location`` - is where the cache dir is written, decoupled per #1774. A legacy - 3-tuple (no cache_location) is still accepted for back-compat. + args: ``(index, path_str, root_str)``. Cache lookup and persistence stay + in the parent process because the cache is part of Helix state. Returns: (index, result_dict) so results can be placed back in order. """ - if len(args) == 4: - idx, path_str, root_str, cache_location_str = args - else: # legacy 3-tuple: location == anchor - idx, path_str, root_str = args - cache_location_str = root_str + idx, path_str, root_str = args[:3] path = Path(path_str) root = Path(root_str) - cache_location = Path(cache_location_str) _raise_recursion_limit() - bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES - - # Check cache first (avoid re-extraction) - if not bypass_cache: - cached = load_cached(path, root, cache_root=cache_location) - if cached is not None: - return idx, cached extractor = _get_extractor(path) if extractor is None: return idx, {"nodes": [], "edges": []} result = _safe_extract_with_xaml_root(extractor, path, root) - # Never cache a zero-node result for an extractable file. Every supported - # source produces at least a file node, so an empty node list is anomalous - # (e.g. a transient batch/parallel hiccup). Caching it makes the empty - # byte-stable across runs and silently blinds affected/explain to and - # through the file (#1666); skipping the write lets a rerun self-heal. - if not bypass_cache and "error" not in result and result.get("nodes"): - save_cached(path, result, root, cache_root=cache_location) return idx, result @@ -4281,7 +4260,7 @@ def _extract_parallel( root: Path, max_workers: int | None, total_files: int, - cache_location: Path | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> bool: """Extract uncached files in parallel using ProcessPoolExecutor. @@ -4318,11 +4297,9 @@ def _extract_parallel( max_workers = min(max_workers, 61) max_workers = max(max_workers, 1) - # root anchors hash keys / node ids / XAML boundary; cache_location is where - # the cache dir is written (defaults to root when not decoupled) (#1774). root_str = str(root) - cache_loc_str = str(cache_location if cache_location is not None else root) - work_items = [(idx, str(path), root_str, cache_loc_str) for idx, path in uncached_work] + work_items = [(idx, str(path), root_str) for idx, path in uncached_work] + paths_by_index = {idx: path for idx, path in uncached_work} done_count = 0 _PROGRESS_INTERVAL = 100 @@ -4336,6 +4313,13 @@ def _extract_parallel( try: idx, result = future.result() per_file[idx] = result + path = paths_by_index[idx] + if ( + path.suffix not in _JS_CACHE_BYPASS_SUFFIXES + and "error" not in result + and result.get("nodes") + ): + save_cached(path, result, root, cache=cache) except Exception as exc: pos = futures[future] print( @@ -4384,7 +4368,7 @@ def _extract_sequential( per_file: list[dict | None], root: Path, total_files: int, - cache_location: Path | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> None: """Extract uncached files sequentially (fallback for small batches).""" _PROGRESS_INTERVAL = 100 @@ -4407,7 +4391,7 @@ def _extract_sequential( result = _safe_extract_with_xaml_root(extractor, path, root) # See _extract_single_file: don't cache an anomalous zero-node result (#1666). if not bypass_cache and "error" not in result and result.get("nodes"): - save_cached(path, result, root, cache_root=cache_location) + save_cached(path, result, root, cache=cache) per_file[idx] = result if total_files >= _PROGRESS_INTERVAL: # Consistent denominator with the intermediate lines (#1693). @@ -4425,6 +4409,7 @@ def extract( root: Path | None = None, parallel: bool = True, max_workers: int | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> dict: """Extract AST nodes and edges from a list of code files. @@ -4439,14 +4424,13 @@ def extract( symbol resolution. Pass the SCAN root whenever the cache lives somewhere else (`--out`); without it the anchor falls back to cache_root and every scanned file reads as out-of-root (#1941). - cache_root: explicit root for graphify-out/cache/ (overrides the - inferred common path prefix). Pass Path('.') when running on a - subdirectory so the cache stays at ./graphify-out/cache/. - Anchors ids/source_file only as a fallback when `root` is unset. + cache_root: deprecated compatibility anchor. No filesystem cache is + read or written. parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, use ProcessPoolExecutor for multi-core extraction. max_workers: max subprocess count. Defaults to cpu_count (or the value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). + cache: mutable extraction-cache mapping loaded from Helix state. """ paths = [Path(p) for p in paths] anchor_root = Path(root) if root is not None else None @@ -4483,13 +4467,6 @@ def extract( root = cache_root root = root.resolve() - # #1774: the cache is an OUTPUT, so when no explicit cache_root is given it is - # written under the current working directory — never `root` (the inferred - # common parent of the inputs), which would drop graphify-out/ inside a - # read-only or foreign corpus. `root` still anchors the content-hash keys, - # node ids, symbol resolution, and the XAML project-scan boundary; only the - # cache directory's location diverges from it. - cache_location = (cache_root if cache_root is not None else Path(".")).resolve() total = len(paths) # Phase 1: separate cached hits from uncached work @@ -4502,7 +4479,7 @@ def extract( continue bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES if not bypass_cache: - cached = load_cached(path, root, cache_root=cache_location) + cached = load_cached(path, root, cache=cache) if cached is not None: per_file[i] = cached continue @@ -4513,10 +4490,10 @@ def extract( ran_parallel = False if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD: ran_parallel = _extract_parallel( - uncached_work, per_file, root, max_workers, total, cache_location + uncached_work, per_file, root, max_workers, total, cache ) if not ran_parallel: - _extract_sequential(uncached_work, per_file, root, total, cache_location) + _extract_sequential(uncached_work, per_file, root, total, cache) # Fill any remaining None slots (shouldn't happen, but defensive) for i in range(total): @@ -4620,7 +4597,7 @@ def extract( _merge_decl_def_classes(all_nodes, all_edges) # Remap file node IDs from absolute-path-derived to the canonical - # {parent_dir}_{stem} spec form so (a) graph.json edge endpoints are stable + # {parent_dir}_{stem} spec form so (a) native graph edge endpoints are stable # across machines (#502) and (b) AST file nodes match the IDs semantic # subagents generate (#1033). Resolve before relativizing so paths passed in # relative form still anchor to the (resolved) root. @@ -5205,7 +5182,7 @@ def _has_import_evidence(candidate_id: str) -> bool: # Relativize source_file fields so paths are portable across machines (#555). # A target OUTSIDE the scan root (an out-of-root ProjectReference/.sln/bash # `source`) can't be made relative to root; leaving it absolute leaked the - # scan path including the OS username into a committed graph.json (#1899). + # scan path including the OS username into a committed native graph (#1899). # Fall back to a walk-up relative form, or the bare basename when that would # still embed foreign path segments (a far-away or cross-drive target). When # the node's id was itself minted from the absolute path, remap it to a @@ -5257,20 +5234,21 @@ def _portable_out_of_root_sf(p: Path) -> str: # origin_file is an internal disambiguation hint (#1462): the colliding-id pass # above reads it to keep same-named cross-file stubs distinct, after which nothing - # consumes it. Drop it from the returned nodes so it never ships into graph.json as + # consumes it. Drop it from the returned nodes so it never ships into native graph as # an absolute, machine-specific path — the same "no absolute paths in output" # contract that relativizes source_file just above (#555, #932). The per-file AST # cache keeps its own copy, which is what the colliding-id pass reads on a cache hit. for n in all_nodes: n.pop("origin_file", None) - n.pop("_callable", None) # internal indirect_call marker — never ships to graph.json + n.pop("_callable", None) # internal indirect_call marker — never ships to native graph # local_alias is a transient import-resolution hint (#2082), same shape as # target_file (#1814): it exists only so the module arm of # _resolve_python_member_calls (run above via run_language_resolvers) can # match an aliased receiver against the import edge it came from. Nothing # reads it after that pass runs, so drop it here rather than let an internal - # local variable name ship into graph.json. Popped post-resolution, unlike + # local variable name ship into persistent graph state. Popped + # post-resolution, unlike # target_file (which _disambiguate_colliding_node_ids pops earlier in the # pipeline) — local_alias must survive until run_language_resolvers has run, # so it cannot be popped at that earlier point without breaking the fix. diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index 723606e64..a70e50740 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -12,7 +12,7 @@ def _csharp_namespace_id(dotted_name: str) -> str: - digest = hashlib.sha1(dotted_name.encode("utf-8")).hexdigest()[:16] + digest = hashlib.sha1(dotted_name.encode("utf-8"), usedforsecurity=False).hexdigest()[:16] return f"csharp_namespace:{digest}" REFERENCE_CONTEXTS = frozenset({ @@ -2324,7 +2324,7 @@ def walk(node, parent_class_nid: str | None = None) -> None: # Module-level import handlers (Swift) name a module, not a file # path, so there is no pre-existing node to anchor the edge to. # They return (id, label) pairs for which we materialize a - # `type=module` node; otherwise build_from_json prunes every such + # `type=module` node; otherwise build_from_extraction prunes every such # import edge as a dangling/external reference. The same module # imported from N files shares one id (file_type=code keeps # build.py validation happy; `type=module` exempts it from diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index e88e36372..cd779c7b0 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -628,7 +628,9 @@ def _disambiguate_colliding_node_ids( if not source_key: continue if source_key in needs_hash: - salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + salt = hashlib.sha1( + source_key.encode("utf-8"), usedforsecurity=False + ).hexdigest()[:6] new_id = _make_id(source_key, old_id, salt) else: new_id = naive.get(source_key) or _make_id(source_key, old_id) @@ -640,7 +642,7 @@ def _disambiguate_colliding_node_ids( # No colliding ids to salt apart, but the transient `target_file` hint an # importer stamps on every resolved import (#1814) still has to be dropped # here — this early exit skips the edge loop below, so without it a - # non-colliding import would carry its absolute path into graph.json. + # non-colliding import would carry its absolute path into native storage. for edge in edges: edge.pop("target_file", None) return @@ -684,7 +686,7 @@ def _disambiguate_colliding_node_ids( # target file, key the target salt by THAT file so the salt lands on the # correct sibling. Generalizes the #1475 C/ObjC header carve-out (below) to # every language and to re_exports. `pop` it as we consume it: this is the - # hint's only reader, and its absolute path must not persist into graph.json. + # hint's only reader, and its absolute path must not persist into Helix. target_file = edge.pop("target_file", None) if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"): target_edge_key = _source_key(str(target_file), root) @@ -1653,8 +1655,9 @@ def _resolve_python_module_path(module_name: str, current_path: Path, root: Path break # left the scan root; stop walking up if anc == root: continue # already probed root/rel above - # Only probe sys.path-root candidates — dirs that are NOT themselves part - # of a package. Probing a package dir would resolve an absolute + # Only probe interpreter search-root candidates — dirs that are NOT + # themselves part of a package. Probing a package dir would resolve an + # absolute # `from helpers import x` to a sibling in the current package (Python-2 # implicit-relative semantics), fabricating edges to what may be an # external dependency (#2072 review). A src-layout root (src/, no diff --git a/graphify/extractors/sln.py b/graphify/extractors/sln.py index 936ff9fff..76fb71b96 100644 --- a/graphify/extractors/sln.py +++ b/graphify/extractors/sln.py @@ -37,7 +37,7 @@ def extract_sln(path: Path) -> dict: # A solution folder is a VIRTUAL grouping, not a file: Visual Studio writes # its name as both the display name and the "path" (proj_name == proj_path, # no real file). Resolving it to an absolute path and keying the node id off - # that leaked the absolute scan path (incl. the OS username) into graph.json, + # that leaked the absolute scan path (incl. the OS username) into native graph, # because the CLI's id-relativization only remaps ids of real files in the # scan set — a virtual folder never matches, so its absolute id survived # (#1789). Use the folder name itself (relative, no filesystem resolution). diff --git a/graphify/global_graph.py b/graphify/global_graph.py index eddd0c92a..da0ee051a 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -1,184 +1,164 @@ +"""Cross-project aggregation streamed directly between embedded Helix stores.""" + from __future__ import annotations -import json -import hashlib -import sys + +import copy from datetime import datetime, timezone from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph as _jg - -_GLOBAL_DIR = Path.home() / ".graphify" -_GLOBAL_GRAPH = _GLOBAL_DIR / "global-graph.json" -_GLOBAL_MANIFEST = _GLOBAL_DIR / "global-manifest.json" - - -def _load_manifest() -> dict: - if _GLOBAL_MANIFEST.exists(): - try: - return json.loads(_GLOBAL_MANIFEST.read_text(encoding="utf-8")) - except Exception as exc: - # Don't silently wipe the user's manifest on a parse error: that - # deletes every tracked repo. Back the bad file up and surface the - # error so the user can recover or report it. - backup = _GLOBAL_MANIFEST.with_suffix( - _GLOBAL_MANIFEST.suffix + f".corrupt.{int(datetime.now(timezone.utc).timestamp())}" +from typing import Iterable + +from .helix.persistence import DEFAULT_GLOBAL_STORE, HelixEmbeddedStore +from .helix.state import new_state + + +def _project_store(path: str | Path) -> Path: + candidate = Path(path).expanduser().resolve() + if candidate.name == "graph.helix": + store = candidate + elif (candidate / "graph.helix").is_dir(): + store = candidate / "graph.helix" + else: + store = candidate / "graphify-out" / "graph.helix" + if not store.is_dir(): + if candidate.suffix.lower() == ".json" or candidate.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; rebuild the project and pass " + "its graph.helix store" ) - try: - _GLOBAL_MANIFEST.rename(backup) - print( - f"[graphify global] manifest at {_GLOBAL_MANIFEST} failed to parse ({exc}); " - f"moved to {backup} and starting fresh. Restore from the backup if this was " - f"unexpected.", - file=sys.stderr, - ) - except Exception as rename_exc: - print( - f"[graphify global] manifest at {_GLOBAL_MANIFEST} failed to parse ({exc}) " - f"and could not be backed up ({rename_exc}). Starting fresh.", - file=sys.stderr, - ) - return {"version": 1, "repos": {}} - - -def _save_manifest(manifest: dict) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_MANIFEST, manifest, indent=2) - - -def _load_global_graph() -> nx.Graph: - if _GLOBAL_GRAPH.exists(): - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(_GLOBAL_GRAPH) - data = json.loads(_GLOBAL_GRAPH.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - return _jg.node_link_graph(data, edges="links") - except TypeError: - return _jg.node_link_graph(data) - return nx.Graph() - - -def _save_global_graph(G: nx.Graph) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - try: - data = _jg.node_link_data(G, edges="links") - except TypeError: - data = _jg.node_link_data(G) - from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_GRAPH, data, indent=2) - - -def _file_hash(path: Path) -> str: - h = hashlib.sha256() - h.update(path.read_bytes()) - return h.hexdigest()[:16] - - -def global_add(source_path: Path, repo_tag: str) -> dict: - """Add or update a project graph in the global graph. - - Returns a summary dict with keys: repo_tag, nodes_added, nodes_removed, skipped. - Skipped=True means the source graph hasn't changed since last add. - """ - from graphify.build import prefix_graph_for_global, prune_repo_from_graph - - if not source_path.exists(): - raise FileNotFoundError(f"graph not found: {source_path}") - - manifest = _load_manifest() - src_hash = _file_hash(source_path) - - existing = manifest["repos"].get(repo_tag, {}) - existing_path = existing.get("source_path", "") - if existing_path and existing_path != str(source_path.resolve()): - print( - f"[graphify global] warning: repo tag '{repo_tag}' previously pointed to " - f"{existing_path!r}, now updating to {str(source_path.resolve())!r}. " - f"Use --as to give it a different name.", - file=sys.stderr, - ) - if existing.get("source_hash") == src_hash: - return {"repo_tag": repo_tag, "nodes_added": 0, "nodes_removed": 0, "skipped": True} - - # Load source graph - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(source_path) - data = json.loads(source_path.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - src_G = _jg.node_link_graph(data, edges="links") - except TypeError: - src_G = _jg.node_link_graph(data) - - # Prefix IDs for cross-project isolation - prefixed = prefix_graph_for_global(src_G, repo_tag) - - # Load global graph and prune stale nodes for this repo - G = _load_global_graph() - removed = prune_repo_from_graph(G, repo_tag) - - # Merge external-library nodes (no source_file) by label to avoid duplication - external_labels = { - d.get("label", ""): n - for n, d in G.nodes(data=True) - if not d.get("source_file") and d.get("label") - } - # Map each deduplicated external onto the existing global node so that - # edges incident to it can be rewired instead of dropped. - remap = {} - for node, data in prefixed.nodes(data=True): - if not data.get("source_file") and data.get("label") in external_labels: - remap[node] = external_labels[data["label"]] - - # Compose: add prefixed nodes (except deduplicated externals) into global graph - for node, data in prefixed.nodes(data=True): - if node not in remap: - G.add_node(node, **data) - for u, v, data in prefixed.edges(data=True): - u = remap.get(u, u) - v = remap.get(v, v) - if u != v: # don't introduce self-loops via remapping - G.add_edge(u, v, **data) - - added = prefixed.number_of_nodes() - len(remap) - _save_global_graph(G) - - manifest["repos"][repo_tag] = { - "added_at": datetime.now(timezone.utc).isoformat(), - "source_path": str(source_path.resolve()), - "node_count": added, - "edge_count": prefixed.number_of_edges(), - "source_hash": src_hash, - } - _save_manifest(manifest) + raise FileNotFoundError(f"Helix store not found: {store}") + return store - return {"repo_tag": repo_tag, "nodes_added": added, "nodes_removed": removed, "skipped": False} +def _state() -> dict: + if not DEFAULT_GLOBAL_STORE.is_dir(): + return new_state(build={"kind": "global-aggregate"}) + with HelixEmbeddedStore(DEFAULT_GLOBAL_STORE, read_only=True) as store: + return copy.deepcopy(store.read_state()) -def global_remove(repo_tag: str) -> int: - """Remove all nodes for repo_tag from the global graph. Returns count removed.""" - from graphify.build import prune_repo_from_graph - manifest = _load_manifest() - if repo_tag not in manifest["repos"]: - raise KeyError(f"repo '{repo_tag}' not in global graph") +def _repos(state: dict) -> dict: + global_state = state.setdefault("global", {}) + return global_state.setdefault("repos", {}) + - G = _load_global_graph() - removed = prune_repo_from_graph(G, repo_tag) - _save_global_graph(G) +def _source_records(state: dict) -> list[tuple[Path, str, str]]: + records: list[tuple[Path, str, str]] = [] + for repo, value in sorted(_repos(state).items()): + if not isinstance(value, dict): + raise RuntimeError(f"global repository record {repo!r} is invalid") + source = Path(str(value.get("source_path", ""))).expanduser().resolve() + generation = value.get("source_generation") + if not source.is_dir() or not isinstance(generation, str): + raise RuntimeError( + f"global repository {repo!r} source is unavailable; re-add it" + ) + records.append((source, generation, repo)) + return records + + +def _save_state(state: dict, *, retain_rollback: bool) -> None: + with HelixEmbeddedStore( + DEFAULT_GLOBAL_STORE, retain_rollback=retain_rollback + ) as store: + store.save_aggregate_sources(_source_records(state), state) + + +def global_add( + source_path: Path, + repo_tag: str, + *, + retain_rollback: bool = False, +) -> dict: + """Add or replace a project by rebuilding the aggregate through native pages.""" + source = _project_store(source_path) + with HelixEmbeddedStore(source, read_only=True) as store: + source_generation = store.active_generation + metadata = store._metadata(source_generation) + state = _state() + repos = _repos(state) + existing = repos.get(repo_tag, {}) + if ( + existing.get("source_path") == str(source) + and existing.get("source_generation") == source_generation + ): + return { + "repo_tag": repo_tag, + "nodes_added": 0, + "nodes_removed": 0, + "skipped": True, + } + removed = int(existing.get("node_count", 0)) if isinstance(existing, dict) else 0 + repos[repo_tag] = { + "added_at": datetime.now(timezone.utc).isoformat(), + "source_path": str(source), + "source_generation": source_generation, + "node_count": int(metadata.get("node_count", 0)), + "edge_count": int(metadata.get("edge_count", 0)), + } + _save_state(state, retain_rollback=retain_rollback) + return { + "repo_tag": repo_tag, + "nodes_added": int(metadata.get("node_count", 0)), + "nodes_removed": removed, + "skipped": False, + } - del manifest["repos"][repo_tag] - _save_manifest(manifest) + +def global_remove(repo_tag: str, *, retain_rollback: bool = False) -> int: + if not DEFAULT_GLOBAL_STORE.is_dir(): + raise KeyError(f"repo '{repo_tag}' not in global graph") + state = _state() + repos = _repos(state) + if repo_tag not in repos: + raise KeyError(f"repo '{repo_tag}' not in global graph") + record = repos.pop(repo_tag) + removed = int(record.get("node_count", 0)) if isinstance(record, dict) else 0 + _save_state(state, retain_rollback=retain_rollback) return removed def global_list() -> dict: - """Return the manifest repos dict.""" - return _load_manifest().get("repos", {}) + if not DEFAULT_GLOBAL_STORE.is_dir(): + return {} + return dict(_repos(_state())) def global_path() -> Path: - return _GLOBAL_GRAPH + return DEFAULT_GLOBAL_STORE + + +def aggregate( + project_stores: Iterable[str | Path], + output: str | Path = DEFAULT_GLOBAL_STORE, + *, + retain_rollback: bool = False, +) -> Path: + """Create an aggregate through bounded public Helix reads and writes.""" + sources: list[tuple[Path, str, str]] = [] + state = new_state(build={"kind": "global-aggregate"}) + repos = _repos(state) + tags: dict[str, int] = {} + for raw_path in project_stores: + path = _project_store(raw_path) + base = path.parent.parent.name or "project" + tags[base] = tags.get(base, 0) + 1 + repo = base if tags[base] == 1 else f"{base}-{tags[base]}" + with HelixEmbeddedStore(path, read_only=True) as store: + generation = store.active_generation + metadata = store._metadata(generation) + repos[repo] = { + "source_path": str(path), + "source_generation": generation, + "node_count": int(metadata.get("node_count", 0)), + "edge_count": int(metadata.get("edge_count", 0)), + } + sources.append((path, generation, repo)) + destination = Path(output).expanduser().resolve() + with HelixEmbeddedStore( + destination, retain_rollback=retain_rollback + ) as store: + store.save_aggregate_sources(sources, state) + return destination + + +__all__ = ["aggregate", "global_add", "global_list", "global_path", "global_remove"] diff --git a/graphify/helix/__init__.py b/graphify/helix/__init__.py new file mode 100644 index 000000000..c32a6b891 --- /dev/null +++ b/graphify/helix/__init__.py @@ -0,0 +1,31 @@ +"""Native, revision-pinned embedded Helix runtime for Graphify.""" + +from .model import EdgeData, GraphBuildData, LoadedGraph, NodeData +from .native import NativeBackendUnavailable, native_backend_info +from .persistence import ( + DEFAULT_GLOBAL_STORE, + DEFAULT_PROJECT_STORE, + HelixEmbeddedStore, + HelixGraphReader, + graph_storage_exists, + load_graph, + persist_graph, + persist_graph_data, +) + +__all__ = [ + "HelixEmbeddedStore", + "HelixGraphReader", + "DEFAULT_GLOBAL_STORE", + "DEFAULT_PROJECT_STORE", + "NativeBackendUnavailable", + "EdgeData", + "GraphBuildData", + "LoadedGraph", + "NodeData", + "graph_storage_exists", + "load_graph", + "native_backend_info", + "persist_graph", + "persist_graph_data", +] diff --git a/graphify/helix/access.py b/graphify/helix/access.py new file mode 100644 index 000000000..abe1cc465 --- /dev/null +++ b/graphify/helix/access.py @@ -0,0 +1,76 @@ +"""Small projections over the native immutable Helix graph surface. + +These helpers keep Graphify's attribute conventions in one place. Callers use +native Helix records and algorithms directly. +""" + +from __future__ import annotations + +from typing import Any, Iterable + +from .model import edge_attributes, graphify_attributes, node_attributes + + +def node_ids(graph: Any) -> tuple[Any, ...]: + return tuple(node.id for node in graph.nodes()) + + +def node_rows(graph: Any) -> tuple[tuple[Any, dict[str, Any]], ...]: + return tuple((node.id, graphify_attributes(node.attributes)) for node in graph.nodes()) + + +def edge_rows( + graph: Any, node_id: Any | None = None +) -> tuple[tuple[Any, Any, dict[str, Any], Any], ...]: + records: Iterable[Any] + if node_id is None: + records = graph.edges() + else: + records = ( + edge + for edge_id in graph.incident_edge_ids(node_id) + if (edge := graph.edge(edge_id)) is not None + ) + return tuple( + (edge.source, edge.target, edge_attributes(edge), edge) + for edge in records + ) + + +def degree_map(graph: Any) -> dict[Any, int]: + return {row.node_id: int(row.degree) for row in graph.degrees()} + + +def degree(graph: Any, node_id: Any) -> int: + return int(graph.degree(node_id).degree) + + +def first_edge_attributes(graph: Any, source: Any, target: Any) -> dict[str, Any]: + edge_ids = graph.edges_between(source, target) + if not edge_ids and not graph.directed: + edge_ids = graph.edges_between(target, source) + edge = graph.edge(edge_ids[0]) if edge_ids else None + return edge_attributes(edge) if edge is not None else {} + + +def all_edge_attributes(graph: Any, source: Any, target: Any) -> list[dict[str, Any]]: + edge_ids = list(graph.edges_between(source, target)) + if not edge_ids and not graph.directed: + edge_ids = list(graph.edges_between(target, source)) + return [ + edge_attributes(edge) + for edge_id in edge_ids + if (edge := graph.edge(edge_id)) is not None + ] + + +__all__ = [ + "all_edge_attributes", + "degree", + "degree_map", + "edge_rows", + "first_edge_attributes", + "node_attributes", + "node_ids", + "node_rows", +] diff --git a/graphify/helix/model.py b/graphify/helix/model.py new file mode 100644 index 000000000..7de1c98f1 --- /dev/null +++ b/graphify/helix/model.py @@ -0,0 +1,228 @@ +"""Graphify's narrow construction and loaded-generation boundaries. + +``GraphBuildData`` is a disposable batch payload. It intentionally has no +adjacency structure or graph algorithms. ``LoadedGraph`` owns the native +immutable Helix snapshot and the durable state selected from the same +generation; it intentionally does not copy topology into Python. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any, Iterable, Literal, Mapping + + +GraphKind = Literal["graph", "digraph", "multigraph", "multidigraph"] + + +def _identity_key(value: Any) -> str: + import json + from helixdb.graph import external_id_to_json + + return json.dumps(external_id_to_json(value), sort_keys=True, separators=(",", ":")) + + +def import_identity(value: Any) -> Any: + """Decode an explicit JSON-export identity, leaving ordinary IDs unchanged.""" + if isinstance(value, dict) and set(value) == {"__helix_external_id_v1"}: + from helixdb.graph import external_id_from_json + + return external_id_from_json(value) + return value + + +def _export_identity(value: Any) -> Any: + if isinstance(value, (bytes, tuple, frozenset)): + from helixdb.graph import external_id_to_json + + return external_id_to_json(value) + return value + + +@dataclass(frozen=True, slots=True) +class NodeData: + id: Any + attributes: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class EdgeData: + source: Any + target: Any + attributes: Mapping[str, Any] = field(default_factory=dict) + key: Any | None = None + + +@dataclass +class GraphBuildData: + """Transient records awaiting one staged Helix generation write.""" + + kind: GraphKind = "graph" + nodes: list[NodeData] = field(default_factory=list) + edges: list[EdgeData] = field(default_factory=list) + attributes: dict[str, Any] = field(default_factory=dict) + extras: dict[str, Any] = field(default_factory=dict) + + @property + def directed(self) -> bool: + return self.kind in {"digraph", "multidigraph"} + + @property + def multigraph(self) -> bool: + return self.kind in {"multigraph", "multidigraph"} + + @property + def node_count(self) -> int: + return len(self.nodes) + + @property + def edge_count(self) -> int: + return len(self.edges) + + @classmethod + def from_node_link(cls, payload: Mapping[str, Any]) -> "GraphBuildData": + if not isinstance(payload, Mapping): + raise TypeError("node-link graph payload must be a mapping") + directed = bool(payload.get("directed", False)) + multigraph = bool(payload.get("multigraph", False)) + kind: GraphKind = ( + "multidigraph" + if directed and multigraph + else "digraph" + if directed + else "multigraph" + if multigraph + else "graph" + ) + raw_nodes = payload.get("nodes", []) + raw_edges = payload.get("links", payload.get("edges", [])) + if not isinstance(raw_nodes, list) or not isinstance(raw_edges, list): + raise TypeError("node-link nodes and links must be lists") + nodes: list[NodeData] = [] + seen: set[str] = set() + for index, raw in enumerate(raw_nodes): + if not isinstance(raw, Mapping) or "id" not in raw: + raise TypeError(f"nodes[{index}] must contain an id") + node_id = import_identity(raw["id"]) + identity_key = _identity_key(node_id) + if identity_key in seen: + raise ValueError(f"duplicate node identifier at nodes[{index}]") + seen.add(identity_key) + nodes.append(NodeData(node_id, {k: v for k, v in raw.items() if k != "id"})) + edges: list[EdgeData] = [] + for index, raw in enumerate(raw_edges): + if not isinstance(raw, Mapping) or "source" not in raw or "target" not in raw: + raise TypeError(f"links[{index}] must contain source and target") + source, target = import_identity(raw["source"]), import_identity(raw["target"]) + if _identity_key(source) not in seen or _identity_key(target) not in seen: + raise ValueError(f"links[{index}] references a missing node") + edges.append( + EdgeData( + source, + target, + {k: v for k, v in raw.items() if k not in {"source", "target", "key"}}, + import_identity(raw.get("key")) if multigraph else None, + ) + ) + reserved = {"directed", "multigraph", "graph", "nodes", "links", "edges", "graphify_state"} + return cls( + kind=kind, + nodes=nodes, + edges=edges, + attributes=dict(payload.get("graph", {})), + extras={k: v for k, v in payload.items() if k not in reserved}, + ) + + def to_node_link( + self, + *, + state: Mapping[str, Any] | None = None, + tagged_identities: bool = False, + ) -> dict[str, Any]: + identity = _export_identity if tagged_identities else lambda value: value + payload: dict[str, Any] = { + "directed": self.directed, + "multigraph": self.multigraph, + "graph": dict(self.attributes), + "nodes": [{"id": identity(node.id), **dict(node.attributes)} for node in self.nodes], + "links": [], + **self.extras, + } + for edge in self.edges: + record = { + "source": identity(edge.source), + "target": identity(edge.target), + **dict(edge.attributes), + } + if self.multigraph: + record["key"] = identity(edge.key) + payload["links"].append(record) + if state is not None: + payload["graphify_state"] = dict(state) + return payload + + +@dataclass(frozen=True) +class LoadedGraph: + """One immutable native topology and durable state from the same generation.""" + + graph: Any + generation: str + state: Mapping[str, Any] + metadata: Mapping[str, Any] + store_path: Path + query: Any | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "state", MappingProxyType(dict(self.state))) + object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata))) + + +def graphify_attributes(attributes: Mapping[str, Any]) -> dict[str, Any]: + """Return selected Graphify attributes from a native record.""" + nested = attributes.get("attrs") + return dict(nested) if isinstance(nested, Mapping) else dict(attributes) + + +def node_attributes(graph: Any, node_id: Any) -> dict[str, Any]: + node = graph.node(node_id) + if node is None: + raise KeyError(node_id) + return graphify_attributes(node.attributes) + + +def edge_attributes(edge: Any) -> dict[str, Any]: + """Project a native semantic label into a transient/output attribute DTO.""" + nested = edge.attributes.get("attrs") + attributes = dict(nested) if isinstance(nested, Mapping) else {} + if edge.weight is not None: + attributes.setdefault("weight", edge.weight) + attributes.setdefault("relation", edge.label) + return attributes + + +def edge_records(graph: Any, edge_ids: Iterable[Any] | None = None) -> tuple[Any, ...]: + if edge_ids is None: + return graph.edges() + records = [] + for edge_id in edge_ids: + edge = graph.edge(edge_id) + if edge is not None: + records.append(edge) + return tuple(records) + + +__all__ = [ + "EdgeData", + "GraphBuildData", + "GraphKind", + "LoadedGraph", + "NodeData", + "edge_attributes", + "edge_records", + "graphify_attributes", + "import_identity", + "node_attributes", +] diff --git a/graphify/helix/native.py b/graphify/helix/native.py new file mode 100644 index 000000000..c3552d19e --- /dev/null +++ b/graphify/helix/native.py @@ -0,0 +1,150 @@ +"""Validated public-SDK boundary for Graphify's embedded Helix runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +import importlib.metadata +import inspect +from pathlib import Path +from typing import Any + +import helixdb + + +HELIX_PACKAGE_INDEX = "https://pypi.org/project/helix-db/0.2.0b4/" +HELIX_PYTHON_VERSION = "0.2.0b4" +HELIX_EMBEDDED_DISTRIBUTION = "helix-db-embedded" +HELIX_EMBEDDED_VERSION = "0.2.0b4" +_DATABASE_NAME = "graphify" +_ID_LEASE_SIZE = 100_000 +_CACHE_DISABLED = helixdb.EmbeddedCacheConfig( + vector_memory_bytes=1, + mode=helixdb.VectorMemoryOnly(), +) +_READ_CACHE = helixdb.EmbeddedCacheConfig( + vector_memory_bytes=1, + mode=helixdb.MemoryCache(), +) + + +class NativeBackendUnavailable(RuntimeError): + """Raised when the pinned embedded Helix SDK cannot be loaded safely.""" + + +@dataclass(frozen=True) +class NativeBackendInfo: + module: str + version: str | None + embedded_version: str + + +@dataclass(frozen=True) +class _NativeSurface: + helixdb_attrs: frozenset[str] + + +_REQUIRED = _NativeSurface( + helixdb_attrs=frozenset( + { + "Client", + "Disk", + "NodeRef", + "ShortestPathDirection", + "SourcePredicate", + "g", + "read_batch", + "write_batch", + "GraphSelection", + "GraphMetadataSelection", + "IdentitySelection", + "GraphEdgeId", + "EmbeddedCacheConfig", + "VectorMemoryOnly", + "MemoryCache", + "LeidenOptions", + "NativeGraph", + } + ), +) + + +@lru_cache(maxsize=1) +def validate_native_backend() -> None: + """Validate the statically imported public SDK and matching embedded wheel.""" + try: + version = importlib.metadata.version("helix-db") + except importlib.metadata.PackageNotFoundError: + version = None + if version != HELIX_PYTHON_VERSION: + raise NativeBackendUnavailable( + f"embedded Helix SDK version mismatch: expected {HELIX_PYTHON_VERSION}, got {version!r}" + ) + + missing = sorted(name for name in _REQUIRED.helixdb_attrs if not hasattr(helixdb, name)) + if missing: + raise NativeBackendUnavailable( + f"helixdb is missing required public embedded SDK APIs: {', '.join(missing)}" + ) + try: + embedded_version = importlib.metadata.version(HELIX_EMBEDDED_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError as exc: + raise NativeBackendUnavailable( + f"{HELIX_EMBEDDED_DISTRIBUTION}=={HELIX_EMBEDDED_VERSION} is required " + "for embedded Helix storage" + ) from exc + if embedded_version != HELIX_EMBEDDED_VERSION: + raise NativeBackendUnavailable( + "embedded Helix payload version mismatch: " + f"expected {HELIX_EMBEDDED_VERSION}, got {embedded_version!r}" + ) + + +def native_backend_info() -> NativeBackendInfo: + validate_native_backend() + return NativeBackendInfo( + module=helixdb.__name__, + version=importlib.metadata.version("helix-db"), + embedded_version=importlib.metadata.version(HELIX_EMBEDDED_DISTRIBUTION), + ) + + +def open_embedded_client( + path: str | Path, + *, + read_only: bool = False, + disable_cache: bool = False, +) -> Any: + """Open the official in-process, on-disk Helix client at ``path``.""" + validate_native_backend() + root = Path(path) + if read_only: + if not root.is_dir(): + raise FileNotFoundError(f"embedded Helix store not found: {root}") + else: + root.mkdir(parents=True, exist_ok=True) + source = helixdb.Disk(str(root.resolve()), _DATABASE_NAME) + cache = _CACHE_DISABLED if disable_cache else _READ_CACHE + if read_only: + return helixdb.Client.embedded_reader(source, cache=cache) + if "id_lease_size" in inspect.signature(helixdb.Client.embedded).parameters: + embedded: Any = helixdb.Client.embedded + return embedded( + source, + cache=cache, + id_lease_size=_ID_LEASE_SIZE, + ) + return helixdb.Client.embedded(source, cache=cache) + + +__all__ = [ + "HELIX_PACKAGE_INDEX", + "HELIX_PYTHON_VERSION", + "HELIX_EMBEDDED_DISTRIBUTION", + "HELIX_EMBEDDED_VERSION", + "NativeBackendInfo", + "NativeBackendUnavailable", + "native_backend_info", + "open_embedded_client", + "validate_native_backend", +] diff --git a/graphify/helix/persistence.py b/graphify/helix/persistence.py new file mode 100644 index 000000000..326103a06 --- /dev/null +++ b/graphify/helix/persistence.py @@ -0,0 +1,3767 @@ +"""Persistent Graphify schema implemented on the official embedded Helix SDK.""" + +from __future__ import annotations + +import base64 +from collections.abc import Iterable, Iterator +from concurrent import futures +from contextlib import contextmanager +import copy +from dataclasses import dataclass +from enum import Enum, auto +import hashlib +import json +import math +import os +from pathlib import Path +import re +import threading +import time +from typing import Any +import unicodedata +import uuid + +import helixdb +from helixdb.dsl import RepeatConfig, SourcePredicate, SubTraversal, g, read_batch +from helixdb.graph import external_id_from_json, external_id_to_json + +from .model import GraphBuildData, LoadedGraph, import_identity +from .native import open_embedded_client, validate_native_backend + + +_SCHEMA_VERSION = 9 +_NODE_LABEL = "GraphifyNode" +_META_LABEL = "GraphifyMeta" +_CONTROL_LABEL = "GraphifyControl" +_STATE_LABEL = "GraphifyState" +_EXTERNAL_KEY = "external_key" +_STORAGE_KEY = "storage_key" +_GENERATION = "graphify_generation" +_CONTROL_KEY = "control_key" +_ACTIVE_GENERATION = "active_generation" +_PREVIOUS_GENERATION = "previous_generation" +_ATTRS = "attrs" +_ORDER = "graphify_order" +_LEGACY_TARGET_KEY = "target_key" +_EDGE_KEY = "edge_key" +_NATIVE_WEIGHT = "graphify_weight" +_EDGE_CONTEXT = "graphify_context" +_RECORD_HASH = "record_hash" +_EDGE_IDENTITY = "edge_identity" +_IDENTITY_BUCKET = "identity_bucket" +_TOPOLOGY_REVISION = "topology_revision" +_NODE_BUCKET_HASHES = "node_bucket_hashes" +_EDGE_BUCKET_HASHES = "edge_bucket_hashes" +_SEARCH_LABEL = "search_label" +_SEARCH_TEXT = "search_text" +_WRITER_LOCK_FILE = ".graphify-writer.lock" +_WRITER_LOCK_TIMEOUT_SECONDS = 120.0 +_WRITE_CHUNK_SIZE = 1_000 +_STAGED_EDGE_WRITE_CHUNK_SIZE = 750 +_STATE_WRITE_CHUNK_SIZE = 1_000 +_BUFFERED_WRITE_CONCURRENCY = 2 +_IDENTITY_BUCKET_COUNT = 1_024 +_DELTA_MAX_MUTATIONS = 2_000 +_DELTA_MAX_RATIO = 0.10 +DEFAULT_MAX_NODES = 1_000_000 +DEFAULT_MAX_EDGES = 5_000_000 +DEFAULT_PROJECT_STORE = Path("graphify-out/graph.helix") +DEFAULT_GLOBAL_STORE = Path.home() / ".graphify" / "global-graph.helix" +_DURABLE_STATE = "graphify_state" +_STATE_TYPE = "$graphify_state_type" +_STATE_KIND = "state_kind" +_STATE_KEY = "state_key" +_STATE_PAYLOAD = "payload" +_STATE_REVISION = "state_revision" +_ACTIVE_STATE_REVISION = "active_state_revision" +_CHECKSUM_MODE = "checksum_mode" +_SPLIT_CHECKSUM_MODE = "topology-state-v1" +_STREAM_CHECKSUM_MODE = "topology-stream-v1" +_TOPOLOGY_CHECKSUM = "topology_checksum" +_STATE_KINDS = ("section", "community", "file", "cache") +_STATE_REVISION_KEYS = {kind: f"active_{kind}_revision" for kind in _STATE_KINDS} +_STATE_CHECKSUM_KEYS = {kind: f"{kind}_state_checksum" for kind in _STATE_KINDS} +_STATE_COUNT_KEYS = {kind: f"{kind}_state_count" for kind in _STATE_KINDS} +_SEARCH_TOKEN_RE = re.compile(r"\w+") + + +@dataclass(frozen=True, slots=True) +class _PreparedNode: + encoded_id: str + external_id: Any + attributes: tuple[tuple[str, Any], ...] + search_label: str + search_text: str + identity_bucket: int + record_hash: str + + +@dataclass(frozen=True, slots=True) +class _PreparedEdge: + source: str + target: str + source_id: Any + target_id: Any + key: Any + relation: str + stored_attributes: tuple[tuple[str, Any], ...] + context: str | None + weight: float | None + stable_identity: str + identity_bucket: int + record_hash: str + + @property + def shape(self) -> tuple[str, bool, bool, bool]: + return ( + self.relation, + self.context is not None, + self.weight is not None, + bool(self.stored_attributes), + ) + + +@dataclass(frozen=True, slots=True) +class _CanonicalEdge: + source: str + target: str + source_id: Any + target_id: Any + key: Any + key_identity: dict[str, Any] + relation: str + stored_attributes: tuple[tuple[str, Any], ...] + context: str | None + weight: float | None + encoded_record: bytes + + +@dataclass(frozen=True, slots=True) +class _PreparedTopology: + directed: bool + multigraph: bool + graph_attributes: dict[str, Any] + extras: dict[str, Any] + nodes: list[_PreparedNode] + edges: list[_PreparedEdge] + checksum: str + node_bucket_hashes: list[str] + edge_bucket_hashes: list[str] + + +@dataclass(frozen=True, slots=True) +class _PreparedState: + encoded: dict[str, Any] | None + records: list[tuple[str, str, Any, int]] + category_records: dict[str, list[tuple[str, str, Any, int]]] + category_checksums: dict[str, str] + category_counts: dict[str, int] + checksum: str + + +@dataclass(frozen=True, slots=True) +class _StoredNode: + internal_id: int + record_hash: str + + +@dataclass(frozen=True, slots=True) +class _StoredEdge: + internal_id: int + record_hash: str + + +@dataclass(frozen=True, slots=True) +class _TopologyDelta: + added_nodes: list[_PreparedNode] + updated_nodes: list[tuple[_StoredNode, _PreparedNode]] + dropped_nodes: list[_StoredNode] + added_edges: list[_PreparedEdge] + dropped_edges: list[_StoredEdge] + + @property + def mutation_count(self) -> int: + return ( + len(self.added_nodes) + + len(self.updated_nodes) + + len(self.dropped_nodes) + + len(self.added_edges) + + len(self.dropped_edges) + ) + + +@dataclass(frozen=True, slots=True) +class _StagedProposal: + graph: Any + prepared: _PreparedTopology + + +@dataclass(frozen=True, slots=True) +class _PublishedStage: + pass + + +@dataclass(slots=True) +class _StagedGraph: + generation: str + state: dict[str, Any] + metadata: dict[str, Any] + store_path: Path + _phase: _StagedProposal | _PublishedStage + query: None = None + + @property + def graph(self) -> Any: + phase = self._phase + if isinstance(phase, _PublishedStage): + raise RuntimeError("staged Helix proposal has already been activated") + return phase.graph + + @property + def prepared(self) -> _PreparedTopology: + phase = self._phase + if isinstance(phase, _PublishedStage): + raise RuntimeError("staged Helix proposal has already been activated") + return phase.prepared + + def mark_published(self) -> None: + if isinstance(self._phase, _PublishedStage): + raise RuntimeError("staged Helix proposal has already been activated") + self._phase = _PublishedStage() + + +@dataclass(slots=True) +class _StagedDeltaGraph(_StagedGraph): + pass + + +@dataclass(slots=True) +class _StagedFullGraph(_StagedGraph): + pass + + +class _DeltaAttempt(Enum): + PUBLISHED = auto() + FALLBACK = auto() + FALLBACK_NATIVE_VALIDATED = auto() + + +def _close_public_client(client: Any) -> None: + """Close the synchronous public SDK outside an active asyncio loop. + + The public b3 client intentionally rejects synchronous embedded calls from + an event-loop thread. Resource finalizers can run on such a thread, so move + the public ``close()`` call to a short-lived worker in that one case. + """ + try: + import asyncio + + asyncio.get_running_loop() + except RuntimeError: + client.close() + return + + errors: list[BaseException] = [] + + def close() -> None: + try: + client.close() + except BaseException as exc: # pragma: no cover - propagated below + errors.append(exc) + + worker = threading.Thread(target=close, name="graphify-helix-close") + worker.start() + worker.join() + if errors: + raise errors[0] + + +def _public_store_rebuild_message(exc: BaseException, path: Path) -> str | None: + """Translate public SDK format/index failures into a source-rebuild action.""" + detail = str(exc) + blockers = ( + "Migration required: writer migration must complete", + "Index lifecycle unavailable for secondary", + ) + if not any(blocker in detail for blocker in blockers): + return None + return ( + f"embedded Helix store at {path} was created by an incompatible public " + "runtime and cannot be upgraded safely; move that graph.helix directory " + "aside and run graphify update from source" + ) + + +class _StoreLock: + """Cross-platform process lock held for the lifetime of an embedded handle.""" + + def __init__( + self, + path: Path, + *, + shared: bool, + timeout: float = _WRITER_LOCK_TIMEOUT_SECONDS, + ) -> None: + self.path = path + self.shared = shared + self.timeout = timeout + self._stream: Any | None = None + + def acquire(self) -> None: + # Helix's embedded reader is snapshot-safe alongside a writer. Only + # serialize competing writers; reader/writer exclusion would prevent + # readers from retaining the previous active generation during staging. + if self.shared: + return + self.path.parent.mkdir(parents=True, exist_ok=True) + stream = self.path.open("rb" if self.shared else "a+b") + if not self.shared and self.path.stat().st_size == 0: + stream.write(b"\0") + stream.flush() + deadline = time.monotonic() + self.timeout + while True: + try: + self._lock(stream) + break + except (BlockingIOError, OSError) as exc: + if time.monotonic() >= deadline: + stream.close() + raise TimeoutError( + f"timed out waiting for embedded Helix store lock {self.path}" + ) from exc + time.sleep(0.05) + if not self.shared: + stream.seek(0) + stream.truncate() + stream.write(f"{os.getpid()}\n".encode("ascii")) + stream.flush() + self._stream = stream + + def _lock(self, stream: Any) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + mode = msvcrt.LK_NBRLCK if self.shared else msvcrt.LK_NBLCK + msvcrt.locking(stream.fileno(), mode, 1) + else: + import fcntl + + mode = fcntl.LOCK_SH if self.shared else fcntl.LOCK_EX + fcntl.flock(stream.fileno(), mode | fcntl.LOCK_NB) + + def release(self) -> None: + stream = self._stream + if stream is None: + return + try: + if os.name == "nt": + import msvcrt + + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + stream.close() + self._stream = None + + +def _json_value(value: Any, context: str) -> Any: + """Validate and normalize a value to the JSON types accepted by Helix.""" + try: + encoded = json.dumps(value, ensure_ascii=False, allow_nan=False) + return json.loads(encoded) + except (TypeError, ValueError) as exc: + raise TypeError(f"{context} must contain JSON-compatible values: {exc}") from exc + + +def _encode_state_value(value: Any) -> Any: + """Encode typed graph identifiers inside otherwise JSON-compatible state.""" + if isinstance(value, bytes): + return {_STATE_TYPE: "bytes", "value": base64.b64encode(value).decode("ascii")} + if isinstance(value, tuple): + return {_STATE_TYPE: "tuple", "value": [_encode_state_value(item) for item in value]} + if isinstance(value, frozenset): + return { + _STATE_TYPE: "frozenset", + "value": [_encode_state_value(item) for item in sorted(value, key=_encode_key)], + } + if isinstance(value, list): + return [_encode_state_value(item) for item in value] + if isinstance(value, dict): + if all(isinstance(key, str) for key in value): + return {key: _encode_state_value(item) for key, item in value.items()} + return { + _STATE_TYPE: "mapping", + "value": [ + [_encode_state_value(key), _encode_state_value(item)] for key, item in value.items() + ], + } + return value + + +def _decode_state_value(value: Any) -> Any: + if isinstance(value, list): + return [_decode_state_value(item) for item in value] + if not isinstance(value, dict): + return value + kind = value.get(_STATE_TYPE) if set(value) == {_STATE_TYPE, "value"} else None + if kind is None: + return {key: _decode_state_value(item) for key, item in value.items()} + raw = value.get("value") + if kind == "bytes" and isinstance(raw, str): + return base64.b64decode(raw, validate=True) + if kind == "tuple" and isinstance(raw, list): + return tuple(_decode_state_value(item) for item in raw) + if kind == "frozenset" and isinstance(raw, list): + return frozenset(_decode_state_value(item) for item in raw) + if kind == "mapping" and isinstance(raw, list): + return { + _decode_state_value(pair[0]): _decode_state_value(pair[1]) + for pair in raw + if isinstance(pair, list) and len(pair) == 2 + } + raise RuntimeError(f"embedded Helix durable state has invalid typed value {kind!r}") + + +def _tagged_key(value: Any) -> dict[str, Any]: + """Return Helix's canonical tagged external-identity envelope.""" + return external_id_to_json(value) + + +def _encode_key(value: Any) -> str: + return json.dumps( + _tagged_key(value), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _decode_tagged_key(value: dict[str, Any]) -> Any: + try: + return external_id_from_json(value) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"embedded Helix graph identifier is invalid: {exc}") from exc + + +def _decode_key(value: str) -> Any: + try: + tagged = json.loads(value) + except (TypeError, json.JSONDecodeError) as exc: + raise RuntimeError("embedded Helix graph contains an invalid encoded identifier") from exc + if not isinstance(tagged, dict): + raise RuntimeError("embedded Helix graph identifier is not a tagged object") + return _decode_tagged_key(tagged) + + +def _decode_identity(value: Any) -> Any: + if not isinstance(value, dict): + raise RuntimeError("embedded Helix identity property is not tagged") + return _decode_tagged_key(value) + + +def _normalize_search_text(value: Any) -> str: + text = unicodedata.normalize("NFKD", str(value or "")).lower() + return "".join(character for character in text if not unicodedata.combining(character)) + + +def _search_properties(node_id: Any, attrs: dict[str, Any]) -> dict[str, str]: + """Denormalize public, searchable node fields for Helix predicates.""" + label = _normalize_search_text(attrs.get("norm_label") or attrs.get("label")) + source = _normalize_search_text(attrs.get("source_file")) + identity = _normalize_search_text(node_id) + tokenized_label = " ".join(part for part in _SEARCH_TOKEN_RE.findall(label)) + tokenized_source = " ".join(part for part in _SEARCH_TOKEN_RE.findall(source)) + return { + _SEARCH_LABEL: label, + _SEARCH_TEXT: "\0".join((label, tokenized_label, identity, source, tokenized_source)), + } + + +def _canonical_bytes(payload: Any) -> bytes: + """Encode one value into Graphify's stable checksum representation.""" + + def encode_non_json(value: Any) -> Any: + if isinstance(value, bytes): + return {"$graphify_bytes": base64.b64encode(value).decode("ascii")} + if isinstance(value, frozenset): + return {"$graphify_frozenset": sorted(_encode_key(item) for item in value)} + raise TypeError(f"cannot checksum {type(value).__name__}") + + return json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + default=encode_non_json, + ).encode("utf-8") + + +def _checksum(payload: Any) -> str: + return f"sha256:{hashlib.sha256(_canonical_bytes(payload)).hexdigest()}" + + +def _record_hash(payload: Any) -> str: + """Return a compact comparison hash for one persisted topology record.""" + return _record_hash_bytes(_canonical_bytes(payload)) + + +def _record_hash_bytes(encoded: bytes) -> str: + return hashlib.blake2b(encoded, digest_size=16).hexdigest() + + +def _edge_identity( + *, + directed: bool, + multigraph: bool, + source: str, + target: str, + key: dict[str, Any], + anonymous_ordinal: int | None = None, +) -> str: + """Return a stable logical identity used only for delta comparison.""" + if not directed and target < source: + source, target = target, source + payload: dict[str, Any] = {"source": source, "target": target} + if multigraph: + payload["key"] = key + if anonymous_ordinal is not None: + payload["anonymous_ordinal"] = anonymous_ordinal + return _record_hash(payload) + + +def _identity_bucket(identity: str) -> int: + digest = hashlib.blake2b(identity.encode("utf-8"), digest_size=8).digest() + return int.from_bytes(digest, "big") % _IDENTITY_BUCKET_COUNT + + +def _identity_bucket_hashes(records: list[tuple[str, str]]) -> list[str]: + grouped: list[list[tuple[str, str]]] = [[] for _ in range(_IDENTITY_BUCKET_COUNT)] + for identity, record_hash in records: + grouped[_identity_bucket(identity)].append((identity, record_hash)) + return ["" if not bucket else _record_hash(sorted(bucket)) for bucket in grouped] + + +def _changed_buckets(old: Any, new: list[str]) -> set[int] | None: + if ( + not isinstance(old, list) + or len(old) != _IDENTITY_BUCKET_COUNT + or any(not isinstance(value, str) for value in old) + ): + return None + return {index for index, (left, right) in enumerate(zip(old, new)) if left != right} + + +def _generation_checksum(topology_checksum: str, state_checksum: str) -> str: + """Bind independently verified topology and state into one generation hash.""" + return _checksum( + { + _TOPOLOGY_CHECKSUM: topology_checksum, + "state_checksum": state_checksum, + } + ) + + +class _TopologyStreamChecksum: + """Order-stable checksum that can be produced from bounded record pages.""" + + def __init__( + self, + *, + directed: bool, + multigraph: bool, + graph: dict[str, Any], + extras: dict[str, Any], + ) -> None: + self._digest = hashlib.sha256() + self._add( + b"header", + _canonical_bytes( + { + "header": { + "directed": directed, + "multigraph": multigraph, + "graph": graph, + "extras": extras, + } + } + ), + ) + + def _add(self, kind: bytes, encoded: bytes) -> None: + self._digest.update(kind) + self._digest.update(len(encoded).to_bytes(8, "big")) + self._digest.update(encoded) + + def node(self, value: dict[str, Any] | bytes) -> None: + encoded = value if isinstance(value, bytes) else _canonical_bytes(value) + self._add(b"node", encoded) + + def edge(self, value: dict[str, Any] | bytes) -> None: + encoded = value if isinstance(value, bytes) else _canonical_bytes(value) + self._add(b"edge", encoded) + + def hexdigest(self) -> str: + return f"sha256:{self._digest.hexdigest()}" + + +def _node_topology_record(node_id: Any, attributes: dict[str, Any]) -> dict[str, Any]: + return {"id": node_id, "attributes": attributes} + + +def _edge_topology_record( + source: Any, + target: Any, + key: Any, + relation: str, + attributes: dict[str, Any], + *, + multigraph: bool, +) -> dict[str, Any]: + record = { + "source": source, + "target": target, + "relation": relation, + "attributes": attributes, + } + if multigraph: + record["key"] = key + return record + + +def _prepare_topology( + graph: GraphBuildData, + *, + max_nodes: int, + max_edges: int, +) -> _PreparedTopology: + """Validate one construction DTO without materializing node-link copies.""" + if len(graph.nodes) > max_nodes or len(graph.edges) > max_edges: + raise ValueError( + "graph exceeds configured embedded ingestion bounds: " + f"{len(graph.nodes)}/{len(graph.edges)} > {max_nodes}/{max_edges}" + ) + graph_attributes = _json_value(dict(graph.attributes), "graph metadata") + extras = _json_value(dict(graph.extras), "graph top-level metadata") + if not isinstance(graph_attributes, dict) or not isinstance(extras, dict): + raise TypeError("graph metadata and top-level extras must be mappings") + + node_ids: set[str] = set() + prepared_nodes: list[tuple[_PreparedNode, bytes]] = [] + for input_index, node in enumerate(graph.nodes): + node_id = import_identity(node.id) + encoded_id = _encode_key(node_id) + if encoded_id in node_ids: + raise ValueError(f"duplicate graph node identifier at nodes[{input_index}]") + node_ids.add(encoded_id) + attributes = _json_value(dict(node.attributes), f"graph nodes[{input_index}] attributes") + if not isinstance(attributes, dict): + raise TypeError(f"graph nodes[{input_index}] attributes must be a mapping") + search = _search_properties(node_id, attributes) + encoded_record = _canonical_bytes(_node_topology_record(node_id, attributes)) + prepared_nodes.append( + ( + _PreparedNode( + encoded_id=encoded_id, + external_id=node_id, + attributes=tuple(attributes.items()), + search_label=search[_SEARCH_LABEL], + search_text=search[_SEARCH_TEXT], + identity_bucket=_identity_bucket(encoded_id), + record_hash=_record_hash_bytes(encoded_record), + ), + encoded_record, + ) + ) + + edge_identities: set[str] = set() + prepared_edges: list[tuple[_PreparedEdge, bytes]] = [] + anonymous_groups: dict[tuple[str, str], list[_CanonicalEdge]] = {} + none_identity = _tagged_key(None) + for input_index, edge in enumerate(graph.edges): + source = import_identity(edge.source) + target = import_identity(edge.target) + encoded_source = _encode_key(source) + encoded_target = _encode_key(target) + if encoded_source not in node_ids or encoded_target not in node_ids: + raise ValueError(f"graph edges[{input_index}] references a missing node") + key = import_identity(edge.key) if graph.multigraph else None + raw_attributes = dict(edge.attributes) + raw_weight = raw_attributes.get("weight") + if raw_weight is not None and ( + isinstance(raw_weight, bool) + or not isinstance(raw_weight, (int, float)) + or not math.isfinite(float(raw_weight)) + ): + raise TypeError(f"graph edges[{input_index}] weight must be finite numeric") + attributes = _json_value(raw_attributes, f"graph edges[{input_index}] attributes") + if not isinstance(attributes, dict): + raise TypeError(f"graph edges[{input_index}] attributes must be a mapping") + relation = attributes.pop("relation", "related_to") + if not isinstance(relation, str) or not relation: + raise TypeError(f"graph edges[{input_index}] relation must be a non-empty string") + key_identity = none_identity if key is None else _tagged_key(key) + context = attributes.get("context") + native_context = context if isinstance(context, str) and context else None + weight = attributes.get("weight") + native_weight = None if weight is None else float(weight) + encoded_record = _canonical_bytes( + _edge_topology_record( + source, + target, + key, + relation, + attributes, + multigraph=graph.multigraph, + ) + ) + canonical = _CanonicalEdge( + source=encoded_source, + target=encoded_target, + source_id=source, + target_id=target, + key=key, + key_identity=key_identity, + relation=relation, + stored_attributes=tuple(item for item in attributes.items() if item[0] != "weight"), + context=native_context, + weight=native_weight, + encoded_record=encoded_record, + ) + if graph.multigraph and key is None: + anonymous_pair = ( + (encoded_source, encoded_target) + if graph.directed or encoded_source <= encoded_target + else (encoded_target, encoded_source) + ) + anonymous_groups.setdefault(anonymous_pair, []).append(canonical) + continue + stable_identity = _edge_identity( + directed=graph.directed, + multigraph=graph.multigraph, + source=canonical.source, + target=canonical.target, + key=canonical.key_identity, + ) + if stable_identity in edge_identities: + raise ValueError(f"duplicate graph edge identity {stable_identity!r}") + edge_identities.add(stable_identity) + prepared_edges.append( + ( + _PreparedEdge( + source=canonical.source, + target=canonical.target, + source_id=canonical.source_id, + target_id=canonical.target_id, + key=canonical.key, + relation=canonical.relation, + stored_attributes=canonical.stored_attributes, + context=canonical.context, + weight=canonical.weight, + stable_identity=stable_identity, + identity_bucket=_identity_bucket(stable_identity), + record_hash=_record_hash_bytes(canonical.encoded_record), + ), + canonical.encoded_record, + ) + ) + + for group in anonymous_groups.values(): + for ordinal, canonical in enumerate(sorted(group, key=lambda item: item.encoded_record)): + stable_identity = _edge_identity( + directed=graph.directed, + multigraph=True, + source=canonical.source, + target=canonical.target, + key=canonical.key_identity, + anonymous_ordinal=ordinal, + ) + assert stable_identity not in edge_identities, "anonymous edge identity must be unique" + edge_identities.add(stable_identity) + prepared_edges.append( + ( + _PreparedEdge( + source=canonical.source, + target=canonical.target, + source_id=canonical.source_id, + target_id=canonical.target_id, + key=None, + relation=canonical.relation, + stored_attributes=canonical.stored_attributes, + context=canonical.context, + weight=canonical.weight, + stable_identity=stable_identity, + identity_bucket=_identity_bucket(stable_identity), + record_hash=_record_hash_bytes(canonical.encoded_record), + ), + canonical.encoded_record, + ) + ) + + prepared_nodes.sort(key=lambda item: item[0].encoded_id) + prepared_edges.sort(key=lambda item: item[0].stable_identity) + + checksum = _TopologyStreamChecksum( + directed=graph.directed, + multigraph=graph.multigraph, + graph=graph_attributes, + extras=extras, + ) + for _, encoded_record in prepared_nodes: + checksum.node(encoded_record) + for _, encoded_record in prepared_edges: + checksum.edge(encoded_record) + nodes = [node for node, _ in prepared_nodes] + edges = [edge for edge, _ in prepared_edges] + + return _PreparedTopology( + directed=graph.directed, + multigraph=graph.multigraph, + graph_attributes=graph_attributes, + extras=extras, + nodes=nodes, + edges=edges, + checksum=checksum.hexdigest(), + node_bucket_hashes=_identity_bucket_hashes( + [(node.encoded_id, node.record_hash) for node in nodes] + ), + edge_bucket_hashes=_identity_bucket_hashes( + [(edge.stable_identity, edge.record_hash) for edge in edges] + ), + ) + + +def _state_revision(metadata: dict[str, Any], kind: str) -> str | None: + revision = metadata.get(_STATE_REVISION_KEYS[kind]) + if not isinstance(revision, str): + revision = metadata.get(_ACTIVE_STATE_REVISION) + return revision if isinstance(revision, str) else None + + +def _state_category_checksum( + records: list[tuple[str, str, Any, int]], +) -> str: + return _checksum( + { + "records": [ + {"kind": kind, "key": key, "payload": payload, "order": order} + for kind, key, payload, order in records + ] + } + ) + + +def _combined_state_checksum(checksums: dict[str, str]) -> str: + return _checksum({"categories": checksums}) + + +def _state_category_value(state: dict[str, Any], kind: str, generation: str) -> Any: + if kind == "community": + return state.get("communities", []) + incremental = state.get("incremental", {}) + if not isinstance(incremental, dict): + incremental = {} + if kind == "file": + return incremental.get("files", {}) + if kind == "cache": + return incremental.get("extraction_cache", {}) + + sections: dict[str, Any] = {} + for key, value in state.items(): + if key == "communities": + if isinstance(value, list) and not value: + sections[key] = [] + continue + if key == "incremental" and isinstance(value, dict): + metadata = { + name: item + for name, item in value.items() + if name not in {"files", "extraction_cache"} + } + metadata["last_successful_generation"] = generation + if "files" in value and not value.get("files"): + metadata["files"] = {} + if "extraction_cache" in value and not value.get("extraction_cache"): + metadata["extraction_cache"] = {} + sections[key] = metadata + continue + if key == "build" and isinstance(value, dict): + build = dict(value) + build["generation"] = generation + sections[key] = build + continue + sections[key] = value + sections.setdefault("build", {"generation": generation}) + sections.setdefault("incremental", {"last_successful_generation": generation}) + return sections + + +def _properties(row: Any, context: str) -> dict[str, Any]: + if not isinstance(row, dict): + raise RuntimeError(f"embedded Helix {context} row is not a mapping") + nested = row.get("properties") + if isinstance(nested, dict): + return {**row, **nested} + return row + + +def _rows(result: Any, name: str) -> list[Any]: + if not isinstance(result, dict): + raise RuntimeError("embedded Helix query returned a non-mapping response") + value = result.get(name) + if value is None: + return [] + if not isinstance(value, list): + raise RuntimeError(f"embedded Helix query variable {name!r} is not a list") + return value + + +class HelixNodeQuery: + """Long-lived public reader for bounded native node predicates.""" + + def __init__( + self, + path: str | Path, + generation: str, + *, + max_candidates: int = 50_000, + client: Any | None = None, + ) -> None: + self.path = Path(path).expanduser().resolve() + self.generation = generation + self.max_candidates = max_candidates + self._client = client or open_embedded_client( + self.path, + read_only=True, + disable_cache=True, + ) + self._lock = threading.RLock() + self._closed = False + + def _query(self, batch: Any) -> Any: + with self._lock: + if self._closed: + raise RuntimeError("native Helix node query is closed") + return self._client.query(batch.to_query_request()) + + def _predicate(self, property_name: str, values: list[str]) -> Any | None: + normalized = list( + dict.fromkeys(value for raw in values if (value := _normalize_search_text(raw))) + ) + if not normalized: + return None + matches = [SourcePredicate.contains(property_name, value) for value in normalized] + match = matches[0] if len(matches) == 1 else SourcePredicate.or_(matches) + return SourcePredicate.and_( + ( + SourcePredicate.eq(_GENERATION, self.generation), + match, + ) + ) + + def candidate_ids(self, values: list[str]) -> list[Any]: + """Return an order-stable, bounded superset from public predicates.""" + predicate = self._predicate(_SEARCH_TEXT, values) + if predicate is None: + return [] + batch = ( + read_batch() + .var_as( + "nodes", + g() + .n_with_label_where(_NODE_LABEL, predicate) + .limit(self.max_candidates) + .value_map(), + ) + .returning(["nodes"]) + ) + rows = [_properties(row, "search node") for row in _rows(self._query(batch), "nodes")] + rows.sort(key=lambda row: str(row.get(_STORAGE_KEY, ""))) + return [ + _decode_identity(row[_EXTERNAL_KEY]) + for row in rows + if isinstance(row.get(_EXTERNAL_KEY), dict) + ] + + def document_frequencies(self, terms: list[str]) -> dict[str, int]: + """Count label matches natively without reconstructing topology.""" + normalized = list( + dict.fromkeys(value for raw in terms if (value := _normalize_search_text(raw))) + ) + if not normalized: + return {} + batch = read_batch() + variables: list[str] = [] + for index, term in enumerate(normalized): + variable = f"count_{index}" + variables.append(variable) + predicate = SourcePredicate.and_( + ( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.contains(_SEARCH_LABEL, term), + ) + ) + batch = batch.var_as( + variable, + g().n_with_label_where(_NODE_LABEL, predicate).count(), + ) + result = self._query(batch.returning(variables)) + return {term: int(result.get(variable, 0)) for term, variable in zip(normalized, variables)} + + def traverse_ids( + self, + seeds: list[Any], + depth: int, + *, + contexts: set[str], + ) -> list[Any]: + """Traverse context-filtered edges through the public query DSL.""" + if not seeds: + return [] + storage_keys = [ + HelixEmbeddedStore._storage_key(self.generation, _encode_key(seed)) for seed in seeds + ] + seed_predicate = SourcePredicate.and_( + ( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.is_in(_STORAGE_KEY, storage_keys), + ) + ) + traversal = g().n_with_label_where(_NODE_LABEL, seed_predicate) + if depth > 0 and contexts: + edge_predicate = SourcePredicate.and_( + ( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.is_in(_EDGE_CONTEXT, sorted(contexts)), + ) + ) + step = SubTraversal.new().both_e().where(edge_predicate).other_n().dedup() + traversal = traversal.repeat(RepeatConfig.new(step).times(depth).emit_all()) + batch = ( + read_batch() + .var_as( + "nodes", + traversal.dedup().limit(self.max_candidates).value_map(), + ) + .returning(["nodes"]) + ) + rows = [_properties(row, "traversed node") for row in _rows(self._query(batch), "nodes")] + rows.sort(key=lambda row: str(row.get(_STORAGE_KEY, ""))) + return [ + _decode_identity(row[_EXTERNAL_KEY]) + for row in rows + if isinstance(row.get(_EXTERNAL_KEY), dict) + ] + + def close(self) -> None: + if not self._closed: + _close_public_client(self._client) + self._closed = True + + # Deliberately no __del__: async native close can re-enter while + # Python is garbage-collecting inside another embedded request. + + +@dataclass +class HelixEmbeddedStore: + """Graphify's durable graph schema on an in-process Helix ``Disk`` client.""" + + path: Path + _client: Any | None + _helix: Any + _read_only: bool + _closed: bool + _store_lock: _StoreLock + _open_lock: threading.Lock + _open_future: futures.Future[Any] | None + _open_executor: futures.ThreadPoolExecutor | None + _fresh: bool + _max_nodes: int + _max_edges: int + _retain_rollback: bool + + def __init__( + self, + path: str | Path, + *, + read_only: bool = False, + retain_rollback: bool = False, + max_nodes: int = DEFAULT_MAX_NODES, + max_edges: int = DEFAULT_MAX_EDGES, + ) -> None: + for name, value in (("max_nodes", max_nodes), ("max_edges", max_edges)): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + self.path = Path(path).expanduser().resolve() + if read_only: + if not self.path.is_dir(): + raise FileNotFoundError(f"embedded Helix store not found: {self.path}") + else: + self.path.mkdir(parents=True, exist_ok=True) + self._fresh = not any(child.name != _WRITER_LOCK_FILE for child in self.path.iterdir()) + validate_native_backend() + self._helix = helixdb + self._read_only = read_only + self._retain_rollback = bool(retain_rollback) + self._max_nodes = max_nodes + self._max_edges = max_edges + self._closed = False + self._client = None + self._open_lock = threading.Lock() + self._open_future = None + self._open_executor = None + self._store_lock = _StoreLock( + self.path / _WRITER_LOCK_FILE, + shared=read_only, + ) + self._store_lock.acquire() + if not read_only and self._fresh: + try: + self._open_executor = futures.ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="graphify-helix-open", + ) + self._open_future = self._open_executor.submit( + open_embedded_client, + self.path, + read_only=False, + disable_cache=True, + ) + except Exception: + self._store_lock.release() + raise + return + try: + self._client = open_embedded_client( + self.path, + read_only=read_only, + disable_cache=True, + ) + except Exception as exc: + self._store_lock.release() + if message := _public_store_rebuild_message(exc, self.path): + raise RuntimeError(message) from exc + raise + if not read_only: + try: + self._validate_active_schema() + self._cleanup_inactive_generations() + self._cleanup_inactive_state_revisions() + except Exception as exc: + try: + _close_public_client(self._client) + finally: + self._store_lock.release() + if message := _public_store_rebuild_message(exc, self.path): + raise RuntimeError(message) from exc + raise + + def _ensure_client(self) -> Any: + if self._closed: + raise RuntimeError("embedded Helix store is closed") + if self._client is not None: + return self._client + with self._open_lock: + if self._client is not None: + return self._client + future = self._open_future + assert future is not None, "an open store must own a client or open future" + try: + self._client = future.result() + except BaseException as exc: + self._closed = True + self._store_lock.release() + if message := _public_store_rebuild_message(exc, self.path): + raise RuntimeError(message) from exc + raise + finally: + executor = self._open_executor + self._open_future = None + self._open_executor = None + if executor is not None: + executor.shutdown(wait=False) + return self._client + + def _query( + self, + batch: Any, + *, + params: Any | None = None, + values: dict[str, Any] | None = None, + client: Any | None = None, + await_durability: bool | None = None, + ) -> Any: + if self._closed: + raise RuntimeError("embedded Helix store is closed") + selected = client if client is not None else self._ensure_client() + request = batch.to_query_request(params, values) + if await_durability is None: + return selected.query(request) + return selected.execute(request, await_durability=await_durability) + + def _snapshot_query(self, batch: Any, client: Any | None) -> Any: + return self._query(batch) if client is None else self._query(batch, client=client) + + def _run_buffered_queries( + self, + jobs: Iterable[tuple[Any, Any, dict[str, Any]]], + ) -> None: + """Execute a bounded number of independent buffered pages concurrently.""" + pending: set[futures.Future[Any]] = set() + with futures.ThreadPoolExecutor(max_workers=_BUFFERED_WRITE_CONCURRENCY) as executor: + for batch, params, values in jobs: + pending.add( + executor.submit( + self._query, + batch, + params=params, + values=values, + await_durability=False, + ) + ) + if len(pending) < _BUFFERED_WRITE_CONCURRENCY: + continue + completed, pending = futures.wait( + pending, + return_when=futures.FIRST_COMPLETED, + ) + for result in completed: + result.result() + for result in pending: + result.result() + + def _validate_active_schema(self) -> None: + """Reject stores written by an obsolete Graphify Helix schema. + + Production readers never reconstruct a graph in Python to upgrade it. + Rebuild the project from source when an older store is encountered. + """ + generation = self._active_generation(required=False) + if generation is None: + return + meta = self._metadata(generation) + version = meta.get("schema_version") + if version != _SCHEMA_VERSION: + raise RuntimeError( + "unsupported embedded Helix graph schema: " + f"expected {_SCHEMA_VERSION}, got {version!r}; rebuild from source" + ) + + def save(self, graph: GraphBuildData, *, state: dict[str, Any] | None = None) -> None: + self._save_graph(graph, state=state, activate=True) + + def save_generation(self, graph: GraphBuildData, state: dict[str, Any]) -> None: + """Atomically stage topology and every durable Graphify record together.""" + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + prepared = _prepare_topology( + graph, + max_nodes=self._max_nodes, + max_edges=self._max_edges, + ) + attempt = _DeltaAttempt.FALLBACK if self._fresh else self._try_delta(prepared, state) + if attempt is _DeltaAttempt.PUBLISHED: + return + self._save_prepared_graph( + prepared, + state=state, + activate=True, + native_validated=attempt is _DeltaAttempt.FALLBACK_NATIVE_VALIDATED, + ) + + def topology_matches(self, graph: GraphBuildData) -> bool: + """Return whether build data exactly matches the active native topology.""" + generation = self._active_generation(required=False) + if generation is None: + return False + expected = self._metadata(generation).get(_TOPOLOGY_CHECKSUM) + if not isinstance(expected, str): + return False + prepared = _prepare_topology( + graph, + max_nodes=self._max_nodes, + max_edges=self._max_edges, + ) + return expected == prepared.checksum + + def save_data(self, payload: dict[str, Any]) -> None: + """Stage, verify, and atomically activate a durable graph generation.""" + self._save_data(payload, activate=True) + + def _save_data(self, payload: dict[str, Any], *, activate: bool) -> str: + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + if not isinstance(payload, dict): + raise TypeError("node-link graph payload must be a mapping") + raw_state = payload.get(_DURABLE_STATE) + graph = GraphBuildData.from_node_link(payload) + return self._save_graph(graph, state=raw_state, activate=activate) + + def _save_graph( + self, + graph: GraphBuildData, + *, + state: dict[str, Any] | None, + activate: bool, + ) -> str: + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + prepared = _prepare_topology( + graph, + max_nodes=self._max_nodes, + max_edges=self._max_edges, + ) + return self._save_prepared_graph(prepared, state=state, activate=activate) + + def _save_prepared_graph( + self, + prepared: _PreparedTopology, + *, + state: dict[str, Any] | None, + activate: bool, + generation: str | None = None, + native_validated: bool = False, + ) -> str: + if not native_validated: + self._validate_proposed_native_graph(prepared) + generation = generation or uuid.uuid4().hex + prepared_state = self._prepare_state(state, generation) + manifest = { + "schema_version": _SCHEMA_VERSION, + "directed": prepared.directed, + "multigraph": prepared.multigraph, + "graph": prepared.graph_attributes, + "extras": prepared.extras, + "node_count": len(prepared.nodes), + "edge_count": len(prepared.edges), + "state_record_count": len(prepared_state.records), + "state_checksum": prepared_state.checksum, + _ACTIVE_STATE_REVISION: generation, + _CHECKSUM_MODE: _STREAM_CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM: prepared.checksum, + _TOPOLOGY_REVISION: generation, + _NODE_BUCKET_HASHES: prepared.node_bucket_hashes, + _EDGE_BUCKET_HASHES: prepared.edge_bucket_hashes, + **{key: generation for key in _STATE_REVISION_KEYS.values()}, + **{ + _STATE_CHECKSUM_KEYS[kind]: prepared_state.category_checksums[kind] + for kind in _STATE_KINDS + }, + **{ + _STATE_COUNT_KEYS[kind]: prepared_state.category_counts[kind] + for kind in _STATE_KINDS + }, + "checksum": _generation_checksum(prepared.checksum, prepared_state.checksum), + } + + previous_generation = self._active_generation(required=False) + try: + self._stage_generation( + generation, + manifest, + prepared.nodes, + prepared.edges, + prepared_state.encoded, + ) + if activate: + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + try: + self._drop_generation(generation) + except Exception: + pass + raise + + if activate: + self._fresh = False + self._cleanup_inactive_generations() + return generation + + def _try_delta( + self, + prepared: _PreparedTopology, + state: dict[str, Any], + *, + native_validated: bool = False, + ) -> _DeltaAttempt: + """Publish a small compatible topology/state change in one transaction.""" + generation = self._active_generation(required=False) + if generation is None or self._retain_rollback: + return _DeltaAttempt.FALLBACK + metadata = self._metadata(generation) + if ( + metadata.get("directed") is not prepared.directed + or metadata.get("multigraph") is not prepared.multigraph + or metadata.get("graph") != prepared.graph_attributes + or metadata.get("extras") != prepared.extras + ): + return _DeltaAttempt.FALLBACK + + prepared_state = self._prepare_state(state, generation) + changed_state_kinds = [ + kind + for kind in _STATE_KINDS + if metadata.get(_STATE_CHECKSUM_KEYS[kind]) != prepared_state.category_checksums[kind] + or metadata.get(_STATE_COUNT_KEYS[kind]) != prepared_state.category_counts[kind] + ] + topology_changed = metadata.get(_TOPOLOGY_CHECKSUM) != prepared.checksum + if not topology_changed and not changed_state_kinds: + return _DeltaAttempt.PUBLISHED + + delta = _TopologyDelta([], [], [], [], []) + stored_nodes: dict[str, _StoredNode] = {} + if topology_changed: + if native_validated: + candidate = self._topology_delta_candidate(generation, metadata, prepared) + else: + with futures.ThreadPoolExecutor(max_workers=2) as executor: + validation = executor.submit(self._validate_proposed_native_graph, prepared) + candidate = self._topology_delta_candidate(generation, metadata, prepared) + validation.result() + if candidate is None: + return ( + _DeltaAttempt.FALLBACK + if native_validated + else _DeltaAttempt.FALLBACK_NATIVE_VALIDATED + ) + delta, stored_nodes = candidate + + revision = uuid.uuid4().hex + try: + for kind in changed_state_kinds: + self._write_state_records( + generation, + prepared_state.category_records[kind], + revision, + await_durability=False, + ) + self._publish_delta( + generation, + prepared, + prepared_state, + metadata, + changed_state_kinds, + revision, + delta, + stored_nodes, + ) + except Exception: + # A storage commit can succeed even if its response is lost. Trust + # the atomically published metadata before removing staged rows. + published = False + try: + current = self._metadata(generation) + published = current.get(_TOPOLOGY_CHECKSUM) == prepared.checksum and all( + current.get(_STATE_REVISION_KEYS[kind]) == revision + and current.get(_STATE_CHECKSUM_KEYS[kind]) + == prepared_state.category_checksums[kind] + and current.get(_STATE_COUNT_KEYS[kind]) == prepared_state.category_counts[kind] + for kind in changed_state_kinds + ) + except Exception: + pass + if published: + self._cleanup_inactive_state_revisions() + return _DeltaAttempt.PUBLISHED + for kind in changed_state_kinds: + try: + self._drop_state_revision(generation, revision, kind=kind) + except Exception: + pass + raise + self._cleanup_inactive_state_revisions() + return _DeltaAttempt.PUBLISHED + + def _topology_delta_candidate( + self, + generation: str, + metadata: dict[str, Any], + prepared: _PreparedTopology, + ) -> tuple[_TopologyDelta, dict[str, _StoredNode]] | None: + if not hasattr(self._helix, "NativeGraphBuilder"): + return None + changed_node_buckets = _changed_buckets( + metadata.get(_NODE_BUCKET_HASHES), prepared.node_bucket_hashes + ) + changed_edge_buckets = _changed_buckets( + metadata.get(_EDGE_BUCKET_HASHES), prepared.edge_bucket_hashes + ) + if changed_node_buckets is None or changed_edge_buckets is None: + return None + stored_nodes, stored_edges = self._stored_topology( + generation, changed_node_buckets, changed_edge_buckets + ) + delta = self._topology_delta( + [node for node in prepared.nodes if node.identity_bucket in changed_node_buckets], + [edge for edge in prepared.edges if edge.identity_bucket in changed_edge_buckets], + stored_nodes, + stored_edges, + ) + added_node_ids = {node.encoded_id for node in delta.added_nodes} + required_endpoint_ids = { + encoded_id + for edge in delta.added_edges + for encoded_id in (edge.source, edge.target) + if encoded_id not in added_node_ids and encoded_id not in stored_nodes + } + stored_nodes.update(self._stored_node_ids(generation, required_endpoint_ids)) + old_size = metadata.get("node_count", 0) + metadata.get("edge_count", 0) + if ( + delta.mutation_count > _DELTA_MAX_MUTATIONS + or delta.mutation_count / max(1, old_size) > _DELTA_MAX_RATIO + ): + return None + return delta, stored_nodes + + def _stored_topology( + self, + generation: str, + node_buckets: set[int], + edge_buckets: set[int], + ) -> tuple[dict[str, _StoredNode], dict[str, _StoredEdge]]: + node_predicate = self._bucket_predicate(generation, node_buckets) + edge_predicate = self._bucket_predicate(generation, edge_buckets) + batch = self._helix.read_batch() + returned: list[str] = [] + if node_buckets: + returned.append("nodes") + batch = batch.var_as( + "nodes", + self._helix.g() + .n_with_label_where(_NODE_LABEL, node_predicate) + .value_map(["$id", _STORAGE_KEY, _RECORD_HASH]), + ) + if edge_buckets: + returned.append("edges") + batch = batch.var_as( + "edges", + self._helix.g() + .e_where(edge_predicate) + .value_map(["$id", _EDGE_IDENTITY, _RECORD_HASH]), + ) + result = self._query(batch.returning(returned)) + nodes: dict[str, _StoredNode] = {} + for raw in _rows(result, "nodes"): + row = _properties(raw, "delta node") + internal_id = row.get("$id") + storage_key = row.get(_STORAGE_KEY) + record_hash = row.get(_RECORD_HASH) + if ( + not isinstance(internal_id, int) + or not isinstance(storage_key, str) + or not isinstance(record_hash, str) + or not storage_key.startswith(f"{generation}:") + ): + raise RuntimeError("embedded Helix delta node is missing schema fields") + external_key = storage_key[len(generation) + 1 :] + if external_key in nodes: + raise RuntimeError("embedded Helix delta nodes duplicate an identity") + nodes[external_key] = _StoredNode(internal_id, record_hash) + + edges: dict[str, _StoredEdge] = {} + for raw in _rows(result, "edges"): + row = _properties(raw, "delta edge") + internal_id = row.get("$id") + identity = row.get(_EDGE_IDENTITY) + record_hash = row.get(_RECORD_HASH) + if ( + not isinstance(internal_id, int) + or not isinstance(identity, str) + or not isinstance(record_hash, str) + ): + raise RuntimeError("embedded Helix delta edge is missing schema fields") + if identity in edges: + raise RuntimeError("embedded Helix delta edges duplicate an identity") + edges[identity] = _StoredEdge(internal_id, record_hash) + return nodes, edges + + def _bucket_predicate(self, generation: str, buckets: set[int]) -> Any: + generation_predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + if not buckets: + return self._helix.SourcePredicate.and_( + ( + generation_predicate, + self._helix.SourcePredicate.eq(_IDENTITY_BUCKET, -1), + ) + ) + return self._helix.SourcePredicate.and_( + ( + generation_predicate, + self._helix.SourcePredicate.is_in(_IDENTITY_BUCKET, sorted(buckets)), + ) + ) + + def _stored_node_ids(self, generation: str, encoded_ids: set[str]) -> dict[str, _StoredNode]: + if not encoded_ids: + return {} + storage_keys = [ + self._storage_key(generation, encoded_id) for encoded_id in sorted(encoded_ids) + ] + predicate = self._helix.SourcePredicate.and_( + ( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.is_in(_STORAGE_KEY, storage_keys), + ) + ) + result = self._query( + self._helix.read_batch() + .var_as( + "nodes", + self._helix.g() + .n_with_label_where(_NODE_LABEL, predicate) + .value_map(["$id", _STORAGE_KEY]), + ) + .returning(["nodes"]) + ) + stored: dict[str, _StoredNode] = {} + for raw in _rows(result, "nodes"): + row = _properties(raw, "delta edge endpoint") + internal_id = row.get("$id") + storage_key = row.get(_STORAGE_KEY) + if not isinstance(internal_id, int) or not isinstance(storage_key, str): + raise RuntimeError("embedded Helix delta endpoint is missing schema fields") + encoded_id = storage_key[len(generation) + 1 :] + stored[encoded_id] = _StoredNode(internal_id, "") + if stored.keys() != encoded_ids: + raise RuntimeError("embedded Helix delta edge endpoint is missing") + return stored + + @staticmethod + def _topology_delta( + prepared_nodes: list[_PreparedNode], + prepared_edges: list[_PreparedEdge], + stored_nodes: dict[str, _StoredNode], + stored_edges: dict[str, _StoredEdge], + ) -> _TopologyDelta: + proposed_nodes = {node.encoded_id: node for node in prepared_nodes} + proposed_edges = {edge.stable_identity: edge for edge in prepared_edges} + changed_edges = { + identity + for identity in proposed_edges.keys() & stored_edges.keys() + if proposed_edges[identity].record_hash != stored_edges[identity].record_hash + } + return _TopologyDelta( + added_nodes=[ + proposed_nodes[key] for key in sorted(proposed_nodes.keys() - stored_nodes.keys()) + ], + updated_nodes=[ + (stored_nodes[key], proposed_nodes[key]) + for key in sorted(proposed_nodes.keys() & stored_nodes.keys()) + if proposed_nodes[key].record_hash != stored_nodes[key].record_hash + ], + dropped_nodes=[ + stored_nodes[key] for key in sorted(stored_nodes.keys() - proposed_nodes.keys()) + ], + added_edges=[ + proposed_edges[key] + for key in sorted((proposed_edges.keys() - stored_edges.keys()) | changed_edges) + ], + dropped_edges=[ + stored_edges[key] + for key in sorted((stored_edges.keys() - proposed_edges.keys()) | changed_edges) + ], + ) + + def _validate_proposed_native_graph(self, prepared: _PreparedTopology) -> Any: + kind = ( + "multidigraph" + if prepared.directed and prepared.multigraph + else "digraph" + if prepared.directed + else "multigraph" + if prepared.multigraph + else "graph" + ) + builder = self._helix.NativeGraphBuilder(kind, len(prepared.nodes), len(prepared.edges)) + for offset in range(0, len(prepared.nodes), _WRITE_CHUNK_SIZE): + builder.add_nodes( + [ + self._helix.GraphNode( + node.external_id, + _NODE_LABEL, + {_ATTRS: dict(node.attributes)}, + ) + for node in prepared.nodes[offset : offset + _WRITE_CHUNK_SIZE] + ] + ) + edges = builder.begin_edges() + for offset in range(0, len(prepared.edges), _STAGED_EDGE_WRITE_CHUNK_SIZE): + edges.add_edges( + [ + self._helix.GraphEdge( + self._helix.GraphEdgeId.original(edge.stable_identity), + edge.source_id, + edge.target_id, + edge.key if prepared.multigraph else None, + edge.relation, + edge.weight, + ({_ATTRS: dict(edge.stored_attributes)} if edge.stored_attributes else {}), + ) + for edge in prepared.edges[offset : offset + _STAGED_EDGE_WRITE_CHUNK_SIZE] + ] + ) + return edges.finish(prepared.graph_attributes) + + def _publish_delta( + self, + generation: str, + prepared: _PreparedTopology, + prepared_state: _PreparedState, + metadata: dict[str, Any], + changed_state_kinds: list[str], + revision: str, + delta: _TopologyDelta, + stored_nodes: dict[str, _StoredNode], + ) -> None: + batch = self._helix.write_batch() + if delta.dropped_edges: + batch = batch.var_as( + "drop_delta_edges", + self._helix.g().drop_edge_by_id( + self._helix.EdgeRef.ids(edge.internal_id for edge in delta.dropped_edges) + ), + ) + + added_node_variables: dict[str, str] = {} + for index, node in enumerate(delta.added_nodes): + variable = f"add_delta_node_{index}" + added_node_variables[node.encoded_id] = variable + batch = batch.var_as( + variable, + self._helix.g().add_n( + _NODE_LABEL, + self._node_properties(generation, node), + ), + ) + for index, (stored, node) in enumerate(delta.updated_nodes): + traversal = self._helix.g().n(self._helix.NodeRef.id(stored.internal_id)) + for key, value in self._node_properties(generation, node).items(): + traversal = traversal.set_property(key, value) + batch = batch.var_as(f"update_delta_node_{index}", traversal) + + if delta.dropped_nodes: + batch = batch.var_as( + "drop_delta_nodes", + self._helix.g() + .n(self._helix.NodeRef.ids(node.internal_id for node in delta.dropped_nodes)) + .drop(), + ) + + for index, edge in enumerate(delta.added_edges): + source = self._delta_node_ref(edge.source, added_node_variables, stored_nodes) + target = self._delta_node_ref(edge.target, added_node_variables, stored_nodes) + batch = batch.var_as( + f"add_delta_edge_{index}", + self._helix.g() + .n(source) + .add_e( + edge.relation, + target, + self._edge_properties( + generation, + edge, + multigraph=prepared.multigraph, + ), + ), + ) + + state_checksum = prepared_state.checksum + updates = { + "node_count": len(prepared.nodes), + "edge_count": len(prepared.edges), + "state_record_count": len(prepared_state.records), + "state_checksum": state_checksum, + _TOPOLOGY_CHECKSUM: prepared.checksum, + _NODE_BUCKET_HASHES: prepared.node_bucket_hashes, + _EDGE_BUCKET_HASHES: prepared.edge_bucket_hashes, + "checksum": _generation_checksum(prepared.checksum, state_checksum), + **{ + _STATE_CHECKSUM_KEYS[kind]: prepared_state.category_checksums[kind] + for kind in changed_state_kinds + }, + **{ + _STATE_COUNT_KEYS[kind]: prepared_state.category_counts[kind] + for kind in changed_state_kinds + }, + **{_STATE_REVISION_KEYS[kind]: revision for kind in changed_state_kinds}, + } + if delta.mutation_count: + updates[_TOPOLOGY_REVISION] = revision + if changed_state_kinds: + updates[_ACTIVE_STATE_REVISION] = revision + elif not isinstance(metadata.get(_ACTIVE_STATE_REVISION), str): + updates[_ACTIVE_STATE_REVISION] = generation + traversal = self._helix.g().n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + for key, value in updates.items(): + traversal = traversal.set_property(key, value) + batch = batch.var_as("publish_delta", traversal) + for index, kind in enumerate(changed_state_kinds): + old_revision = _state_revision(metadata, kind) + if old_revision is None: + continue + predicate = self._helix.SourcePredicate.and_( + ( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + self._helix.SourcePredicate.eq(_STATE_REVISION, old_revision), + ) + ) + batch = batch.var_as( + f"drop_old_delta_state_{index}", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + self._query(batch, await_durability=True) + + def _delta_node_ref( + self, + encoded_id: str, + added_node_variables: dict[str, str], + stored_nodes: dict[str, _StoredNode], + ) -> Any: + if encoded_id in added_node_variables: + return self._helix.NodeRef.var(added_node_variables[encoded_id]) + stored = stored_nodes.get(encoded_id) + if stored is None: + raise RuntimeError("delta edge references a missing persisted node") + return self._helix.NodeRef.id(stored.internal_id) + + @staticmethod + def _node_properties(generation: str, node: _PreparedNode) -> dict[str, Any]: + return { + _GENERATION: generation, + _STORAGE_KEY: HelixEmbeddedStore._storage_key(generation, node.encoded_id), + _EXTERNAL_KEY: _tagged_key(node.external_id), + _ATTRS: dict(node.attributes), + _IDENTITY_BUCKET: node.identity_bucket, + _RECORD_HASH: node.record_hash, + _SEARCH_LABEL: node.search_label, + _SEARCH_TEXT: node.search_text, + } + + @staticmethod + def _edge_properties( + generation: str, + edge: _PreparedEdge, + *, + multigraph: bool, + ) -> dict[str, Any]: + properties = { + _GENERATION: generation, + _EDGE_IDENTITY: edge.stable_identity, + _IDENTITY_BUCKET: edge.identity_bucket, + _RECORD_HASH: edge.record_hash, + } + if multigraph: + properties[_EDGE_KEY] = _tagged_key(edge.key) + if edge.stored_attributes: + properties[_ATTRS] = dict(edge.stored_attributes) + if edge.shape[1]: + properties[_EDGE_CONTEXT] = edge.context + if edge.shape[2]: + properties[_NATIVE_WEIGHT] = edge.weight + return properties + + @staticmethod + def _global_identity(repo: str, node_id: Any, attrs: dict[str, Any]) -> str: + """Return a stable aggregate ID, coalescing external nodes by label.""" + label = attrs.get("label") + if not attrs.get("source_file") and label: + normalized = _normalize_search_text(label).strip() + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:24] + return f"external::{digest}" + return f"{repo}::{node_id}" + + def _source_node_pages(self, generation: str) -> Any: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + offset = 0 + while True: + traversal = ( + self._helix.g() + .n_with_label_where(_NODE_LABEL, predicate) + .order_by(_STORAGE_KEY, self._helix.Order.ASC) + .skip(offset) + .limit(_WRITE_CHUNK_SIZE) + .project( + ( + self._helix.Projection.property(_EXTERNAL_KEY), + self._helix.Projection.property(_ATTRS), + self._helix.Projection.property(_STORAGE_KEY, _ORDER), + ) + ) + ) + rows = _rows( + self._query( + self._helix.read_batch().var_as("nodes", traversal).returning(["nodes"]) + ), + "nodes", + ) + if not rows: + return + yield rows + if len(rows) < _WRITE_CHUNK_SIZE: + return + offset += len(rows) + + def _source_edge_pages(self, generation: str) -> Any: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + offset = 0 + while True: + traversal = ( + self._helix.g() + .e_where(predicate) + .order_by(_EDGE_IDENTITY, self._helix.Order.ASC) + .skip(offset) + .limit(_WRITE_CHUNK_SIZE) + .project( + ( + self._helix.Projection.from_endpoint(_EXTERNAL_KEY, "source_external_key"), + self._helix.Projection.to_endpoint(_EXTERNAL_KEY, "target_external_key"), + self._helix.Projection.from_endpoint(_ATTRS, "source_attrs"), + self._helix.Projection.to_endpoint(_ATTRS, "target_attrs"), + self._helix.Projection.property(_EDGE_KEY), + self._helix.Projection.property(_ATTRS), + self._helix.Projection.property(_NATIVE_WEIGHT), + self._helix.Projection.property("$label", "relation"), + self._helix.Projection.property(_EDGE_IDENTITY, _ORDER), + ) + ) + ) + rows = _rows( + self._query( + self._helix.read_batch().var_as("edges", traversal).returning(["edges"]) + ), + "edges", + ) + if not rows: + return + yield rows + if len(rows) < _WRITE_CHUNK_SIZE: + return + offset += len(rows) + + def _write_aggregate_nodes( + self, + generation: str, + rows: list[tuple[str, dict[str, Any], str]], + ) -> None: + batch = self._helix.write_batch() + for local_index, (node_id, attrs, order) in enumerate(rows): + encoded_id = _encode_key(node_id) + search = _search_properties(node_id, attrs) + variable = f"node_{local_index}" + batch = batch.var_as( + variable, + self._helix.g().add_n( + _NODE_LABEL, + { + _GENERATION: generation, + _STORAGE_KEY: self._storage_key(generation, encoded_id), + _EXTERNAL_KEY: _tagged_key(node_id), + _ATTRS: attrs, + _ORDER: order, + _IDENTITY_BUCKET: _identity_bucket(encoded_id), + _RECORD_HASH: _record_hash(_node_topology_record(node_id, attrs)), + **search, + }, + ), + ) + if rows: + self._query(batch, await_durability=False) + + def _write_aggregate_edges( + self, + generation: str, + rows: list[tuple[str, str, str, dict[str, Any], str]], + ) -> None: + batch = self._helix.write_batch() + for local_index, (source, target, relation, attrs, order) in enumerate(rows): + source_var = f"source_{local_index}" + target_var = f"target_{local_index}" + edge_var = f"edge_{local_index}" + source_key = self._storage_key(generation, _encode_key(source)) + target_key = self._storage_key(generation, _encode_key(target)) + stable_identity = _edge_identity( + directed=False, + multigraph=False, + source=_encode_key(source), + target=_encode_key(target), + key=_tagged_key(None), + ) + batch = ( + batch.var_as( + source_var, + self._helix.g().n_with_label_where( + _NODE_LABEL, + self._helix.SourcePredicate.eq(_STORAGE_KEY, source_key), + ), + ) + .var_as( + target_var, + self._helix.g().n_with_label_where( + _NODE_LABEL, + self._helix.SourcePredicate.eq(_STORAGE_KEY, target_key), + ), + ) + .var_as( + edge_var, + self._helix.g() + .n(self._helix.NodeRef.var(source_var)) + .add_e( + relation, + self._helix.NodeRef.var(target_var), + { + _GENERATION: generation, + _EDGE_KEY: _tagged_key(None), + _ATTRS: attrs, + _ORDER: order, + _EDGE_IDENTITY: stable_identity, + _IDENTITY_BUCKET: _identity_bucket(stable_identity), + _RECORD_HASH: _record_hash( + _edge_topology_record( + source, + target, + None, + relation, + attrs, + multigraph=False, + ) + ), + **( + {_EDGE_CONTEXT: attrs["context"]} + if isinstance(attrs.get("context"), str) and attrs["context"] + else {} + ), + **( + {_NATIVE_WEIGHT: float(attrs["weight"])} + if isinstance(attrs.get("weight"), (int, float)) + and not isinstance(attrs.get("weight"), bool) + and math.isfinite(float(attrs["weight"])) + else {} + ), + }, + ), + ) + ) + if rows: + self._query(batch, await_durability=False) + + def save_aggregate_sources( + self, + sources: list[tuple[Path, str, str]], + state: dict[str, Any], + ) -> None: + """Stream project generations into one staged aggregate generation. + + Source topology is read in bounded public-query pages and written + directly to Helix. No Python graph, adjacency structure, or full ID map + is constructed. + """ + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + generation = uuid.uuid4().hex + previous_generation = self._active_generation(required=False) + checksum = _TopologyStreamChecksum(directed=False, multigraph=False, graph={}, extras={}) + external_ids: set[str] = set() + node_count = 0 + edge_count = 0 + try: + for source_path, source_generation, repo in sources: + if source_path.resolve() == self.path: + raise ValueError("aggregate destination cannot also be a source") + with HelixEmbeddedStore(source_path, read_only=True) as source: + source._metadata(source_generation) + for page in source._source_node_pages(source_generation): + output: list[tuple[str, dict[str, Any], str]] = [] + for raw in page: + row = _properties(raw, "aggregate source node") + old_id = _decode_identity(row.get(_EXTERNAL_KEY)) + attrs = row.get(_ATTRS, {}) + if not isinstance(attrs, dict): + raise RuntimeError("aggregate source node has invalid attributes") + node_id = self._global_identity(repo, old_id, attrs) + if node_id.startswith("external::"): + if node_id in external_ids: + continue + external_ids.add(node_id) + projected = dict(attrs) + projected["repo"] = repo + projected.setdefault("local_id", old_id) + output.append((node_id, projected, _encode_key(node_id))) + checksum.node(_node_topology_record(node_id, projected)) + node_count += 1 + self._write_aggregate_nodes(generation, output) + for page in source._source_edge_pages(source_generation): + output_edges: list[tuple[str, str, str, dict[str, Any], str]] = [] + for raw in page: + row = _properties(raw, "aggregate source edge") + source_attrs = row.get("source_attrs") + target_attrs = row.get("target_attrs") + attrs = row.get(_ATTRS, {}) + relation = row.get("relation") + if ( + not isinstance(source_attrs, dict) + or not isinstance(target_attrs, dict) + or not isinstance(attrs, dict) + or not isinstance(relation, str) + ): + raise RuntimeError( + "aggregate source edge has invalid schema fields" + ) + old_source = _decode_identity(row.get("source_external_key")) + old_target = _decode_identity(row.get("target_external_key")) + source_id = self._global_identity(repo, old_source, source_attrs) + target_id = self._global_identity(repo, old_target, target_attrs) + if source_id == target_id: + continue + projected_edge = dict(attrs) + weight = row.get(_NATIVE_WEIGHT) + if isinstance(weight, (int, float)) and not isinstance(weight, bool): + projected_edge.setdefault("weight", float(weight)) + stable_identity = _edge_identity( + directed=False, + multigraph=False, + source=_encode_key(source_id), + target=_encode_key(target_id), + key=_tagged_key(None), + ) + output_edges.append( + ( + source_id, + target_id, + relation, + projected_edge, + stable_identity, + ) + ) + checksum.edge( + _edge_topology_record( + source_id, + target_id, + None, + relation, + projected_edge, + multigraph=False, + ) + ) + edge_count += 1 + self._write_aggregate_edges(generation, output_edges) + + encoded_state_value = copy.deepcopy(state) + build_state = encoded_state_value.setdefault("build", {}) + incremental_state = encoded_state_value.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance(incremental_state, dict): + raise TypeError("durable build and incremental state must be mappings") + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value( + _encode_state_value(encoded_state_value), "durable graph state" + ) + state_records = self._state_records(encoded_state) + category_records = { + kind: [record for record in state_records if record[0] == kind] + for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) for kind in _STATE_KINDS + } + state_checksum = _combined_state_checksum(category_checksums) + topology_checksum = checksum.hexdigest() + manifest = { + "schema_version": _SCHEMA_VERSION, + "directed": False, + "multigraph": False, + "graph": {}, + "extras": {}, + "node_count": node_count, + "edge_count": edge_count, + "state_record_count": len(state_records), + "state_checksum": state_checksum, + _ACTIVE_STATE_REVISION: generation, + _CHECKSUM_MODE: _STREAM_CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM: topology_checksum, + _TOPOLOGY_REVISION: generation, + _NODE_BUCKET_HASHES: [""] * _IDENTITY_BUCKET_COUNT, + _EDGE_BUCKET_HASHES: [""] * _IDENTITY_BUCKET_COUNT, + **{key: generation for key in _STATE_REVISION_KEYS.values()}, + **{_STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] for kind in _STATE_KINDS}, + **{_STATE_COUNT_KEYS[kind]: len(category_records[kind]) for kind in _STATE_KINDS}, + "checksum": _generation_checksum(topology_checksum, state_checksum), + _GENERATION: generation, + } + self._query( + self._helix.write_batch().var_as( + "meta", self._helix.g().add_n(_META_LABEL, manifest) + ), + await_durability=False, + ) + self._write_state_records( + generation, + state_records, + generation, + await_durability=False, + ) + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + try: + self._drop_generation(generation) + except Exception: + pass + raise + self._fresh = False + self._cleanup_inactive_generations() + + @contextmanager + def staged_graph(self, graph: GraphBuildData): + """Yield a validated native proposal before any full-topology writes.""" + prepared = _prepare_topology( + graph, + max_nodes=self._max_nodes, + max_edges=self._max_edges, + ) + generation = self._active_generation(required=False) + if generation is not None and not self._retain_rollback: + metadata = self._metadata(generation) + compatible = ( + metadata.get("directed") is prepared.directed + and metadata.get("multigraph") is prepared.multigraph + and metadata.get("graph") == prepared.graph_attributes + and metadata.get("extras") == prepared.extras + ) + topology_changed = metadata.get(_TOPOLOGY_CHECKSUM) != prepared.checksum + delta_candidate = ( + self._topology_delta_candidate(generation, metadata, prepared) + if compatible and topology_changed + else None + ) + if compatible and (not topology_changed or delta_candidate is not None): + staged = _StagedDeltaGraph( + generation=generation, + state={}, + metadata=metadata, + store_path=self.path, + _phase=_StagedProposal( + graph=( + self._validate_proposed_native_graph(prepared) + if topology_changed + else self.native_graph(generation, metadata=metadata) + ), + prepared=prepared, + ), + ) + del prepared + yield staged + return + + generation = uuid.uuid4().hex + staged = _StagedFullGraph( + generation=generation, + state={}, + metadata={}, + store_path=self.path, + _phase=_StagedProposal( + graph=self._validate_proposed_native_graph(prepared), + prepared=prepared, + ), + ) + del prepared + yield staged + + def activate_staged( + self, + staged: LoadedGraph | _StagedDeltaGraph | _StagedFullGraph, + state: dict[str, Any], + ) -> LoadedGraph: + """Attach durable state, verify, and activate an existing staged topology.""" + if self._read_only: + raise RuntimeError("cannot activate through a read-only embedded Helix store") + generation = staged.generation + if staged.store_path != self.path: + raise ValueError("staged graph belongs to a different embedded Helix store") + if isinstance(staged, _StagedDeltaGraph): + if self._active_generation() != generation: + raise RuntimeError("active generation changed during native delta analysis") + native = staged.graph + prepared = staged.prepared + if ( + self._try_delta(prepared, state, native_validated=True) + is not _DeltaAttempt.PUBLISHED + ): + raise RuntimeError("native delta became ineligible during analysis") + topology_checksum = prepared.checksum + staged.mark_published() + del prepared + return self._load_generation_snapshot( + generation, + attach_query=True, + native=native, + expected_topology_checksum=topology_checksum, + ) + if isinstance(staged, _StagedFullGraph): + native = staged.graph + prepared = staged.prepared + self._save_prepared_graph( + prepared, + state=state, + activate=True, + generation=generation, + native_validated=True, + ) + topology_checksum = prepared.checksum + staged.mark_published() + del prepared + return self._load_generation_snapshot( + generation, + attach_query=True, + native=native, + expected_topology_checksum=topology_checksum, + ) + if self._metadata(generation).get("state_record_count") != 0: + raise RuntimeError("staged Helix generation has already been finalized") + + encoded = copy.deepcopy(state) + build_state = encoded.setdefault("build", {}) + incremental_state = encoded.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance(incremental_state, dict): + raise TypeError("durable build and incremental state must be mappings") + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value(_encode_state_value(encoded), "durable graph state") + records = self._state_records(encoded_state) + category_records = { + kind: [record for record in records if record[0] == kind] for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) for kind in _STATE_KINDS + } + previous_generation = self._active_generation(required=False) + try: + meta = self._metadata(generation) + topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if not isinstance(topology_checksum, str): + raise RuntimeError("embedded Helix metadata has no topology checksum") + self._write_state_records( + generation, + records, + generation, + await_durability=False, + ) + written = self._state_records_from_rows( + self._read_state_rows(generation, revision=generation) + ) + state_checksum = _combined_state_checksum(category_checksums) + if written != records: + raise RuntimeError("embedded Helix staged state failed checksum verification") + updates = { + "state_record_count": len(records), + "state_checksum": state_checksum, + "checksum": _generation_checksum(topology_checksum, state_checksum), + **{_STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] for kind in _STATE_KINDS}, + **{_STATE_COUNT_KEYS[kind]: len(category_records[kind]) for kind in _STATE_KINDS}, + } + traversal = self._helix.g().n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + for key, value in updates.items(): + traversal = traversal.set_property(key, value) + self._query( + self._helix.write_batch().var_as("finalize", traversal), + await_durability=False, + ) + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + self._drop_generation(generation) + raise + self._cleanup_inactive_generations() + return self.load_generation(generation) + + def replace_state( + self, + state: dict[str, Any], + *, + previous_state: dict[str, Any] | None = None, + snapshot: LoadedGraph | None = None, + ) -> None: + """Atomically replace only changed native-state categories.""" + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + if snapshot is not None: + if snapshot.store_path != self.path: + raise ValueError("native snapshot belongs to a different Helix store") + generation = snapshot.generation + meta = dict(snapshot.metadata) + else: + generation = self.active_generation + meta = self._metadata(generation) + topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if not isinstance(topology_checksum, str): + raise RuntimeError("embedded Helix metadata has invalid state revision fields") + + current = previous_state if previous_state is not None else self.read_state() + for value in (current, state): + if not isinstance(value.get("build", {}), dict) or not isinstance( + value.get("incremental", {}), dict + ): + raise TypeError("durable build and incremental state must be mappings") + changed_kinds = [ + kind + for kind in _STATE_KINDS + if _state_category_value(current, kind, generation) + != _state_category_value(state, kind, generation) + ] + if not changed_kinds: + return + + revision = uuid.uuid4().hex + records_by_kind = { + kind: self._state_records_for_kind(state, generation, kind) for kind in changed_kinds + } + changed_records = [record for kind in changed_kinds for record in records_by_kind[kind]] + category_checksums: dict[str, str] = {} + category_counts: dict[str, int] = {} + for kind in _STATE_KINDS: + if kind in records_by_kind: + category_checksums[kind] = _state_category_checksum(records_by_kind[kind]) + category_counts[kind] = len(records_by_kind[kind]) + continue + checksum = meta.get(_STATE_CHECKSUM_KEYS[kind]) + count = meta.get(_STATE_COUNT_KEYS[kind]) + if not isinstance(checksum, str) or not isinstance(count, int): + raise RuntimeError("embedded Helix metadata has invalid state category fields") + category_checksums[kind] = checksum + category_counts[kind] = count + state_checksum = _combined_state_checksum(category_checksums) + state_record_count = sum(category_counts.values()) + try: + traversal = self._helix.g().n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + updates = { + "state_record_count": state_record_count, + "state_checksum": state_checksum, + "checksum": _generation_checksum(topology_checksum, state_checksum), + **{_STATE_REVISION_KEYS[kind]: revision for kind in changed_kinds}, + **{_STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] for kind in changed_kinds}, + **{_STATE_COUNT_KEYS[kind]: category_counts[kind] for kind in changed_kinds}, + } + for key, value in updates.items(): + traversal = traversal.set_property(key, value) + transaction_size = len(changed_records) + len(changed_kinds) + 1 + atomic_write = transaction_size <= _STATE_WRITE_CHUNK_SIZE + batch = self._helix.write_batch() + if atomic_write: + for index, record in enumerate(changed_records): + batch = batch.var_as( + f"new_state_{index}", + self._helix.g().add_n( + _STATE_LABEL, + self._state_record_properties(generation, revision, record), + ), + ) + else: + self._write_state_records( + generation, + changed_records, + revision, + await_durability=False, + ) + batch = batch.var_as("activate_state", traversal) + for index, kind in enumerate(changed_kinds): + old_revision = _state_revision(meta, kind) + if old_revision is None: + continue + variable = f"drop_old_state_{index}" + predicate = self._helix.SourcePredicate.and_( + ( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_REVISION, old_revision), + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + ) + ) + batch = batch.var_as( + variable, + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + self._query(batch, await_durability=True) + except Exception: + for kind in changed_kinds: + self._drop_state_revision(generation, revision, kind=kind) + raise + + @staticmethod + def _storage_key(generation: str, external_key: str) -> str: + return f"{generation}:{external_key}" + + @staticmethod + def _state_records( + state: dict[str, Any] | None, + ) -> list[tuple[str, str, Any, int]]: + if state is None: + return [] + records: list[tuple[str, str, Any, int]] = [] + order = 0 + for section, payload in state.items(): + if section == "communities" and isinstance(payload, list): + if not payload: + records.append(("section", section, [], order)) + order += 1 + for index, community in enumerate(payload): + records.append(("community", str(index), community, order)) + order += 1 + continue + if section == "incremental" and isinstance(payload, dict): + files = payload.get("files", {}) + if not isinstance(files, dict): + raise TypeError("incremental durable state files must be a mapping") + extraction_cache = payload.get("extraction_cache", {}) + if not isinstance(extraction_cache, dict): + raise TypeError("incremental durable state extraction cache must be a mapping") + metadata = { + key: value + for key, value in payload.items() + if key not in {"files", "extraction_cache"} + } + if "files" in payload and not files: + metadata["files"] = {} + if "extraction_cache" in payload and not extraction_cache: + metadata["extraction_cache"] = {} + records.append(("section", section, metadata, order)) + order += 1 + for path, file_state in files.items(): + if not isinstance(path, str): + raise TypeError("incremental durable state file paths must be strings") + records.append(("file", path, file_state, order)) + order += 1 + for cache_key, cache_value in extraction_cache.items(): + if not isinstance(cache_key, str): + raise TypeError("incremental durable extraction cache keys must be strings") + records.append(("cache", cache_key, cache_value, order)) + order += 1 + continue + records.append(("section", section, payload, order)) + order += 1 + kind_order = {kind: 0 for kind in _STATE_KINDS} + normalized: list[tuple[str, str, Any, int]] = [] + for kind, key, payload, _ in records: + normalized.append((kind, key, payload, kind_order[kind])) + kind_order[kind] += 1 + kind_rank = {kind: index for index, kind in enumerate(_STATE_KINDS)} + normalized.sort(key=lambda item: (kind_rank[item[0]], item[3])) + return normalized + + def _prepare_state(self, state: dict[str, Any] | None, generation: str) -> _PreparedState: + encoded_state: dict[str, Any] | None = None + if state is not None: + if not isinstance(state, dict): + raise TypeError("durable graph state must be a mapping") + durable_state = copy.deepcopy(state) + build_state = durable_state.setdefault("build", {}) + incremental_state = durable_state.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance(incremental_state, dict): + raise TypeError("durable build and incremental state sections must be mappings") + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value(_encode_state_value(durable_state), "durable graph state") + records = self._state_records(encoded_state) + category_records = { + kind: [record for record in records if record[0] == kind] for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) for kind in _STATE_KINDS + } + return _PreparedState( + encoded=encoded_state, + records=records, + category_records=category_records, + category_checksums=category_checksums, + category_counts={kind: len(category_records[kind]) for kind in _STATE_KINDS}, + checksum=_combined_state_checksum(category_checksums), + ) + + def _state_records_for_kind( + self, state: dict[str, Any], generation: str, kind: str + ) -> list[tuple[str, str, Any, int]]: + value = _state_category_value(state, kind, generation) + if kind == "section": + partial = value + elif kind == "community": + partial = {"communities": value} + else: + partial = {"incremental": {"files" if kind == "file" else "extraction_cache": value}} + encoded = _json_value(_encode_state_value(partial), f"durable {kind} state") + return [record for record in self._state_records(encoded) if record[0] == kind] + + def _stage_generation( + self, + generation: str, + manifest: dict[str, Any], + nodes: list[_PreparedNode], + edges: list[_PreparedEdge], + state: dict[str, Any] | None, + ) -> None: + metadata = {**manifest, _GENERATION: generation} + multigraph = bool(manifest.get("multigraph")) + self._query( + self._helix.write_batch().var_as("meta", self._helix.g().add_n(_META_LABEL, metadata)), + await_durability=False, + ) + + row_params = self._helix.define_params( + {"rows": self._helix.param.array(self._helix.param.object())} + ) + node_body = self._helix.write_batch().var_as( + "node", + self._helix.g().add_n( + _NODE_LABEL, + { + _GENERATION: generation, + _STORAGE_KEY: self._helix.PropertyInput.param(_STORAGE_KEY), + _EXTERNAL_KEY: self._helix.PropertyInput.param(_EXTERNAL_KEY), + _ATTRS: self._helix.PropertyInput.param(_ATTRS), + _IDENTITY_BUCKET: self._helix.PropertyInput.param(_IDENTITY_BUCKET), + _SEARCH_LABEL: self._helix.PropertyInput.param(_SEARCH_LABEL), + _SEARCH_TEXT: self._helix.PropertyInput.param(_SEARCH_TEXT), + _RECORD_HASH: self._helix.PropertyInput.param(_RECORD_HASH), + }, + ), + ) + node_batch = self._helix.write_batch().for_each_param("rows", node_body) + node_pages = ( + ( + node_batch, + row_params, + { + "rows": [ + { + _STORAGE_KEY: self._storage_key(generation, node.encoded_id), + _EXTERNAL_KEY: _tagged_key(node.external_id), + _ATTRS: dict(node.attributes), + _IDENTITY_BUCKET: node.identity_bucket, + _RECORD_HASH: node.record_hash, + _SEARCH_LABEL: node.search_label, + _SEARCH_TEXT: node.search_text, + } + for node in nodes[offset : offset + _WRITE_CHUNK_SIZE] + ] + }, + ) + for offset in range(0, len(nodes), _WRITE_CHUNK_SIZE) + ) + self._run_buffered_queries(node_pages) + + node_ids: dict[str, int] = {} + if nodes: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + result = self._query( + self._helix.read_batch() + .var_as( + "nodes", + self._helix.g() + .n_with_label_where(_NODE_LABEL, predicate) + .value_map(["$id", _STORAGE_KEY]), + ) + .returning(["nodes"]) + ) + for raw in _rows(result, "nodes"): + row = _properties(raw, "staged node identity") + storage_key = row.get(_STORAGE_KEY) + node_id = row.get("$id") + if isinstance(storage_key, str) and isinstance(node_id, int): + node_ids[storage_key] = node_id + + if len(node_ids) != len(nodes): + raise RuntimeError("embedded Helix staged node count does not match the input graph") + + groups: dict[tuple[str, bool, bool, bool], list[_PreparedEdge]] = {} + for edge in edges: + groups.setdefault(edge.shape, []).append(edge) + + def edge_pages() -> Iterator[tuple[Any, Any, dict[str, Any]]]: + for (relation, has_context, has_weight, has_attributes), group in groups.items(): + properties = { + _GENERATION: generation, + _EDGE_IDENTITY: self._helix.PropertyInput.param(_EDGE_IDENTITY), + _IDENTITY_BUCKET: self._helix.PropertyInput.param(_IDENTITY_BUCKET), + _RECORD_HASH: self._helix.PropertyInput.param(_RECORD_HASH), + } + if multigraph: + properties[_EDGE_KEY] = self._helix.PropertyInput.param(_EDGE_KEY) + if has_attributes: + properties[_ATTRS] = self._helix.PropertyInput.param(_ATTRS) + if has_context: + properties[_EDGE_CONTEXT] = self._helix.PropertyInput.param(_EDGE_CONTEXT) + if has_weight: + properties[_NATIVE_WEIGHT] = self._helix.PropertyInput.param(_NATIVE_WEIGHT) + edge_body = self._helix.write_batch().var_as( + "edge", + self._helix.g() + .n(self._helix.NodeRef.param("source")) + .add_e( + relation, + self._helix.NodeRef.param("target"), + properties, + ), + ) + edge_batch = self._helix.write_batch().for_each_param("rows", edge_body) + for offset in range(0, len(group), _STAGED_EDGE_WRITE_CHUNK_SIZE): + rows = [] + for edge in group[offset : offset + _STAGED_EDGE_WRITE_CHUNK_SIZE]: + row = { + "source": [node_ids[self._storage_key(generation, edge.source)]], + "target": [node_ids[self._storage_key(generation, edge.target)]], + _EDGE_IDENTITY: edge.stable_identity, + _IDENTITY_BUCKET: edge.identity_bucket, + _RECORD_HASH: edge.record_hash, + } + if multigraph: + row[_EDGE_KEY] = _tagged_key(edge.key) + if has_attributes: + row[_ATTRS] = dict(edge.stored_attributes) + if has_context: + row[_EDGE_CONTEXT] = edge.context + if has_weight: + row[_NATIVE_WEIGHT] = edge.weight + rows.append(row) + yield edge_batch, row_params, {"rows": rows} + + self._run_buffered_queries(edge_pages()) + + self._write_state_records( + generation, + self._state_records(state), + generation, + await_durability=False, + ) + # The publication fence can transiently duplicate SlateDB's pending + # write buffers. These canonical records have already been staged and + # are intentionally consumed here so they do not overlap that peak. + nodes.clear() + edges.clear() + + def _write_state_records( + self, + generation: str, + state_records: list[tuple[str, str, Any, int]], + revision: str, + *, + await_durability: bool | None = None, + ) -> None: + # State rows are independent native records. A fixed planner-safe batch + # size keeps transaction cost bounded without exception-driven retries. + for offset in range(0, len(state_records), _STATE_WRITE_CHUNK_SIZE): + self._write_state_chunk( + generation, + state_records[offset : offset + _STATE_WRITE_CHUNK_SIZE], + revision, + await_durability=await_durability, + ) + + def _write_state_chunk( + self, + generation: str, + records: list[tuple[str, str, Any, int]], + revision: str, + *, + await_durability: bool | None = None, + ) -> None: + groups: dict[tuple[str, ...], list[dict[str, Any]]] = {} + for record in records: + properties = self._state_record_properties(generation, revision, record) + groups.setdefault(tuple(properties), []).append(properties) + row_params = self._helix.define_params( + {"rows": self._helix.param.array(self._helix.param.object())} + ) + for property_names, rows in groups.items(): + body = self._helix.write_batch().var_as( + "state", + self._helix.g().add_n( + _STATE_LABEL, + {name: self._helix.PropertyInput.param(name) for name in property_names}, + ), + ) + batch = self._helix.write_batch().for_each_param("rows", body) + self._query( + batch, + params=row_params, + values={"rows": rows}, + await_durability=await_durability, + ) + + @staticmethod + def _state_record_properties( + generation: str, + revision: str, + record: tuple[str, str, Any, int], + ) -> dict[str, Any]: + kind, key, payload, order = record + properties = { + _GENERATION: generation, + _STATE_REVISION: revision, + _STATE_KIND: kind, + _STATE_KEY: key, + _STATE_PAYLOAD: "json:" + + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + _ORDER: order, + } + if kind == "community" and isinstance(payload, dict): + for name in ( + "id", + "name", + "naming_source", + "signature", + "cohesion", + ): + if name in payload and payload[name] is not None: + properties[name] = payload[name] + elif kind == "file" and isinstance(payload, dict): + properties["relative_path"] = key + for name in ("content_hash", "semantic_hash"): + if name in payload: + properties[name] = payload[name] + return properties + + def _active_generation(self, *, required: bool = True, client: Any | None = None) -> str | None: + batch = ( + self._helix.read_batch() + .var_as( + "control", + self._helix.g() + .n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ) + .value_map(), + ) + .returning(["control"]) + ) + rows = _rows(self._snapshot_query(batch, client), "control") + if not rows: + if required: + raise RuntimeError("embedded Helix graph has no active generation") + return None + if len(rows) != 1: + raise RuntimeError("embedded Helix graph has duplicated control rows") + generation = _properties(rows[0], "control").get(_ACTIVE_GENERATION) + if not isinstance(generation, str): + raise RuntimeError("embedded Helix graph control row has no active generation") + return generation + + def _activate_generation( + self, + generation: str, + *, + create: bool, + previous: str | None, + retain_previous: bool, + ) -> None: + if create: + traversal = self._helix.g().add_n( + _CONTROL_LABEL, + {_CONTROL_KEY: "active", _ACTIVE_GENERATION: generation}, + ) + else: + traversal = ( + self._helix.g() + .n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ) + .set_property(_ACTIVE_GENERATION, generation) + ) + if retain_previous and previous is not None: + traversal = traversal.set_property(_PREVIOUS_GENERATION, previous) + else: + traversal = traversal.remove_property(_PREVIOUS_GENERATION) + batch = self._helix.write_batch().var_as("activate", traversal) + self._query(batch, await_durability=True) + + def rollback(self) -> LoadedGraph: + """Activate the explicitly retained previous generation.""" + if self._read_only: + raise RuntimeError("cannot roll back through a read-only embedded Helix store") + control = self._control_properties() + assert control is not None + active = control.get(_ACTIVE_GENERATION) + previous = control.get(_PREVIOUS_GENERATION) + if not isinstance(active, str) or not isinstance(previous, str): + raise RuntimeError( + "no rollback generation was retained; rebuild or update with --retain-rollback" + ) + self.load_generation(previous, attach_query=False) + self._activate_generation( + previous, + create=False, + previous=active, + retain_previous=True, + ) + self._cleanup_inactive_generations() + return self.load_generation(previous) + + def _drop_generation(self, generation: str) -> None: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + batch = ( + self._helix.write_batch() + .var_as( + "drop_edges", + self._helix.g().e_where(predicate).drop(), + ) + .var_as( + "drop_nodes", + self._helix.g().n_with_label_where(_NODE_LABEL, predicate).drop(), + ) + .var_as( + "drop_meta", + self._helix.g().n_with_label_where(_META_LABEL, predicate).drop(), + ) + .var_as( + "drop_state", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + ) + self._query(batch) + + def _drop_state_revision( + self, generation: str, revision: str, *, kind: str | None = None + ) -> None: + predicates = [ + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_REVISION, revision), + ] + if kind is not None: + predicates.append(self._helix.SourcePredicate.eq(_STATE_KIND, kind)) + predicate = self._helix.SourcePredicate.and_(predicates) + self._query( + self._helix.write_batch().var_as( + "drop_state_revision", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + ) + + def _cleanup_inactive_generations(self) -> None: + active = self._active_generation(required=False) + retained = {active} if active is not None else set() + control = self._control_properties(required=False) + previous = control.get(_PREVIOUS_GENERATION) if control is not None else None + if isinstance(previous, str): + retained.add(previous) + batch = ( + self._helix.read_batch() + .var_as("meta", self._helix.g().n_with_label(_META_LABEL).value_map()) + .returning(["meta"]) + ) + generations = { + generation + for raw in _rows(self._query(batch), "meta") + if isinstance(generation := _properties(raw, "metadata").get(_GENERATION), str) + } + for generation in generations: + if generation not in retained: + self._drop_generation(generation) + + def _cleanup_inactive_state_revisions(self) -> None: + batch = ( + self._helix.read_batch() + .var_as("meta", self._helix.g().n_with_label(_META_LABEL).value_map()) + .returning(["meta"]) + ) + for raw in _rows(self._query(batch), "meta"): + row = _properties(raw, "metadata") + generation = row.get(_GENERATION) + if not isinstance(generation, str): + continue + for kind in _STATE_KINDS: + revision = _state_revision(row, kind) + if revision is None: + continue + predicate = self._helix.SourcePredicate.and_( + ( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + self._helix.SourcePredicate.neq(_STATE_REVISION, revision), + ) + ) + self._query( + self._helix.write_batch().var_as( + "drop_inactive_state", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + ) + + def _control_properties( + self, *, required: bool = True, client: Any | None = None + ) -> dict[str, Any] | None: + batch = ( + self._helix.read_batch() + .var_as( + "control", + self._helix.g() + .n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ) + .value_map(), + ) + .returning(["control"]) + ) + rows = _rows(self._snapshot_query(batch, client), "control") + if not rows: + if required: + raise RuntimeError("embedded Helix graph has no active generation") + return None + if len(rows) != 1: + raise RuntimeError("embedded Helix graph has duplicated control rows") + return _properties(rows[0], "control") + + def _read_rows(self, generation: str) -> tuple[dict[str, Any], list[Any], list[Any], list[Any]]: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + batch = ( + self._helix.read_batch() + .var_as( + "meta", + self._helix.g().n_with_label_where(_META_LABEL, predicate).value_map(), + ) + .var_as( + "nodes", + self._helix.g().n_with_label_where(_NODE_LABEL, predicate).value_map(), + ) + .var_as( + "edges", + self._helix.g().e_where(predicate).value_map(), + ) + .var_as( + "state", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).value_map(), + ) + .returning(["meta", "nodes", "edges", "state"]) + ) + result = self._query(batch) + meta_rows = _rows(result, "meta") + if len(meta_rows) != 1: + raise RuntimeError( + f"embedded Helix graph must contain exactly one metadata node; " + f"found {len(meta_rows)}" + ) + return ( + _properties(meta_rows[0], "metadata"), + _rows(result, "nodes"), + _rows(result, "edges"), + _rows(result, "state"), + ) + + def _read_state_rows( + self, + generation: str, + *, + metadata: dict[str, Any] | None = None, + revision: str | None = None, + client: Any | None = None, + ) -> list[Any]: + generation_predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + if revision is not None: + predicate = self._helix.SourcePredicate.and_( + ( + generation_predicate, + self._helix.SourcePredicate.eq(_STATE_REVISION, revision), + ) + ) + else: + meta = metadata or self._metadata(generation, client=client) + kind_predicates = [] + for kind in _STATE_KINDS: + selected_revision = _state_revision(meta, kind) + if selected_revision is None: + predicate = generation_predicate + break + kind_predicates.append( + self._helix.SourcePredicate.and_( + ( + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + self._helix.SourcePredicate.eq(_STATE_REVISION, selected_revision), + ) + ) + ) + else: + predicate = self._helix.SourcePredicate.and_( + ( + generation_predicate, + self._helix.SourcePredicate.or_(kind_predicates), + ) + ) + batch = ( + self._helix.read_batch() + .var_as( + "state", + self._helix.g() + .n_with_label_where( + _STATE_LABEL, + predicate, + ) + .value_map((_STATE_KIND, _STATE_KEY, _STATE_PAYLOAD, _ORDER)), + ) + .returning(["state"]) + ) + return _rows(self._snapshot_query(batch, client), "state") + + @staticmethod + def _state_records_from_rows( + rows: list[Any], + ) -> list[tuple[str, str, Any, int]]: + ordered: list[tuple[str, str, Any, int]] = [] + seen_orders: set[tuple[str, int]] = set() + for raw in rows: + row = _properties(raw, "durable state") + kind = row.get(_STATE_KIND) + key = row.get(_STATE_KEY) + order = row.get(_ORDER) + if ( + kind not in {"section", "community", "file", "cache"} + or not isinstance(key, str) + or not isinstance(order, int) + or isinstance(order, bool) + or _STATE_PAYLOAD not in row + ): + raise RuntimeError("embedded Helix durable state record is missing schema fields") + order_key = (kind, order) + if order_key in seen_orders: + raise RuntimeError("embedded Helix durable state has duplicate ordering") + seen_orders.add(order_key) + payload = row[_STATE_PAYLOAD] + if isinstance(payload, str) and payload.startswith("json:"): + try: + payload = json.loads(payload[5:]) + except json.JSONDecodeError as exc: + raise RuntimeError( + "embedded Helix durable state has invalid encoded payload" + ) from exc + ordered.append((kind, key, payload, order)) + kind_rank = {kind: index for index, kind in enumerate(_STATE_KINDS)} + ordered.sort(key=lambda item: (kind_rank[item[0]], item[3])) + return ordered + + @staticmethod + def _state_from_rows(rows: list[Any], expected_count: Any) -> dict[str, Any]: + if not isinstance(expected_count, int) or isinstance(expected_count, bool): + raise RuntimeError("embedded Helix metadata has an invalid state record count") + if len(rows) != expected_count: + raise RuntimeError( + "embedded Helix durable state failed count verification: " + f"expected {expected_count}, read {len(rows)}" + ) + + ordered = HelixEmbeddedStore._state_records_from_rows(rows) + + state: dict[str, Any] = {} + community_records: list[Any] = [] + file_records: dict[str, Any] = {} + cache_records: dict[str, Any] = {} + for kind, key, payload, _ in ordered: + if kind == "section": + if key in state: + raise RuntimeError(f"embedded Helix durable state duplicates section {key!r}") + state[key] = payload + elif kind == "community": + community_records.append(payload) + elif kind == "file": + if key in file_records: + raise RuntimeError(f"embedded Helix durable state duplicates file {key!r}") + file_records[key] = payload + else: + if key in cache_records: + raise RuntimeError(f"embedded Helix durable state duplicates cache key {key!r}") + cache_records[key] = payload + + if community_records: + if "communities" in state: + raise RuntimeError( + "embedded Helix durable state mixes community section and records" + ) + state["communities"] = community_records + if file_records: + incremental = state.setdefault("incremental", {}) + if not isinstance(incremental, dict) or "files" in incremental: + raise RuntimeError("embedded Helix durable state has invalid incremental records") + incremental["files"] = file_records + if cache_records: + incremental = state.setdefault("incremental", {}) + if not isinstance(incremental, dict) or "extraction_cache" in incremental: + raise RuntimeError( + "embedded Helix durable state has invalid extraction cache records" + ) + incremental["extraction_cache"] = cache_records + return state + + @staticmethod + def _verified_state_from_rows(rows: list[Any], metadata: dict[str, Any]) -> dict[str, Any]: + state = HelixEmbeddedStore._state_from_rows(rows, metadata.get("state_record_count")) + has_category_checksums = all( + isinstance(metadata.get(_STATE_CHECKSUM_KEYS[kind]), str) + and isinstance(metadata.get(_STATE_COUNT_KEYS[kind]), int) + for kind in _STATE_KINDS + ) + if has_category_checksums: + records = HelixEmbeddedStore._state_records_from_rows(rows) + checksums: dict[str, str] = {} + for kind in _STATE_KINDS: + category = [record for record in records if record[0] == kind] + expected_count = metadata[_STATE_COUNT_KEYS[kind]] + if len(category) != expected_count: + raise RuntimeError( + "embedded Helix durable state category failed count verification" + ) + checksum = _state_category_checksum(category) + if checksum != metadata[_STATE_CHECKSUM_KEYS[kind]]: + raise RuntimeError( + "embedded Helix durable state category failed checksum verification" + ) + checksums[kind] = checksum + actual_checksum = _combined_state_checksum(checksums) + else: + actual_checksum = _checksum(state) + if metadata.get("state_checksum") != actual_checksum: + raise RuntimeError("embedded Helix durable state failed checksum verification") + return state + + def read_data(self) -> dict[str, Any]: + generation = self._active_generation() + assert generation is not None + return self._read_generation_data(generation) + + @property + def active_generation(self) -> str: + generation = self._active_generation() + assert generation is not None + return generation + + def read_state(self) -> dict[str, Any]: + generation = self.active_generation + meta = self._metadata(generation) + self._validate_metadata(meta) + state = self._verified_state_from_rows( + self._read_state_rows(generation, metadata=meta), meta + ) + decoded = _decode_state_value(state) + if not isinstance(decoded, dict): + raise RuntimeError("embedded Helix generation contains invalid durable state") + return decoded + + def native_graph( + self, + generation: str | None = None, + *, + metadata: dict[str, Any] | None = None, + client: Any | None = None, + include_storage_identity: bool = False, + ) -> Any: + """Load one immutable native snapshot of the active generation.""" + generation = generation or self.active_generation + meta = metadata or self._metadata(generation, client=client) + node_count = meta.get("node_count") + edge_count = meta.get("edge_count") + if ( + not isinstance(node_count, int) + or not isinstance(edge_count, int) + or node_count > self._max_nodes + or edge_count > self._max_edges + ): + raise RuntimeError("embedded Helix generation exceeds configured snapshot bounds") + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + kind = ( + "multidigraph" + if bool(meta.get("directed")) and bool(meta.get("multigraph")) + else "digraph" + if bool(meta.get("directed")) + else "multigraph" + if bool(meta.get("multigraph")) + else "graph" + ) + selection = self._helix.GraphSelection( + node_traversal=self._helix.g().n_with_label_where(_NODE_LABEL, predicate), + edge_traversal=self._helix.g().e_where(predicate), + kind=kind, + metadata=self._helix.GraphMetadataSelection( + self._helix.g().n_with_label_where(_META_LABEL, predicate), + ( + _GENERATION, + "schema_version", + "directed", + "multigraph", + "graph", + "extras", + "node_count", + "edge_count", + _ACTIVE_STATE_REVISION, + *_STATE_REVISION_KEYS.values(), + *_STATE_CHECKSUM_KEYS.values(), + *_STATE_COUNT_KEYS.values(), + _CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM, + _TOPOLOGY_REVISION, + _NODE_BUCKET_HASHES, + _EDGE_BUCKET_HASHES, + "state_checksum", + "checksum", + ), + ), + node_identity=self._helix.IdentitySelection.tagged_property(_EXTERNAL_KEY), + graphify_edge_key=( + self._helix.IdentitySelection.tagged_property(_EDGE_KEY) + if bool(meta.get("multigraph")) + else None + ), + weight_property=_NATIVE_WEIGHT, + node_properties=( + (_ATTRS, _STORAGE_KEY) if include_storage_identity else (_ATTRS,) + ), + edge_properties=( + (_ATTRS, _EDGE_IDENTITY) if include_storage_identity else (_ATTRS,) + ), + max_nodes=self._max_nodes, + max_edges=self._max_edges, + allow_full_scan=True, + ) + selected = client if client is not None else self._ensure_client() + graph = selected.graph(selection) + if graph.node_count != meta.get("node_count") or graph.edge_count != meta.get("edge_count"): + raise RuntimeError("native Helix snapshot failed generation count verification") + return graph + + def _read_generation_data(self, generation: str) -> dict[str, Any]: + meta = self._metadata(generation) + self._validate_metadata(meta) + native = self.native_graph( + generation, + metadata=meta, + include_storage_identity=True, + ) + + nodes_with_order: list[tuple[str, dict[str, Any], dict[str, Any]]] = [] + for record in native.nodes(): + projected = dict(record.attributes) + attrs, order = projected.get(_ATTRS), projected.get(_STORAGE_KEY) + if not isinstance(attrs, dict) or not isinstance(order, str): + raise RuntimeError("native Helix node is missing Graphify schema fields") + nodes_with_order.append( + ( + order, + {"id": record.id, **attrs}, + _node_topology_record(record.id, attrs), + ) + ) + + edges_with_order: list[tuple[str, dict[str, Any], dict[str, Any]]] = [] + multigraph = bool(meta.get("multigraph", False)) + for record in native.edges(): + projected = dict(record.attributes) + attrs, order = projected.get(_ATTRS, {}), projected.get(_EDGE_IDENTITY) + if not isinstance(attrs, dict) or not isinstance(order, str): + raise RuntimeError("native Helix edge is missing Graphify schema fields") + attrs = dict(attrs) + if record.weight is not None: + attrs.setdefault("weight", record.weight) + edge = { + "source": record.source, + "target": record.target, + "relation": record.label, + **attrs, + } + if multigraph: + edge["key"] = record.graphify_key + edges_with_order.append( + ( + order, + edge, + _edge_topology_record( + record.source, + record.target, + record.graphify_key, + record.label, + attrs, + multigraph=multigraph, + ), + ) + ) + + nodes_with_order.sort(key=lambda item: item[0]) + edges_with_order.sort(key=lambda item: item[0]) + extras = meta.get("extras", {}) + graph_attrs = meta.get("graph", {}) + if not isinstance(extras, dict) or not isinstance(graph_attrs, dict): + raise RuntimeError("embedded Helix metadata contains invalid graph attributes") + topology_payload = { + "directed": bool(meta.get("directed", False)), + "multigraph": multigraph, + "graph": graph_attrs, + "nodes": [row for _, row, _ in nodes_with_order], + "links": [row for _, row, _ in edges_with_order], + **extras, + } + payload = dict(topology_payload) + state = self._verified_state_from_rows( + self._read_state_rows(generation, metadata=meta), meta + ) + if state: + payload[_DURABLE_STATE] = state + expected_nodes = meta.get("node_count") + expected_edges = meta.get("edge_count") + expected_checksum = meta.get("checksum") + if expected_nodes != len(nodes_with_order) or expected_edges != len(edges_with_order): + raise RuntimeError( + "embedded Helix graph failed count verification: " + f"expected {expected_nodes}/{expected_edges}, " + f"read {len(nodes_with_order)}/{len(edges_with_order)}" + ) + if meta.get(_CHECKSUM_MODE) in { + _SPLIT_CHECKSUM_MODE, + _STREAM_CHECKSUM_MODE, + }: + if meta.get(_CHECKSUM_MODE) == _STREAM_CHECKSUM_MODE: + stream_checksum = _TopologyStreamChecksum( + directed=topology_payload["directed"], + multigraph=topology_payload["multigraph"], + graph=graph_attrs, + extras=extras, + ) + for _, _, node in nodes_with_order: + stream_checksum.node(node) + for _, _, edge in edges_with_order: + stream_checksum.edge(edge) + topology_checksum = stream_checksum.hexdigest() + else: + topology_checksum = _checksum(topology_payload) + expected_topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if expected_topology_checksum != topology_checksum: + raise RuntimeError( + "embedded Helix topology failed checksum verification: " + f"expected {expected_topology_checksum!r}, got {topology_checksum!r}" + ) + state_checksum = meta.get("state_checksum") + if not isinstance(state_checksum, str): + raise RuntimeError("embedded Helix metadata has no durable state checksum") + actual_checksum = _generation_checksum(topology_checksum, state_checksum) + else: + actual_checksum = _checksum(payload) + if expected_checksum != actual_checksum: + raise RuntimeError( + "embedded Helix graph failed checksum verification: " + f"expected {expected_checksum!r}, got {actual_checksum!r}" + ) + return payload + + def _verify_generation_counts(self, generation: str) -> dict[str, Any]: + """Verify one staged generation with bounded native count projections.""" + metadata = self._metadata(generation) + self._validate_metadata(metadata) + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + counts = self._query( + self._helix.read_batch() + .var_as( + "nodes", + self._helix.g().n_with_label_where(_NODE_LABEL, predicate).count(), + ) + .var_as("edges", self._helix.g().e_where(predicate).count()) + .var_as( + "state_records", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).count(), + ) + .returning(["nodes", "edges", "state_records"]) + ) + if not isinstance(counts, dict): + raise RuntimeError("embedded Helix count verification returned no result") + expected = { + "nodes": metadata.get("node_count"), + "edges": metadata.get("edge_count"), + "state_records": metadata.get("state_record_count"), + } + if any( + not isinstance(expected[name], int) or counts.get(name) != expected[name] + for name in expected + ): + raise RuntimeError( + "embedded Helix generation failed count verification: " + f"expected {expected!r}, read {counts!r}" + ) + return { + "schema_version": _SCHEMA_VERSION, + **counts, + "checksum": metadata.get("checksum"), + } + + @staticmethod + def _validate_metadata(meta: dict[str, Any]) -> None: + if meta.get("schema_version") != _SCHEMA_VERSION: + raise RuntimeError( + "unsupported embedded Helix graph schema: " + f"expected {_SCHEMA_VERSION}, got {meta.get('schema_version')!r}" + ) + + def _load_generation_snapshot( + self, + generation: str | None, + *, + attach_query: bool, + native: Any | None = None, + expected_topology_checksum: str | None = None, + ) -> LoadedGraph: + attempts = 3 if attach_query else 1 + for attempt in range(attempts): + snapshot_client = ( + open_embedded_client( + self.path, + read_only=True, + disable_cache=True, + ) + if attach_query + else None + ) + client = snapshot_client if snapshot_client is not None else self._ensure_client() + try: + selected_generation = generation + if selected_generation is None: + selected_generation = self._active_generation(client=client) + assert selected_generation is not None + meta = self._metadata(selected_generation, client=client) + self._validate_metadata(meta) + if ( + expected_topology_checksum is not None + and meta.get(_TOPOLOGY_CHECKSUM) != expected_topology_checksum + ): + raise RuntimeError( + "validated native topology does not match the published generation" + ) + loaded_native = native + if loaded_native is None: + loaded_native = self.native_graph( + selected_generation, + metadata=meta, + client=client, + ) + elif loaded_native.node_count != meta.get( + "node_count" + ) or loaded_native.edge_count != meta.get("edge_count"): + raise RuntimeError( + "validated native topology does not match published generation counts" + ) + state = self._verified_state_from_rows( + self._read_state_rows( + selected_generation, + metadata=meta, + client=client, + ), + meta, + ) + decoded = _decode_state_value(state) + if not isinstance(decoded, dict): + raise RuntimeError("embedded Helix generation contains invalid durable state") + query = ( + HelixNodeQuery(self.path, selected_generation, client=snapshot_client) + if snapshot_client is not None + else None + ) + return LoadedGraph( + loaded_native, + selected_generation, + decoded, + meta, + self.path, + query, + ) + except Exception as exc: + if snapshot_client is not None: + _close_public_client(snapshot_client) + if attempt + 1 < attempts and "Request read view changed during execution" in str( + exc + ): + continue + raise + raise AssertionError("snapshot retry loop must return or raise") + + def load_generation(self, generation: str | None, *, attach_query: bool = True) -> LoadedGraph: + """Load topology, metadata, state, and query from one reader snapshot.""" + return self._load_generation_snapshot(generation, attach_query=attach_query) + + def load(self) -> LoadedGraph: + return self.load_generation(None) + + def checkpoint(self) -> None: + # Durable publications fence earlier buffered writes; close() flushes the handle. + if self._closed: + raise RuntimeError("embedded Helix store is closed") + + def _metadata(self, generation: str, *, client: Any | None = None) -> dict[str, Any]: + batch = ( + self._helix.read_batch() + .var_as( + "meta", + self._helix.g() + .n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + .value_map(), + ) + .returning(["meta"]) + ) + rows = _rows(self._snapshot_query(batch, client), "meta") + if len(rows) != 1: + raise RuntimeError("embedded Helix graph metadata is missing or duplicated") + return _properties(rows[0], "metadata") + + def verify(self) -> dict[str, Any]: + """Perform a deep topology/state checksum verification.""" + payload = self.read_data() + metadata = self._metadata(self.active_generation) + return { + "schema_version": _SCHEMA_VERSION, + "nodes": len(payload["nodes"]), + "edges": len(payload["links"]), + "checksum": metadata.get("checksum"), + } + + def verify_counts(self) -> dict[str, Any]: + """Perform the lightweight verification used on the write path.""" + return self._verify_generation_counts(self.active_generation) + + def close(self) -> None: + if not self._closed: + try: + _close_public_client(self._ensure_client()) + finally: + if not self._closed: + self._store_lock.release() + self._closed = True + + def __enter__(self) -> "HelixEmbeddedStore": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def __del__(self) -> None: + try: + if hasattr(self, "_closed") and not self._closed: + self.close() + except Exception: + pass + + +class HelixGraphReader: + """Retain one native graph while polling immutable embedded snapshots.""" + + def __init__(self, path: str | Path = DEFAULT_PROJECT_STORE) -> None: + self.path = Path(path) + self._version: tuple[str | None, ...] | None = None + self._graph: LoadedGraph | None = None + self._lock = threading.RLock() + + def get(self) -> LoadedGraph: + with self._lock: + # Helix embedded readers are immutable database snapshots. Reopen + # the lightweight handle to observe a writer's atomic pointer flip; + # retain the expensive native graph while its version is unchanged. + with HelixEmbeddedStore(self.path, read_only=True) as store: + generation = store.active_generation + metadata = store._metadata(generation) + version = ( + generation, + metadata.get(_TOPOLOGY_REVISION), + *(_state_revision(metadata, kind) for kind in _STATE_KINDS), + ) + if self._graph is None or version != self._version: + candidate = store.load() + candidate_version = ( + candidate.generation, + candidate.metadata.get(_TOPOLOGY_REVISION), + *(_state_revision(dict(candidate.metadata), kind) for kind in _STATE_KINDS), + ) + if candidate_version != self._version: + self._graph = candidate + self._version = candidate_version + elif candidate.query is not None: + candidate.query.close() + graph = self._graph + assert graph is not None + return graph + + def close(self) -> None: + with self._lock: + if self._graph is not None and self._graph.query is not None: + self._graph.query.close() + self._graph = None + self._version = None + + # Deliberately no __del__; closing the owned query from cyclic GC has the + # same unsafe b3 re-entrancy as finalizing HelixNodeQuery directly. + + +def persist_graph( + graph: GraphBuildData, + path: str | Path = DEFAULT_PROJECT_STORE, + *, + state: dict[str, Any] | None = None, + retain_rollback: bool = False, +) -> None: + with HelixEmbeddedStore(path, retain_rollback=retain_rollback) as store: + store.save(graph, state=state) + + +def persist_graph_data( + data: dict[str, Any], + path: str | Path, + *, + retain_rollback: bool = False, +) -> None: + with HelixEmbeddedStore(path, retain_rollback=retain_rollback) as store: + store.save_data(data) + + +def load_graph(path: str | Path = DEFAULT_PROJECT_STORE) -> LoadedGraph: + with HelixEmbeddedStore(path, read_only=True) as store: + return store.load() + + +def graph_storage_exists(path: str | Path = DEFAULT_PROJECT_STORE) -> bool: + return Path(path).expanduser().is_dir() + + +__all__ = [ + "HelixEmbeddedStore", + "HelixGraphReader", + "HelixNodeQuery", + "DEFAULT_GLOBAL_STORE", + "DEFAULT_MAX_EDGES", + "DEFAULT_MAX_NODES", + "DEFAULT_PROJECT_STORE", + "graph_storage_exists", + "load_graph", + "persist_graph", + "persist_graph_data", +] diff --git a/graphify/helix/state.py b/graphify/helix/state.py new file mode 100644 index 000000000..c2a17047c --- /dev/null +++ b/graphify/helix/state.py @@ -0,0 +1,103 @@ +"""Versioned durable state stored inside the same Helix graph generation.""" + +from __future__ import annotations + +import hashlib +from typing import Any + + +STATE_SCHEMA_VERSION = 1 + + +def new_state(**overrides: Any) -> dict[str, Any]: + state: dict[str, Any] = { + "schema_version": STATE_SCHEMA_VERSION, + "build": {}, + "communities": [], + "analysis": {}, + "incremental": { + "files": {}, + "extractor_state": {}, + "topology_sources": [], + }, + "learning": {}, + "semantic": {"used": False}, + } + state.update(overrides) + return state + + +def community_records( + communities: dict[int, list[Any]], + *, + labels: dict[int, str] | None = None, + cohesion: dict[int, float] | None = None, + naming_source: str = "generated", +) -> list[dict[str, Any]]: + labels = labels or {} + cohesion = cohesion or {} + from .persistence import _encode_key + + return [ + { + "id": int(cid), + "members": list(members), + "name": labels.get(cid, f"Community {cid}"), + "naming_source": naming_source, + "signature": "sha256:" + hashlib.sha256( + "\n".join(sorted(_encode_key(member) for member in members)).encode("utf-8") + ).hexdigest(), + "cohesion": cohesion.get(cid), + "clustering": { + "algorithm": "helix-leiden", + "seed": 42, + "resolution": 1.0, + "randomness": 0.001, + "trials": 1, + "max_iterations": 100, + }, + } + for cid, members in sorted(communities.items()) + ] + + +def communities_from_state(state: dict[str, Any]) -> dict[int, list[Any]]: + result: dict[int, list[Any]] = {} + for record in state.get("communities", []): + if isinstance(record, dict) and isinstance(record.get("id"), int): + result[record["id"]] = list(record.get("members", [])) + return result + + +def labels_from_state(state: dict[str, Any]) -> dict[int, str]: + return { + record["id"]: record["name"] + for record in state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("name"), str) + } + + +def community_summaries( + graph: Any, + communities: dict[int, list[Any]], + labels: dict[int, str], +) -> list[dict[str, Any]]: + summaries = [] + for community_id, members in sorted(communities.items()): + member_set = set(members) + internal_edges = sum( + 1 + for edge in graph.edges() + if edge.source in member_set and edge.target in member_set + ) + summaries.append( + { + "id": community_id, + "name": labels.get(community_id, f"Community {community_id}"), + "node_count": len(members), + "internal_edge_count": internal_edges, + } + ) + return summaries diff --git a/graphify/hooks.py b/graphify/hooks.py index 77ef39cd3..fbc74c7ae 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -7,8 +7,8 @@ _HOOK_MARKER = "# graphify-hook-start" _HOOK_MARKER_END = "# graphify-hook-end" -_CHECKOUT_MARKER = "# graphify-checkout-hook-start" -_CHECKOUT_MARKER_END = "# graphify-checkout-hook-end" +_CHECKOUT_MARKER = _HOOK_MARKER +_CHECKOUT_MARKER_END = _HOOK_MARKER_END # __PINNED_PYTHON__ is replaced at install time with the absolute path of the # Python interpreter that ran `graphify hook install`. For uv-tool and pipx @@ -137,9 +137,10 @@ _md = (_root / _out) / 'memory' if _md.is_dir() and any(_md.glob('*.md')): from graphify.reflect import reflect as _reflect - _gj = (_root / _out) / 'graph.json' + from graphify.paths import project_graph_store as _graph_store + _gj = _graph_store(_root, _out) _reflect(memory_dir=_md, out_path=(_root / _out) / 'reflections' / 'LESSONS.md', - graph_path=_gj if _gj.exists() else None) + graph_path=_gj if _gj.is_dir() else None) except Exception: pass except TimeoutError as exc: @@ -178,9 +179,10 @@ _md = (_root / _out) / 'memory' if _md.is_dir() and any(_md.glob('*.md')): from graphify.reflect import reflect as _reflect - _gj = (_root / _out) / 'graph.json' + from graphify.paths import project_graph_store as _graph_store + _gj = _graph_store(_root, _out) _reflect(memory_dir=_md, out_path=(_root / _out) / 'reflections' / 'LESSONS.md', - graph_path=_gj if _gj.exists() else None) + graph_path=_gj if _gj.is_dir() else None) except Exception: pass except TimeoutError as exc: @@ -263,9 +265,7 @@ def _detached_launch(rebuild_body: str) -> str: # Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed). # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Keep extraction and generated artifacts deterministic across hook processes. export PYTHONHASHSEED=0 # Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients @@ -314,13 +314,11 @@ def _detached_launch(rebuild_body: str) -> str: _CHECKOUT_SCRIPT = """\ -# graphify-checkout-hook-start +# graphify-hook-start # Auto-rebuilds the knowledge graph (code only) when switching branches. # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Keep extraction and generated artifacts deterministic across hook processes. export PYTHONHASHSEED=0 # Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients @@ -362,7 +360,7 @@ def _detached_launch(rebuild_body: str) -> str: mkdir -p "$(dirname "$_GRAPHIFY_LOG")" export GRAPHIFY_REBUILD_LOG="$_GRAPHIFY_LOG" echo "[graphify] Branch switched - launching background rebuild (log: $_GRAPHIFY_LOG)" -""" + _detached_launch(_REBUILD_BODY_CHECKOUT) + """# graphify-checkout-hook-end +""" + _detached_launch(_REBUILD_BODY_CHECKOUT) + """# graphify-hook-end """ @@ -486,7 +484,7 @@ def _pinned_python() -> str: Applies the same allowlist used in _PYTHON_DETECT: rejects any character that is not a valid plain filesystem path character, preventing $(...), backtick, double-quote, semicolon, etc. from being injected into generated - shell scripts or the merge-driver command line. The allowlist includes ':' + shell scripts. The allowlist includes ':' and '\\' so Windows paths (C:\\...) are accepted. An empty return means callers must fall back to the `graphify` launcher on PATH — safe degradation. """ @@ -495,128 +493,6 @@ def _pinned_python() -> str: return sys.executable -def _merge_attr_line() -> str: - """The .gitattributes line assigning the graphify merge driver to graph.json. - - The graph lives under the configured output directory (graphify.paths, - GRAPHIFY_OUT env override). gitattributes patterns are repo-relative, so an - absolute output-dir override cannot be expressed there — fall back to the - default name in that case. - """ - from graphify.paths import GRAPHIFY_OUT - out = GRAPHIFY_OUT - if not out or Path(out).is_absolute() or "\\" in out: - out = "graphify-out" - return f"{out.rstrip('/')}/graph.json merge=graphify" - - -def _has_merge_attr(content: str) -> bool: - """True if a (non-comment) `<...>graph.json ... merge=graphify` line exists.""" - for raw in content.splitlines(): - line = raw.strip() - if not line or line.startswith("#"): - continue - fields = line.split() - if fields and fields[0].endswith("graph.json") and "merge=graphify" in fields[1:]: - return True - return False - - -def _register_merge_driver(root: Path) -> str: - """Register the graph.json union merge driver in git config + .gitattributes (#1902). - - README and CHANGELOG 0.7.0 document `graphify merge-driver` as being set up - by `hook install`, but install never actually registered it. Writes go - through `git config` (never hand-edit .git/config — in a linked worktree the - effective config is not at root/.git/config). The interpreter is pinned the - same way the hook scripts pin it, so the driver works even when the graphify - launcher is not on PATH at merge time. - """ - import subprocess as _sp - pinned = _pinned_python() - if pinned: - driver = f"{pinned} -m graphify merge-driver %O %A %B" - else: - driver = "graphify merge-driver %O %A %B" - try: - for key, value in ( - ("merge.graphify.name", "graphify graph.json union merge"), - ("merge.graphify.driver", driver), - ): - _sp.run( - ["git", "-C", str(root), "config", key, value], - check=True, capture_output=True, text=True, - ) - except (OSError, _sp.CalledProcessError) as exc: - return f"not registered (git config failed: {exc})" - - line = _merge_attr_line() - attrs = root / ".gitattributes" - if attrs.exists(): - content = attrs.read_text(encoding="utf-8") - if _has_merge_attr(content): - return f"already registered ({line})" - # Never clobber other entries; preserve a trailing newline. - if content and not content.endswith("\n"): - content += "\n" - attrs.write_text(content + line + "\n", encoding="utf-8", newline="\n") - else: - attrs.write_text(line + "\n", encoding="utf-8", newline="\n") - return f"registered ({line})" - - -def _unregister_merge_driver(root: Path) -> str: - """Remove the merge-driver git config keys and the .gitattributes line.""" - import subprocess as _sp - for key in ("merge.graphify.name", "merge.graphify.driver"): - try: - # --unset exits nonzero if the key is absent; that is fine. - _sp.run( - ["git", "-C", str(root), "config", "--unset", key], - capture_output=True, text=True, - ) - except OSError: - pass - attrs = root / ".gitattributes" - if not attrs.exists(): - return "not registered - nothing to remove." - content = attrs.read_text(encoding="utf-8") - kept = [ - raw for raw in content.splitlines() - if not _has_merge_attr(raw) - ] - if kept == content.splitlines(): - return "gitattributes entry not found - nothing to remove." - if kept: - # Other entries survive; the file stays. - attrs.write_text("\n".join(kept) + "\n", encoding="utf-8", newline="\n") - return "removed from .gitattributes (other entries preserved)" - attrs.unlink() - return "removed (.gitattributes deleted - no other entries)" - - -def _merge_driver_status(root: Path) -> str: - """Report whether the merge driver is registered (config + gitattributes).""" - import subprocess as _sp - try: - res = _sp.run( - ["git", "-C", str(root), "config", "--get", "merge.graphify.driver"], - capture_output=True, text=True, - ) - cfg_ok = res.returncode == 0 and bool(res.stdout.strip()) - except OSError: - cfg_ok = False - attrs = root / ".gitattributes" - attr_ok = attrs.exists() and _has_merge_attr(attrs.read_text(encoding="utf-8")) - if cfg_ok and attr_ok: - return "registered" - if cfg_ok: - return "partially registered (git config set, .gitattributes line missing)" - if attr_ok: - return "partially registered (.gitattributes line set, git config missing)" - return "not registered" - - def _user_hooks_dir(hooks_dir: Path) -> Path: """Return the user-editable hooks directory. @@ -652,9 +528,8 @@ def install(path: Path = Path(".")) -> str: commit_msg = _install_hook(hooks_dir, "post-commit", hook, _HOOK_MARKER) checkout_msg = _install_hook(hooks_dir, "post-checkout", checkout, _CHECKOUT_MARKER) - merge_msg = _register_merge_driver(root) - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" + return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}" def uninstall(path: Path = Path(".")) -> str: @@ -666,9 +541,8 @@ def uninstall(path: Path = Path(".")) -> str: hooks_dir = _user_hooks_dir(_hooks_dir(root)) commit_msg = _uninstall_hook(hooks_dir, "post-commit", _HOOK_MARKER, _HOOK_MARKER_END) checkout_msg = _uninstall_hook(hooks_dir, "post-checkout", _CHECKOUT_MARKER, _CHECKOUT_MARKER_END) - merge_msg = _unregister_merge_driver(root) - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" + return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}" def status(path: Path = Path(".")) -> str: @@ -686,5 +560,4 @@ def _check(name: str, marker: str) -> str: commit = _check("post-commit", _HOOK_MARKER) checkout = _check("post-checkout", _CHECKOUT_MARKER) - merge = _merge_driver_status(root) - return f"post-commit: {commit}\npost-checkout: {checkout}\nmerge driver: {merge}" + return f"post-commit: {commit}\npost-checkout: {checkout}" diff --git a/graphify/impact.py b/graphify/impact.py new file mode 100644 index 000000000..86c36d040 --- /dev/null +++ b/graphify/impact.py @@ -0,0 +1,82 @@ +"""Pull-request/file impact analysis over an active Helix generation.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +from typing import Iterable + +from .helix.model import LoadedGraph, node_attributes + + +def changed_files(base: str, *, cwd: str | Path = ".") -> list[str]: + """Return files changed from ``base`` to HEAD without invoking a shell.""" + result = subprocess.run( + ["git", "diff", "--name-only", "--diff-filter=ACMRD", f"{base}...HEAD"], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def analyze_impact( + loaded: LoadedGraph, + files: Iterable[str | Path], + *, + depth: int = 2, +) -> dict: + """Return nodes, dependencies, and communities affected by source files.""" + graph = loaded.graph + requested = {Path(file).as_posix() for file in files} + requested_names = {Path(file).name for file in requested} + if loaded.query is None: + raise RuntimeError("impact analysis requires the native Helix query interface") + candidates = loaded.query.candidate_ids(sorted(requested | requested_names)) + seeds = { + node_id + for node_id in candidates + if (attrs := node_attributes(graph, node_id)) is not None + if (source := attrs.get("source_file")) + and ( + Path(str(source)).as_posix() in requested + or Path(str(source)).name in requested_names + or any(Path(str(source)).as_posix().endswith(f"/{item}") for item in requested) + ) + } + impacted = set(seeds) + traversed: list[tuple] = [] + if seeds: + from helixdb.graph import TraversalOptions + + result = graph.traverse(TraversalOptions( + seeds=tuple(sorted(seeds, key=repr)), + max_depth=max(0, depth), + direction="both", + )) + impacted = {visit.node_id for visit in result.visits} + traversed = [(edge.source, edge.target) for edge in result.discovery_edges] + + membership = { + member: record.get("id") + for record in loaded.state.get("communities", []) + if isinstance(record, dict) + for member in record.get("members", []) + } + communities = sorted({ + community_id + for node in impacted + if isinstance((community_id := membership.get(node)), int) + }) + return { + "changed_files": sorted(requested), + "seed_nodes": sorted(seeds, key=str), + "impacted_nodes": sorted(impacted, key=str), + "impacted_communities": communities, + "traversed_edges": [list(edge) for edge in traversed], + "depth": depth, + } + + +__all__ = ["analyze_impact", "changed_files"] diff --git a/graphify/install.py b/graphify/install.py index 1a8d3e3d3..303bedf32 100644 --- a/graphify/install.py +++ b/graphify/install.py @@ -563,11 +563,6 @@ def _print_banner() -> None: if not sys.stdout.isatty(): return try: - if sys.platform == "win32": - import ctypes - ctypes.windll.kernel32.SetConsoleMode( - ctypes.windll.kernel32.GetStdHandle(-11), 7 - ) A = "\033[38;5;214m" D = "\033[38;5;130m" R = "\033[0m" @@ -976,7 +971,7 @@ def _antigravity_install(project_dir: Path) -> None: print(' "graphify": {') print(' "command": "uv",') print( - ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.json"]' + ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.helix"]' ) print(" }") def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: @@ -1033,7 +1028,7 @@ def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: Only use Read/Grep/Glob directly when: 1. graphify has already oriented you and you need to modify or debug specific lines -2. `graphify-out/graph.json` does not exist yet +2. `graphify-out/graph.helix` does not exist yet - If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files - Read `graphify-out/GRAPH_REPORT.md` only for broad architecture review when query/path/explain do not surface enough context @@ -1071,7 +1066,7 @@ def _cursor_uninstall(project_dir: Path) -> None: This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) @@ -1105,7 +1100,7 @@ def _devin_rules_uninstall(project_dir: Path) -> None: return { "tool.execute.before": async (input, output) => { if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; + if (!existsSync(join(directory, "graphify-out", "graph.helix"))) return; if (input.tool === "bash") { // Separate with ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a @@ -1252,7 +1247,7 @@ def _uninstall_kilo_plugin(project_dir: Path) -> None: f" {write_config_file.relative_to(project_dir)} -> plugin deregistered" ) # OpenCode tool.execute.before plugin — fires before every tool call. -# Injects a graph reminder into bash command output when graph.json exists. +# Injects a graph reminder into bash command output when graph.helix exists. _OPENCODE_PLUGIN_JS = """\ // graphify OpenCode plugin // Injects a knowledge graph reminder before bash tool calls when the graph exists. @@ -1271,7 +1266,7 @@ def _uninstall_kilo_plugin(project_dir: Path) -> None: return { "tool.execute.before": async (input, output) => { if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; + if (!existsSync(join(directory, "graphify-out", "graph.helix"))) return; if (input.tool === "bash") { // ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a statement diff --git a/graphify/llm.py b/graphify/llm.py index 8c34cde6b..7e7b3678e 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -608,7 +608,7 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: # slips through it. This closes that intra-file gap with a lenient substring # check and FLAGS (never drops) an unverifiable node with ``verification = # "unverified"``, surfaced by the caller (stderr), reported by the diagnostics, -# and left on the node in graph.json. +# and left on the node in native graph. # Short tokens (len < 3) are ignored: they match too readily to be evidence and # their absence is not a reliable fabrication signal, so skipping them avoids # false positives. @@ -1439,7 +1439,7 @@ def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bo # attached — what would you like me to do with it?"). That prose parses to # zero nodes/edges, so _response_is_hollow flags it as truncation and the # adaptive-retry path bisects the chunk indefinitely, never converging and - # never writing graph.json (verified against Claude Code 2.1.197). + # never writing native graph (verified against Claude Code 2.1.197). # # Putting the full extraction schema plus an explicit imperative in the # user turn — and dropping --system-prompt — makes the CLI emit the JSON @@ -1948,7 +1948,7 @@ def _strip_partial_markers(result: dict) -> None: Call this only AFTER the semantic cache has been saved (the save consumes the marker to stamp affected entries ``partial: True``). Stripping it keeps the - internal flag out of the graph.json nodes/edges the corpus result feeds into. + internal flag out of the native graph nodes/edges the corpus result feeds into. """ for bucket in ("nodes", "edges", "hyperedges"): for item in result.get(bucket, []): @@ -2161,7 +2161,8 @@ def extract_corpus_parallel( max_concurrency: int = 4, max_retry_depth: int = 3, deep_mode: bool = False, - cache_root: "Path | None" = None, + cache: dict | None = None, + cache_root: Path | None = None, ) -> dict: """Extract a corpus in chunks, merging results. @@ -2197,14 +2198,6 @@ def extract_corpus_parallel( output_tokens. Failed chunks are logged to stderr and skipped — one bad chunk does not abort the run. - ``cache_root`` (when given) is where per-chunk checkpoint cache entries are - written, decoupled from ``root`` which anchors content-hash keys and - ``source_file`` resolution — the same split the AST cache uses (#1774). - With ``--out``, cli.py passes the corpus as ``root`` and the output - directory as ``cache_root`` so checkpoints land where the recovery read - looks, instead of creating an unwanted ``graphify-out/`` inside the - analyzed source tree (#1990). - Accepts ``str`` paths as well as ``Path``; string entries are coerced up front so packing/slicing helpers can rely on ``Path`` semantics (#1386). """ @@ -2292,6 +2285,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # authoritative: pass the partial file set so its entry is # stamped ``partial: True`` and re-dispatched next run. partial_source_files=_partial_source_files(result) or None, + cache=cache, ) except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr) @@ -2314,7 +2308,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: else: # Merge in deterministic submission order, NOT completion order. Merging # as chunks finish makes the node/edge ordering in the returned corpus - # (and therefore graph.json) depend on which network call happened to + # (and therefore native graph) depend on which network call happened to # return first — so identical input churned run-to-run (#1632). Collect # results keyed by chunk index and merge in sorted order after the pool # drains; this matches the serial path's order. The progress callback @@ -2359,7 +2353,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # Out-of-scope node filter (#1895). The #1757 cache guard already refuses # to WRITE a cache entry for a node whose source_file is a real file that # was not dispatched, but the node itself still flowed into the merged - # result and landed in graph.json. Mirror the #1757 condition here: resolve + # result and landed in native graph. Mirror the #1757 condition here: resolve # each source_file against root and drop the node only when it resolves to # an existing file (.is_file()) outside the dispatched set — non-file # source_files (concepts, model-invented anchors) pass through untouched. @@ -2782,6 +2776,8 @@ def _community_label_lines(G, communities, gods, max_communities, top_k): representative node labels (god nodes first). Returns (lines, labeled_cids); skips communities with no resolvable nodes.""" # gods may be node-id strings or god_nodes() dicts ({"id": ..., "label": ...}). + from graphify.helix.model import node_attributes + god_set = {g["id"] if isinstance(g, dict) else g for g in (gods or [])} ordered = sorted(communities.items(), key=lambda kv: -len(kv[1])) lines: list[str] = [] @@ -2791,7 +2787,11 @@ def _community_label_lines(G, communities, gods, max_communities, top_k): names: list[str] = [] seen: set[str] = set() for nid in ranked: - label = str(G.nodes[nid].get("label", nid)) if nid in G.nodes else str(nid) + label = ( + str(node_attributes(G, nid).get("label", nid)) + if G.contains_node(nid) + else str(nid) + ) label = label.strip().strip("()")[:_LABEL_MAXLEN] if label and label.lower() not in seen: seen.add(label.lower()) diff --git a/graphify/manifest_ingest.py b/graphify/manifest_ingest.py index ae3aa61fc..55ee3f1a4 100644 --- a/graphify/manifest_ingest.py +++ b/graphify/manifest_ingest.py @@ -16,7 +16,7 @@ from __future__ import annotations import re -import xml.etree.ElementTree as ET +from defusedxml import ElementTree as ET from pathlib import Path from typing import Any @@ -92,7 +92,7 @@ def extract_package_manifest(path: Path) -> dict[str, Any]: seen.add(dep_nid) # The edge targets the dependency's canonical package id. If that package's # own manifest is in the corpus, the edge resolves to its (single) node; if - # the dependency is external, build_from_json prunes the dangling edge. We + # the dependency is external, build_from_extraction prunes the dangling edge. We # deliberately do NOT emit a stub node — a stub with an empty source_file # would risk clobbering the real node's source_file under id-dedup. edges.append({ diff --git a/graphify/multigraph_compat.py b/graphify/multigraph_compat.py deleted file mode 100644 index 7ac62e275..000000000 --- a/graphify/multigraph_compat.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Runtime compatibility probe for Graphify MultiDiGraph mode. - -Verifies that the current NetworkX runtime supports the behaviors a future -opt-in --multigraph build will rely on. The probe is BEHAVIOR-based, not -version-based — both NX 3.4.2 (Py 3.10 lane) and NX 3.6.1+ (Py 3.11+ lane) -pass. The probe result is cached for the process lifetime via lru_cache. - -No call sites added yet; downstream multigraph PRs will gate on -require_multigraph_capabilities() before enabling MDG mode. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from functools import lru_cache -import sys -from typing import Any - -import networkx as nx -from networkx.readwrite import json_graph - - -@dataclass(frozen=True) -class CapabilityCheck: - name: str - ok: bool - detail: str - - -@dataclass(frozen=True) -class MultigraphCapabilityResult: - python_version: str - networkx_version: str - checks: tuple[CapabilityCheck, ...] - - @property - def ok(self) -> bool: - return all(check.ok for check in self.checks) - - @property - def failed(self) -> tuple[CapabilityCheck, ...]: - return tuple(check for check in self.checks if not check.ok) - - def error_message(self) -> str: - if self.ok: - return ( - "Graphify MultiDiGraph capability probe passed " - f"(Python {self.python_version}, NetworkX {self.networkx_version})." - ) - failed = "; ".join(f"{check.name}: {check.detail}" for check in self.failed) - return ( - "error: --multigraph requires NetworkX keyed MultiDiGraph node-link " - "round-trip support. " - f"Detected Python {self.python_version}, NetworkX {self.networkx_version}. " - f"Failed capability check(s): {failed}. " - "Default simple graph mode remains available." - ) - - -def _check(name: str, func: Callable[[], bool | str]) -> CapabilityCheck: - try: - detail = func() - except Exception as exc: - return CapabilityCheck(name, False, f"{type(exc).__name__}: {exc}") - if detail is True: - return CapabilityCheck(name, True, "ok") - if isinstance(detail, str): - return CapabilityCheck(name, False, detail) - return CapabilityCheck(name, False, f"unexpected result {detail!r}") - - -def _build_probe_graph() -> nx.MultiDiGraph: - graph = nx.MultiDiGraph() - graph.add_node("a", label="A") - graph.add_node("b", label="B") - graph.add_edge("a", "b", key="calls:a.py:L1", relation="calls", source_file="a.py") - graph.add_edge("a", "b", key="imports:a.py:L2", relation="imports", source_file="a.py") - return graph - - -def _probe_keyed_parallel_edges() -> bool | str: - graph = _build_probe_graph() - if not graph.is_multigraph() or not graph.is_directed(): - return f"probe graph type was {type(graph).__name__}" - if graph.number_of_edges("a", "b") != 2: - return f"expected 2 keyed parallel edges, got {graph.number_of_edges('a', 'b')}" - keys = set(graph["a"]["b"].keys()) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if keys != expected: - return f"expected keys {sorted(expected)}, got {sorted(keys)}" - return True - - -def _probe_node_link_round_trip() -> bool | str: - graph = _build_probe_graph() - data = json_graph.node_link_data(graph, edges="links") - if data.get("multigraph") is not True: - return f"serialized multigraph flag was {data.get('multigraph')!r}" - if data.get("directed") is not True: - return f"serialized directed flag was {data.get('directed')!r}" - links = data.get("links") - if not isinstance(links, list) or len(links) != 2: - length = 0 if not isinstance(links, list) else len(links) - return f"serialized links length was {length}" - serialized_keys: set[str] = set() - for edge in links: - if isinstance(edge, dict): - edge_key = edge.get("key") - if isinstance(edge_key, str): - serialized_keys.add(edge_key) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if serialized_keys != expected: - return f"serialized keys {sorted(serialized_keys)} did not match {sorted(expected)}" - loaded = json_graph.node_link_graph(data, edges="links") - if not isinstance(loaded, nx.MultiDiGraph): - return f"round-trip graph type was {type(loaded).__name__}" - if loaded.number_of_edges("a", "b") != 2: - return f"round-trip edge count was {loaded.number_of_edges('a', 'b')}" - loaded_keys = set(loaded["a"]["b"].keys()) - if loaded_keys != expected: - return f"round-trip keys {sorted(loaded_keys)} did not match {sorted(expected)}" - return True - - -def _probe_duplicate_key_overwrite_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("x", "y", key="same", marker="first") - graph.add_edge("x", "y", key="same", marker="second") - edges = list(graph.edges(keys=True, data=True)) - if len(edges) != 1: - return f"expected one edge after duplicate-key add, got {len(edges)}" - if edges[0][3].get("marker") != "second": - return f"expected second attr overwrite, got {edges[0][3].get('marker')!r}" - return True - - -def _probe_reserved_key_attr_rejected() -> bool | str: - """Verify the Python language guarantee that NetworkX add_edge inherits. - - Python forbids passing the same keyword argument twice — once explicitly - and once via **kwargs. This probe confirms that protection still applies - to nx.MultiDiGraph.add_edge: a future loader that builds attrs from JSON - will be reliably protected from accidentally setting `key` via attrs while - also passing `key=` explicitly. - - The probe always passes on any Python 3.x version. Its purpose is to - document the invariant explicitly in the probe suite so that if a future - Python version relaxes this rule (extremely unlikely), the probe surfaces - the regression. - """ - graph = nx.MultiDiGraph() - attrs: dict[str, Any] = {"key": "attr-key", "relation": "calls"} - try: - graph.add_edge("a", "b", key="schema-key", **attrs) - except TypeError: - return True - return "add_edge accepted duplicate key keyword and attr; loader must not rely on this" - - -def _probe_remove_edges_from_two_tuple_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("a", "b", key="one") - graph.add_edge("a", "b", key="two") - graph.remove_edges_from([("a", "b")]) - remaining = graph.number_of_edges("a", "b") - if remaining != 1: - return f"expected one remaining edge after two-tuple removal, got {remaining}" - return True - - -def _probe_to_undirected_preserves_multigraph_type() -> bool | str: - graph = _build_probe_graph() - undirected = graph.to_undirected() - undirected_view = graph.to_undirected(as_view=True) - if not isinstance(undirected, nx.MultiGraph): - return f"to_undirected() returned {type(undirected).__name__}" - if not isinstance(undirected_view, nx.MultiGraph): - return f"to_undirected(as_view=True) returned {type(undirected_view).__name__}" - return True - - -@lru_cache(maxsize=1) -def probe_multigraph_capabilities() -> MultigraphCapabilityResult: - checks = ( - _check("keyed_parallel_edges", _probe_keyed_parallel_edges), - _check("node_link_edges_links_round_trip", _probe_node_link_round_trip), - _check("duplicate_key_overwrite_semantics", _probe_duplicate_key_overwrite_semantics), - _check("reserved_key_attr_rejected", _probe_reserved_key_attr_rejected), - _check( - "remove_edges_from_two_tuple_semantics", - _probe_remove_edges_from_two_tuple_semantics, - ), - _check( - "to_undirected_preserves_multigraph_type", - _probe_to_undirected_preserves_multigraph_type, - ), - ) - return MultigraphCapabilityResult( - python_version=( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ), - networkx_version=nx.__version__, - checks=checks, - ) - - -def require_multigraph_capabilities() -> MultigraphCapabilityResult: - result = probe_multigraph_capabilities() - if not result.ok: - raise RuntimeError(result.error_message()) - return result diff --git a/graphify/operations.py b/graphify/operations.py new file mode 100644 index 000000000..0aab04883 --- /dev/null +++ b/graphify/operations.py @@ -0,0 +1,154 @@ +"""Atomic operations over an existing active Helix generation.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +from .analyze import god_nodes, surprising_connections, suggest_questions +from .cluster import cluster, remap_communities_to_previous, score_all +from .helix.persistence import DEFAULT_PROJECT_STORE, HelixEmbeddedStore +from .helix.state import ( + communities_from_state, + community_records, + community_summaries, + labels_from_state, +) + + +def recluster(store_path: str | Path = DEFAULT_PROJECT_STORE) -> dict[int, list]: + """Replace community records and node membership in one generation.""" + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = cluster(graph) + previous_membership = { + member: community_id + for community_id, members in communities_from_state(state).items() + for member in members + } + if previous_membership: + communities = remap_communities_to_previous( + communities, previous_membership + ) + cohesion = score_all(graph, communities) + previous_labels = labels_from_state(state) + labels = { + community_id: previous_labels.get( + community_id, f"Community {community_id}" + ) + for community_id in communities + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source="preserved" if previous_labels else "generated", + ) + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return communities + + +def reanalyze(store_path: str | Path = DEFAULT_PROJECT_STORE) -> dict: + """Replace analysis records while preserving topology and other state.""" + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = communities_from_state(state) + labels = labels_from_state(state) + analysis = state.get("analysis", {}) + report_inputs = analysis.get("report_inputs", {}) if isinstance(analysis, dict) else {} + refreshed = { + "god_nodes": god_nodes(graph), + "surprises": surprising_connections(graph, communities), + "suggested_questions": suggest_questions(graph, communities, labels), + "community_summaries": community_summaries(graph, communities, labels), + "report_inputs": report_inputs, + } + state["analysis"] = refreshed + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return refreshed + + +def relabel( + store_path: str | Path = DEFAULT_PROJECT_STORE, + *, + backend: str | None = None, + model: str | None = None, + missing_only: bool = False, + max_concurrency: int = 4, + batch_size: int = 100, +) -> dict[int, str]: + """Name native communities and activate the labels with the same topology.""" + from .cluster import label_communities_by_hub + from .llm import detect_backend, label_communities + + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = communities_from_state(state) + previous = labels_from_state(state) + selected = backend or detect_backend() + to_label = { + cid: members + for cid, members in communities.items() + if not ( + missing_only + and cid in previous + and not previous[cid].startswith("Community ") + ) + } + if selected and to_label: + generated = label_communities( + graph, + to_label, + backend=selected, + model=model, + gods=state.get("analysis", {}).get("god_nodes", []), + max_concurrency=max_concurrency, + batch_size=batch_size, + ) + elif to_label: + generated = label_communities_by_hub(graph, to_label) + else: + generated = {} + labels = { + cid: ( + previous[cid] + if missing_only + and cid in previous + and not previous[cid].startswith("Community ") + else generated.get(cid, f"Community {cid}") + ) + for cid in communities + } + cohesion = { + record["id"]: float(record["cohesion"]) + for record in state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("cohesion"), (int, float)) + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source=selected or "native-hub", + ) + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return labels + + +__all__ = ["reanalyze", "recluster", "relabel"] diff --git a/graphify/paths.py b/graphify/paths.py index d43bc7766..2a4ca2d0c 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -294,11 +294,15 @@ def out_path(*parts: str) -> Path: return Path(GRAPHIFY_OUT, *parts) -def default_graph_json() -> str: - """Default ``graph.json`` path under the configured output dir. - - The package-wide fallback used by serve/build/benchmark/prs and the CLI read - commands so a ``GRAPHIFY_OUT`` override is honoured everywhere, not just where - the path is passed explicitly (#1423). - """ - return str(out_path("graph.json")) +def default_graph_store() -> str: + """Default embedded Helix store under the configured output directory.""" + return str(out_path("graph.helix")) + + +def project_graph_store( + root: str | Path = Path("."), output_dir: str | Path | None = None +) -> Path: + """Resolve a project's native store for hooks and other project workflows.""" + output = Path(GRAPHIFY_OUT if output_dir is None else output_dir) + base = output if output.is_absolute() else Path(root) / output + return base / "graph.helix" diff --git a/graphify/prs.py b/graphify/prs.py index 9534e6c00..5adbdf843 100644 --- a/graphify/prs.py +++ b/graphify/prs.py @@ -25,12 +25,10 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import TYPE_CHECKING +from typing import Any -if TYPE_CHECKING: - import networkx as nx - -from graphify.paths import default_graph_json as _default_graph_json +from graphify.helix.model import LoadedGraph, graphify_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, load_graph # ── ANSI colours ───────────────────────────────────────────────────────────── @@ -73,7 +71,7 @@ class PRInfo: updated_at: datetime expected_base: str = "main" # set by fetch_prs via _detect_default_branch worktree_path: str | None = None - # Graph impact — populated when graph.json exists + # Graph impact — populated when a Helix store exists communities_touched: list[int] = field(default_factory=list) nodes_affected: int = 0 files_changed: list[str] = field(default_factory=list) @@ -161,8 +159,10 @@ def _detect_default_branch(repo: str | None = None) -> str: if repo: args += ["--repo", repo] data = _gh(*args) - if data and data.get("defaultBranchRef", {}).get("name"): - return data["defaultBranchRef"]["name"] + if isinstance(data, dict): + branch = data.get("defaultBranchRef") + if isinstance(branch, dict) and isinstance(branch.get("name"), str): + return branch["name"] # Fall back to git symbolic-ref for the current repo try: result = subprocess.run( @@ -240,7 +240,7 @@ def fetch_pr_files(number: int, repo: str | None = None) -> list[str]: return [] -# ── Graph-native impact (used by MCP tools — works on nx.Graph directly) ───── +# ── Graph-native impact (used by MCP tools) ────────────────────────────────── def _path_match(graph_src: str, pr_file: str) -> bool: """True if graph_src and pr_file refer to the same file (path-boundary safe).""" @@ -249,7 +249,13 @@ def _path_match(graph_src: str, pr_file: str) -> bool: return graph_src.endswith("/" + pr_file) or pr_file.endswith("/" + graph_src) -def compute_pr_impact(files: list[str], G: "nx.Graph") -> tuple[list[int], int]: +def compute_pr_impact( + files: list[str], + G: Any, + communities: dict[int, list[Any]] | None = None, + *, + native_query: Any, +) -> tuple[list[int], int]: """Return (communities_touched, nodes_affected) for a set of changed files. Builds a file→(communities, count) index first so lookup is O(nodes + files) @@ -258,14 +264,23 @@ def compute_pr_impact(files: list[str], G: "nx.Graph") -> tuple[list[int], int]: # Build index once file_comms: dict[str, set[int]] = {} file_count: dict[str, int] = {} - for _, data in G.nodes(data=True): + membership = { + node_id: cid + for cid, members in (communities or {}).items() + for node_id in members + } + for node_id in native_query.candidate_ids(files): + node = G.node(node_id) + if node is None: + continue + data = graphify_attributes(node.attributes) src = data.get("source_file") or "" if not src: continue if src not in file_comms: file_comms[src] = set() file_count[src] = 0 - c = data.get("community") + c = membership.get(node_id, data.get("community")) if c is not None: file_comms[src].add(int(c)) file_count[src] += 1 @@ -324,27 +339,38 @@ def fetch_worktrees() -> dict[str, str]: # ── Graph impact analysis ───────────────────────────────────────────────────── -def _load_graph_json(graph_path: Path) -> dict | None: +def _load_graph_store(graph_path: Path) -> LoadedGraph | None: if not graph_path.exists(): return None - from graphify.security import check_graph_file_size_cap try: - check_graph_file_size_cap(graph_path) - return json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, ValueError): + return load_graph(graph_path) + except (OSError, RuntimeError, ValueError): return None -def build_community_labels(data: dict, top_n: int = 4) -> dict[int, list[str]]: - """Return {community_id: [top_labels]} extracted from graph node data.""" +def build_community_labels( + loaded: LoadedGraph | dict[str, Any], top_n: int = 4 +) -> dict[int, list[str]]: + """Return community IDs mapped to representative native node labels.""" comm_labels: dict[int, list[str]] = defaultdict(list) - for node in data.get("nodes", []): - c = node.get("community") - if c is None: + if isinstance(loaded, dict): + for node in loaded.get("nodes", []): + if not isinstance(node, dict) or node.get("community") is None: + continue + label = node.get("label") or node.get("id") + if label: + comm_labels[int(node["community"])].append(str(label)) + return {cid: labels[:top_n] for cid, labels in comm_labels.items()} + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): continue - label = node.get("label") or node.get("id") or "" - if label: - comm_labels[int(c)].append(label) + cid = record["id"] + for node_id in record.get("members", []): + node = loaded.graph.node(node_id) + if node is None: + continue + attrs = graphify_attributes(node.attributes) + comm_labels[cid].append(str(attrs.get("label") or node_id)) return {c: labels[:top_n] for c, labels in comm_labels.items()} @@ -352,25 +378,10 @@ def attach_graph_impact( prs: list[PRInfo], graph_path: Path, repo: str | None = None ) -> dict[int, list[str]]: """Fetch PR file lists concurrently, compute graph impact, return community labels.""" - data = _load_graph_json(graph_path) - if not data: + loaded = _load_graph_store(graph_path) + if loaded is None: return {} - # Build file → {community, node_count} index - file_to_communities: dict[str, set[int]] = {} - file_to_nodes: dict[str, int] = {} - for node in data.get("nodes", []): - src = node.get("source_file") or "" - if not src: - continue - comm = node.get("community") - if src not in file_to_communities: - file_to_communities[src] = set() - file_to_nodes[src] = 0 - if comm is not None: - file_to_communities[src].add(int(comm)) - file_to_nodes[src] += 1 - # Fetch diffs concurrently — gh pr diff is the bottleneck (network I/O) actionable = [pr for pr in prs if pr.status != "WRONG-BASE"] workers = min(8, len(actionable)) if actionable else 1 @@ -387,19 +398,62 @@ def attach_graph_impact( files = [] pr.files_changed = files - comms: set[int] = set() - nodes = 0 - matched: set[str] = set() - for f in files: - for gf, gcomms in file_to_communities.items(): - if gf not in matched and _path_match(gf, f): - comms |= gcomms - nodes += file_to_nodes.get(gf, 0) - matched.add(gf) - pr.communities_touched = sorted(comms) - pr.nodes_affected = nodes + # Project only nodes matching changed paths through public Helix predicates. + # Chunk the OR terms so a PR touching many files cannot create an unbounded + # query plan, then use native point reads for exact path matching. + if loaded.query is None: + raise RuntimeError("PR impact requires the native Helix query interface") + changed_files = sorted({path for pr in actionable for path in pr.files_changed}) + terms = list(dict.fromkeys( + value + for path in changed_files + for value in (path, Path(path).name) + if value + )) + candidate_ids: list[Any] = [] + seen_candidates: set[Any] = set() + for offset in range(0, len(terms), 64): + for node_id in loaded.query.candidate_ids(terms[offset : offset + 64]): + if node_id not in seen_candidates: + seen_candidates.add(node_id) + candidate_ids.append(node_id) + + file_to_communities: dict[str, set[int]] = {} + file_to_nodes: dict[str, int] = {} + communities = { + record["id"]: list(record.get("members", [])) + for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + } + membership = {node: cid for cid, members in communities.items() for node in members} + for node_id in candidate_ids: + node = loaded.graph.node(node_id) + if node is None: + continue + attrs = graphify_attributes(node.attributes) + src = str(attrs.get("source_file") or "") + if not src or not any(_path_match(src, path) for path in changed_files): + continue + comm = membership.get(node_id) + file_to_communities.setdefault(src, set()) + file_to_nodes[src] = file_to_nodes.get(src, 0) + 1 + if comm is not None: + file_to_communities[src].add(int(comm)) + + for pr in actionable: + comms: set[int] = set() + nodes = 0 + matched: set[str] = set() + for path in pr.files_changed: + for graph_file, graph_communities in file_to_communities.items(): + if graph_file not in matched and _path_match(graph_file, path): + comms |= graph_communities + nodes += file_to_nodes.get(graph_file, 0) + matched.add(graph_file) + pr.communities_touched = sorted(comms) + pr.nodes_affected = nodes - return build_community_labels(data) + return build_community_labels(loaded) # ── Dashboard rendering ─────────────────────────────────────────────────────── @@ -494,7 +548,7 @@ def render_conflicts( ) -> None: actionable = [p for p in prs if p.base_branch == base and p.communities_touched] if not actionable: - print(dim("\n No graph impact data - run with a valid graph.json to detect conflicts.\n")) + print(dim("\n No graph impact data - build a valid graph.helix store to detect conflicts.\n")) return # Build community → [PRs] map @@ -686,7 +740,7 @@ def cmd_prs(argv: list[str]) -> None: do_conflicts = False show_wrong_base = False pr_number: int | None = None - graph_path = Path(_default_graph_json()) + graph_path = DEFAULT_PROJECT_STORE i = 0 while i < len(argv): diff --git a/graphify/reflect.py b/graphify/reflect.py index 4e04e0589..ec0a96fe4 100644 --- a/graphify/reflect.py +++ b/graphify/reflect.py @@ -17,7 +17,7 @@ trusted lesson. When a graph is in hand, source nodes that no longer exist are dropped. It is deterministic: no LLM, stable sort orders, byte-stable output for a given input -and a given ``now``. When a graph (`graph.json` + `.graphify_analysis.json`) is available +and a given ``now``. When a Helix generation is available the lessons are also grouped by community label; without it they degrade to a single flat section. @@ -39,11 +39,6 @@ _UNCATEGORIZED = "Uncategorized" -# Derived experiential layer written alongside graph.json (a SIDECAR, kept -# separate from the durable structural truth in graph.json — no learning_* -# fields are ever stamped into the graph itself). Read-surface annotations are -# merged in at display time from this file. -LEARNING_SIDECAR_NAME = ".graphify_learning.json" _LEARNING_SCHEMA_VERSION = 1 _PROVENANCE_CAP = 5 # most-recent (question, date, outcome) entries per node @@ -159,52 +154,30 @@ def load_memory_docs(memory_dir: Path) -> list[dict[str, Any]]: # --- graph / community lookup (optional) --------------------------------------- -def _load_node_community(graph_path: Path, analysis_path: Path, - labels_path: Path) -> dict[str, str] | None: +def _load_node_community(graph_path: Path) -> dict[str, str] | None: """Build a lookup from node id AND node label -> community label, or None if the graph isn't available. - Mirrors how `graphify export wiki` reads graph.json + .graphify_analysis.json + - .graphify_labels.json. Community membership in the analysis sidecar is keyed by - node id, but `save-result` cites nodes by label, so both are mapped — otherwise a - cited ``build_from_json()`` never finds its community and every lesson collapses - into Uncategorized. Best-effort: any missing/unparseable artifact disables grouping. + Community state and topology are read from the same active generation. """ - if not graph_path.exists() or not analysis_path.exists(): - return None try: - analysis = json.loads(analysis_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - communities = analysis.get("communities", {}) - if not communities: + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + loaded = load_graph(graph_path) + except (OSError, RuntimeError, ValueError): return None - labels: dict[str, str] = {} - if labels_path.exists(): - try: - labels = json.loads(labels_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - labels = {} - # id -> label from the graph, so a label-form citation resolves to a community too. - id_to_label: dict[str, str] = {} - try: - gdata = json.loads(graph_path.read_text(encoding="utf-8")) - for n in gdata.get("nodes", []): - if isinstance(n, dict) and n.get("id") is not None and n.get("label") is not None: - id_to_label[str(n["id"])] = str(n["label"]) - except (OSError, ValueError): - id_to_label = {} - # Sorted cid iteration + setdefault makes any label collision resolve - # deterministically (smallest community id wins). node_community: dict[str, str] = {} - for cid in sorted(communities, key=str): - label = labels.get(str(cid)) or labels.get(cid) or f"Community {cid}" - for nid in communities[cid]: - nid = str(nid) - node_community.setdefault(nid, label) - nlabel = id_to_label.get(nid) - if nlabel is not None: - node_community.setdefault(nlabel, label) + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): + continue + label = str(record.get("name") or f"Community {record['id']}") + for node_id in record.get("members", []): + node_community.setdefault(str(node_id), label) + node = loaded.graph.node(node_id) + if node is not None: + attrs = graphify_attributes(node.attributes) + node_community.setdefault(str(attrs.get("label", node_id)), label) return node_community @@ -214,26 +187,24 @@ def _load_known_nodes(graph_path: Path) -> set[str] | None: Used to drop source nodes from lessons once the code they pointed at is gone (deleted/renamed) — a stale lesson shouldn't keep getting recommended. Both ids and labels are collected because `save-result` records source nodes by their - human-readable label (what an agent cites, e.g. ``build_from_json()``), while - graph nodes are keyed by id (e.g. ``module_build_from_json``). Matching on either + human-readable label (what an agent cites, e.g. ``build_from_extraction()``), while + graph nodes are keyed by id (e.g. ``module_build_from_extraction``). Matching on either keeps a still-present node and only drops one that survives under neither name — indexing ids alone silently dropped every label-form citation (the common case). """ try: - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - nodes = data.get("nodes") - if not isinstance(nodes, list): + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + graph = load_graph(graph_path).graph + except (OSError, RuntimeError, ValueError): return None known: set[str] = set() - for n in nodes: - if not isinstance(n, dict): - continue - if n.get("id") is not None: - known.add(str(n["id"])) - if n.get("label") is not None: - known.add(str(n["label"])) + for node in graph.nodes(): + known.add(str(node.id)) + attrs = graphify_attributes(node.attributes) + if attrs.get("label") is not None: + known.add(str(attrs["label"])) return known or None @@ -525,11 +496,9 @@ def _topic_key(label: str) -> tuple[int, str]: def lessons_fresh(out_path: Path, memory_dir: Path, - graph_path: Path | None = None, - analysis_path: Path | None = None, - labels_path: Path | None = None) -> bool: + graph_path: Path | None = None) -> bool: """True if ``out_path`` exists and is at least as new as every input that - feeds it (the memory docs, and the graph/sidecars when one is used). + feeds it (the memory docs and native store when one is used). Lets ``graphify reflect --if-stale`` skip a redundant run — e.g. when the git post-commit hook just regenerated ``LESSONS.md`` and an agent then runs reflect @@ -550,7 +519,7 @@ def lessons_fresh(out_path: Path, memory_dir: Path, newest = max(newest, f.stat().st_mtime) except OSError: pass - for input_path in (graph_path, analysis_path, labels_path): + for input_path in (graph_path,): if input_path is None: continue gp = Path(input_path) @@ -563,8 +532,6 @@ def lessons_fresh(out_path: Path, memory_dir: Path, def reflect(memory_dir: Path, out_path: Path, graph_path: Path | None = None, - analysis_path: Path | None = None, - labels_path: Path | None = None, *, now: datetime | None = None, half_life_days: float = _DEFAULT_HALF_LIFE_DAYS, @@ -581,11 +548,7 @@ def reflect(memory_dir: Path, out_path: Path, known_nodes = None if graph_path is not None: graph_path = Path(graph_path) - analysis_path = Path(analysis_path) if analysis_path else ( - graph_path.parent / ".graphify_analysis.json") - labels_path = Path(labels_path) if labels_path else ( - graph_path.parent / ".graphify_labels.json") - node_community = _load_node_community(graph_path, analysis_path, labels_path) + node_community = _load_node_community(graph_path) known_nodes = _load_known_nodes(graph_path) if now is None: @@ -599,51 +562,43 @@ def reflect(memory_dir: Path, out_path: Path, out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(render_lessons_md(agg), encoding="utf-8") - # Also project a derived experiential sidecar next to graph.json when a graph - # is in hand. Best-effort: a sidecar failure must never break LESSONS.md. + # Persist the experiential projection inside a new atomic Helix generation. if graph_path is not None: - try: - write_learning_sidecar(agg, Path(graph_path), now=now) - except Exception: - pass + write_learning_state(agg, Path(graph_path), now=now) return out_path, agg -# --- work-memory overlay sidecar ------------------------------------------------ -# -# A derived, experiential projection of the reflect aggregate, written next to -# graph.json as ``.graphify_learning.json``. It carries which nodes have proven -# preferred/tentative/contested, a code fingerprint for staleness detection, and -# a short provenance trail. graph.json (durable structural truth) is never -# touched — read surfaces merge this overlay in only at display time. +# --- work-memory overlay ------------------------------------------------------- def _build_id_label_maps(graph_path: Path) -> tuple[dict[str, str], dict[str, list[str]], dict[str, dict[str, Any]]]: - """From graph.json build: + """From an active Helix generation build: - ``id_set``: id -> id (every node id, so an id-form citation resolves to itself) - ``label_to_ids``: label -> [ids] (so a label-form citation can be resolved, and ambiguity — one label, many ids — can be detected and skipped) - ``node_by_id``: id -> node dict (for source_file lookup) - Best-effort; an unreadable/garbage graph yields empty maps. + Best-effort; an unreadable store yields empty maps. """ id_set: dict[str, str] = {} label_to_ids: dict[str, list[str]] = {} node_by_id: dict[str, dict[str, Any]] = {} try: - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - except (OSError, ValueError): + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + graph = load_graph(graph_path).graph + except (OSError, RuntimeError, ValueError): return id_set, label_to_ids, node_by_id - for n in data.get("nodes", []): - if not isinstance(n, dict) or n.get("id") is None: - continue - nid = str(n["id"]) + for node in graph.nodes(): + attrs = graphify_attributes(node.attributes) + nid = str(node.id) id_set[nid] = nid - node_by_id[nid] = n - label = n.get("label") + node_by_id[nid] = {"id": node.id, **attrs} + label = attrs.get("label") if label is not None: label_to_ids.setdefault(str(label), []).append(nid) return id_set, label_to_ids, node_by_id @@ -668,9 +623,8 @@ def _resolve_canonical_id(cited: str, id_set: dict[str, str], def _resolve_source_path(src: str, graph_path: Path) -> Path | None: """Locate a node's ``source_file`` on disk, returning an existing file or None. - ``source_file`` is stored relative to the PROJECT root, but graph.json may - live in ``/graphify-out/`` (so its own dir is not the root) or directly - at the root (``extract --out .``). Resolve the root in the most-likely order + ``source_file`` is stored relative to the project root, while graph.helix + normally lives in ``/graphify-out/``. Resolve roots in the most-likely order and return the first candidate where the file actually exists, so a defeated heuristic or a stale marker can never strand the file (every node would then look "changed"). The same search runs at write and read time, so the writer @@ -678,8 +632,8 @@ def _resolve_source_path(src: str, graph_path: Path) -> Path | None: Order: the committed ``.graphify_root`` marker (#686/#1423 — authoritative for an absolute/elsewhere ``GRAPHIFY_OUT`` override); then the layout-appropriate - root *first* — graph.json's parent's parent for the ``graphify-out`` layout, - or graph.json's own dir for a flat layout — which avoids matching a same-named + root first — graph.helix's parent's parent for the standard layout, + or the store's parent for a flat layout — which avoids matching a same-named file one directory up; then the other of the two; then the cwd. """ if not src: @@ -821,24 +775,23 @@ def _add(entry_src: dict[str, Any], status: str) -> None: } -def write_learning_sidecar(agg: dict[str, Any], graph_path: Path, - *, now: datetime | None = None) -> Path: - """Write ``.graphify_learning.json`` next to ``graph_path`` deterministically. +def write_learning_state(agg: dict[str, Any], graph_path: Path, + *, now: datetime | None = None) -> Path: + """Atomically persist the learning projection with native topology.""" + import copy + from graphify.helix.persistence import HelixEmbeddedStore - Sorted keys + indent=2 so re-runs on identical input (and a fixed ``now``) - are byte-identical. Returns the sidecar path. - """ overlay = build_learning_overlay(agg, graph_path, now=now) - sidecar = Path(graph_path).parent / LEARNING_SIDECAR_NAME - sidecar.write_text( - json.dumps(overlay, indent=2, sort_keys=True, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - return sidecar + with HelixEmbeddedStore(graph_path) as store: + previous_state = store.read_state() + state = copy.deepcopy(previous_state) + state["learning"] = overlay + store.replace_state(state, previous_state=previous_state) + return Path(graph_path) def load_learning_overlay(graph_path: Path) -> dict[str, dict[str, Any]]: - """Load the sidecar next to ``graph_path`` and return ``{node_id -> entry}`` + """Load native learning state and return ``{node_id -> entry}`` with a recomputed ``stale: bool`` per entry. Best-effort -> {} on any error. Staleness: recompute ``file_hash(source_file)`` and compare to the entry's @@ -847,10 +800,11 @@ def load_learning_overlay(graph_path: Path) -> dict[str, dict[str, Any]]: direction). An entry with no stored fingerprint AND no current file is not marked stale (nothing to re-verify). """ - sidecar = Path(graph_path).parent / LEARNING_SIDECAR_NAME try: - data = json.loads(sidecar.read_text(encoding="utf-8")) - except (OSError, ValueError): + from graphify.helix.persistence import load_graph + + data = dict(load_graph(graph_path).state.get("learning", {})) + except (OSError, RuntimeError, ValueError): return {} nodes = data.get("nodes") if not isinstance(nodes, dict): diff --git a/graphify/report.py b/graphify/report.py index 248bce9a1..d2d2d58c2 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -2,7 +2,9 @@ from __future__ import annotations import re from datetime import date -import networkx as nx +from typing import Any + +from .helix.model import edge_attributes, graphify_attributes, node_attributes def _safe_community_name(label: str) -> str: @@ -13,12 +15,10 @@ def _safe_community_name(label: str) -> str: def load_learning_for_report(graph_path) -> dict | None: - """Assemble the report's work-memory inputs from sibling artifacts. + """Assemble work-memory inputs from native state and memory documents. - Reads the ``.graphify_learning.json`` overlay (preferred sources) next to - ``graph_path`` and re-aggregates the memory docs for the query-scoped - dead-ends. Best-effort: returns None if neither is available, so the report - simply omits the section. Never raises. + Best-effort: returns None if neither is available, so the report simply + omits the section. Never raises. """ from pathlib import Path as _Path try: @@ -69,7 +69,7 @@ def _learning_section(lines: list, learning: dict | None, top_n: int = 10) -> No def generate( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], cohesion_scores: dict[int, float], community_labels: dict[int, str], @@ -90,13 +90,16 @@ def generate( if community_labels: community_labels = {int(k) if isinstance(k, str) else k: v for k, v in community_labels.items()} - confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] + edge_rows = [ + (edge.source, edge.target, edge_attributes(edge)) for edge in G.edges() + ] + confidences = [data.get("confidence", "EXTRACTED") for _, _, data in edge_rows] total = len(confidences) or 1 ext_pct = round(confidences.count("EXTRACTED") / total * 100) inf_pct = round(confidences.count("INFERRED") / total * 100) amb_pct = round(confidences.count("AMBIGUOUS") / total * 100) - inf_edges = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "INFERRED"] + inf_edges = [(u, v, data) for u, v, data in edge_rows if data.get("confidence") == "INFERRED"] inf_scores = [d.get("confidence_score", 0.5) for _, _, d in inf_edges] inf_avg = round(sum(inf_scores) / len(inf_scores), 2) if inf_scores else None @@ -125,7 +128,7 @@ def generate( lines += [ "", "## Summary", - f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities" + f"- {G.node_count} nodes · {G.edge_count} edges · {len(communities)} communities" + (f" ({shown_count} shown, {thin_count_summary} thin omitted)" if thin_count_summary else ""), f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS" + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""), @@ -189,10 +192,11 @@ def generate( # noise there ("None detected" on every run). Emit it only when the graph # actually contains code (#1657). _has_code = any( - d.get("file_type") == "code" for _, d in G.nodes(data=True) + graphify_attributes(node.attributes).get("file_type") == "code" + for node in G.nodes() ) or any( d.get("relation") in ("imports", "imports_from") - for *_e, d in G.edges(data=True) + for *_e, d in edge_rows ) if _has_code: from .analyze import find_import_cycles @@ -209,7 +213,8 @@ def generate( else: lines.append("- None detected.") - hyperedges = G.graph.get("hyperedges", []) + metadata = dict(G.attributes).get("graph", {}) + hyperedges = metadata.get("hyperedges", []) if isinstance(metadata, dict) else [] if hyperedges: lines += ["", "## Hyperedges (group relationships)"] for h in hyperedges: @@ -229,7 +234,7 @@ def generate( continue if len(real_nodes) < min_community_size: continue - display = [G.nodes[n].get("label", n) for n in real_nodes[:8]] + display = [node_attributes(G, n).get("label", n) for n in real_nodes[:8]] suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else "" lines += [ "", @@ -238,12 +243,12 @@ def generate( f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}", ] - ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"] + ambiguous = [(u, v, data) for u, v, data in edge_rows if data.get("confidence") == "AMBIGUOUS"] if ambiguous: lines += ["", "## Ambiguous Edges - Review These"] for u, v, d in ambiguous: - ul = G.nodes[u].get("label", u) - vl = G.nodes[v].get("label", v) + ul = node_attributes(G, u).get("label", u) + vl = node_attributes(G, v).get("label", v) lines += [ f"- `{ul}` → `{vl}` [AMBIGUOUS]", f" {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}", @@ -253,11 +258,11 @@ def generate( from .analyze import _is_file_node, _is_concept_node isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 - and not _is_file_node(G, n) - and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + node.id for node in G.nodes() + if G.degree(node.id).degree <= 1 + and not _is_file_node(G, node.id) + and not _is_concept_node(G, node.id) + and graphify_attributes(node.attributes).get("file_type") != "rationale" ] thin_communities = { cid: nodes for cid, nodes in communities.items() @@ -268,7 +273,7 @@ def generate( if gap_count > 0 or amb_pct > 20: lines += ["", "## Knowledge Gaps"] if isolated: - isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]] + isolated_labels = [node_attributes(G, n).get("label", n) for n in isolated[:5]] suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else "" lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}") lines.append(" These have ≤1 connection - possible missing edges or undocumented components.") @@ -278,7 +283,7 @@ def generate( lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.") # --- Work-memory lessons (derived overlay) --- - # Preferred sources come from the .graphify_learning.json sidecar; the + # Preferred sources come from native learning state; the # query-scoped dead-ends come from the reflect aggregate. Section omitted # entirely when neither is present, so a graph with no work-memory is # byte-identical to the pre-feature report. diff --git a/graphify/scip_ingest.py b/graphify/scip_ingest.py index bf3d1857c..04f887de0 100644 --- a/graphify/scip_ingest.py +++ b/graphify/scip_ingest.py @@ -14,7 +14,7 @@ extraction result format. All edges emitted are endpoint-safe — the function builds a symbol → node_id index in a first pass and either resolves relationship targets via that index or creates a stub - external node so `build_from_json()` will keep the edge. + external node so `build_from_extraction()` will keep the edge. Supported (simplified) JSON shape: documents[]: { relative_path, language, symbols[] } diff --git a/graphify/security.py b/graphify/security.py index 2dbe5bd77..0bf73ca67 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -21,50 +21,6 @@ _MAX_FETCH_BYTES = 52_428_800 # 50 MB hard cap for binary downloads _MAX_TEXT_BYTES = 10_485_760 # 10 MB hard cap for HTML / text -# Graph-load memory-bomb cap: reject .json files larger than this before -# JSON-parsing them into a dict. Without this, a multi-gigabyte (or -# specifically crafted) graph.json can exhaust process memory during -# json.loads + node_link_graph rehydration. -# Default fallback cap. Kept as a module-level constant so the value is -# discoverable and so existing callers/tests that reference it directly keep -# working; the effective cap is resolved at call time by -# ``_max_graph_file_bytes`` (which lets ``GRAPHIFY_MAX_GRAPH_BYTES`` override it). -_MAX_GRAPH_FILE_BYTES = 512 * 1024 * 1024 # 512 MiB - - -def _max_graph_file_bytes() -> int: - """Return the graph.json size cap in bytes. - - Honors the ``GRAPHIFY_MAX_GRAPH_BYTES`` environment variable so users with - large codebases can raise the limit without editing source. The value may - be plain bytes (``671088640``) or carry an ``MB`` / ``GB`` suffix - (``640MB``, ``2GB`` — case-insensitive, binary multipliers: ``MB`` is - 1024*1024 and ``GB`` is 1024*1024*1024, i.e. MiB / GiB). - Falls back to ``_MAX_GRAPH_FILE_BYTES`` (512 MiB) when the env var is unset, - blank, or unparseable. - - Read fresh on every call so the env var can be set before import and still - take effect. - """ - raw = os.environ.get("GRAPHIFY_MAX_GRAPH_BYTES", "").strip() - if not raw: - return _MAX_GRAPH_FILE_BYTES - text = raw.upper() - multiplier = 1 - if text.endswith("GB"): - multiplier = 1024 * 1024 * 1024 - text = text[:-2].strip() - elif text.endswith("MB"): - multiplier = 1024 * 1024 - text = text[:-2].strip() - try: - value = int(text) - except ValueError: - return _MAX_GRAPH_FILE_BYTES - if value <= 0: - return _MAX_GRAPH_FILE_BYTES - return value * multiplier - # AWS metadata, link-local, and common cloud metadata endpoints _BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"} @@ -308,81 +264,6 @@ def safe_fetch_text(url: str, max_bytes: int = _MAX_TEXT_BYTES, timeout: int = 1 return raw.decode("utf-8", errors="replace") -# --------------------------------------------------------------------------- -# Path validation -# --------------------------------------------------------------------------- - -def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: - """Resolve *path* and verify it stays inside *base*. - - *base* defaults to the `graphify-out` directory relative to CWD. - Also requires the base directory to exist, so a caller cannot - trick graphify into reading files before any graph has been built. - - Raises: - ValueError - path escapes base, or base does not exist - FileNotFoundError - resolved path does not exist - """ - if base is None: - resolved_hint = Path(path).resolve() - for candidate in [resolved_hint, *resolved_hint.parents]: - if candidate.name == GRAPHIFY_OUT_NAME: - base = candidate - break - if base is None: - base = Path(GRAPHIFY_OUT).resolve() - - base = base.resolve() - if not base.exists(): - raise ValueError( - f"Graph base directory does not exist: {base}. " - "Run /graphify first to build the graph." - ) - - resolved = Path(path).resolve() - try: - resolved.relative_to(base) - except ValueError: - raise ValueError( - f"Path {path!r} escapes the allowed directory {base}. " - "Only paths inside graphify-out/ are permitted." - ) - - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - - return resolved - - -def check_graph_file_size_cap(path: Path) -> None: - """Reject *path* if its size exceeds the configured graph-file cap. - - Protects callers from memory bombs by failing fast before a multi-GiB - graph.json is read into memory and JSON-parsed. Silently returns when - ``path.stat()`` cannot be read — the caller's own existence/path check - is expected to surface a clearer error in that case. - - The cap is resolved on every call via :func:`_max_graph_file_bytes`, so the - ``GRAPHIFY_MAX_GRAPH_BYTES`` env var can be set before import and still - apply. - - Raises: - ValueError - file size exceeds the cap. The message includes the - observed size, the cap, and how to raise the limit. - """ - cap = _max_graph_file_bytes() - try: - size = path.stat().st_size - except OSError: - return - if size > cap: - raise ValueError( - f"graph file {path} is {size:_d} bytes, exceeds {cap:_d}-byte cap\n" - f"(set GRAPHIFY_MAX_GRAPH_BYTES= or " - f"GRAPHIFY_MAX_GRAPH_BYTES=GB to raise the limit)" - ) - - # --------------------------------------------------------------------------- # Label sanitisation (mirrors code-review-graph's _sanitize_name pattern) # --------------------------------------------------------------------------- @@ -405,6 +286,18 @@ def sanitize_label(text: str | None) -> str: return text +def validate_store_path(path: str | Path) -> Path: + """Resolve and validate an embedded Helix store directory.""" + resolved = Path(path).expanduser().resolve() + if resolved.suffix.lower() == ".json" or resolved.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; pass a graph.helix store and rebuild from source" + ) + if not resolved.is_dir(): + raise FileNotFoundError(f"Helix store not found: {resolved}") + return resolved + + # --------------------------------------------------------------------------- # Metadata sanitisation (recursive, bounded, HTML-safe) # --------------------------------------------------------------------------- diff --git a/graphify/semantic_cleanup.py b/graphify/semantic_cleanup.py index 09cac2d84..30963413d 100644 --- a/graphify/semantic_cleanup.py +++ b/graphify/semantic_cleanup.py @@ -5,7 +5,7 @@ # `graphify merge-chunks` command — both ingest untrusted agent-written chunk # JSON, and validate_semantic_fragment() rejects malformed/oversized payloads and # crafted node/edge IDs before they touch the graph. The primary build/load paths -# (build_from_json, load_graph_json) deliberately do NOT run this: they must keep +# Build DTO assembly deliberately does not run this: it must keep # loading valid pre-existing graphs whose AST node IDs predate the stricter # semantic-ID charset. from __future__ import annotations @@ -83,7 +83,7 @@ def validate_semantic_fragment(fragment: object) -> list[str]: _validate_semantic_id(errors, f"nodes[{i}].id", node.get("id")) # file_type is intentionally NOT rejected here. It carries no security # risk (it can't exhaust memory or escape a directory), and - # build_from_json already coerces every value via _FILE_TYPE_SYNONYMS + # build_from_extraction already coerces every value via _FILE_TYPE_SYNONYMS # (unknown -> "concept", #840). Rejecting a whole chunk over a synonym # like "markdown"/"tool"/"framework" that the loader would happily map is # pure data loss, so leave file_type normalization to build. diff --git a/graphify/serve.py b/graphify/serve.py index f32a91673..f526a9451 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -4,14 +4,12 @@ import math import re import sys -from array import array from pathlib import Path -from typing import NamedTuple -import networkx as nx -from networkx.readwrite import json_graph -from graphify.security import sanitize_label, check_graph_file_size_cap +from typing import Any, NamedTuple +from graphify.security import sanitize_label, validate_store_path from graphify.build import edge_data, edge_datas -from graphify.paths import default_graph_json as _default_graph_json +from graphify.helix.model import LoadedGraph, edge_attributes, graphify_attributes, node_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, HelixGraphReader, load_graph try: import jieba as _jieba # type: ignore[import-untyped] @@ -19,57 +17,23 @@ _jieba = None -def _load_graph(graph_path: str) -> nx.Graph: +def _load_graph(graph_path: str) -> LoadedGraph: try: - resolved = Path(graph_path).resolve() - if resolved.suffix != ".json": - raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - check_graph_file_size_cap(resolved) - safe = resolved - data = json.loads(safe.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(data.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` for path-qualified IDs.", - file=sys.stderr, - ) - except Exception: - pass - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - # Attach the work-memory overlay (derived sidecar next to graph.json) so - # the query/MCP read surface can annotate NODE lines display-only. Empty - # when no sidecar exists, leaving un-annotated output byte-identical. - try: - from graphify.reflect import load_learning_overlay as _llo - G.graph["_learning_overlay"] = _llo(resolved) - except Exception: - G.graph["_learning_overlay"] = {} - return G - except json.JSONDecodeError as exc: - print(f"error: graph.json is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) - sys.exit(1) + return load_graph(validate_store_path(graph_path)) except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) + except RuntimeError as exc: + print(f"error: Helix store is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) + sys.exit(1) -def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]: - """Reconstruct community dict from community property stored on nodes.""" +def _communities_from_graph(G: LoadedGraph) -> dict[int, list[str]]: + """Read communities from the same durable Helix generation.""" communities: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid = data.get("community") - if cid is not None: - communities.setdefault(int(cid), []).append(node_id) + for record in G.state.get("communities", []): + if isinstance(record, dict) and isinstance(record.get("id"), int): + communities[record["id"]] = list(record.get("members", [])) return communities @@ -190,135 +154,14 @@ def _query_terms(question: str) -> list[str]: _SOURCE_MATCH_BONUS = 0.5 -def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: - """IDF weights for query terms, cached in G.graph['_idf_cache']. - - Common terms like 'error' or 'exception' that match hundreds of nodes get - low weights; rare identifiers like 'FooBarService' get high weights. - Cache is stored on the graph object itself so it auto-invalidates when - a hot-reload replaces G with a new object. - """ - cache: dict[str, float] = G.graph.setdefault("_idf_cache", {}) - N = G.number_of_nodes() or 1 - uncached = [t for t in terms if t not in cache] - if uncached: - df: dict[str, int] = {t: 0 for t in uncached} - for _, data in G.nodes(data=True): - norm_label = ( - data.get("norm_label") or _strip_diacritics(data.get("label") or "") - ).lower() - for t in uncached: - if t in norm_label: - df[t] += 1 - for t in uncached: - cache[t] = math.log(1 + N / (1 + df[t])) - return {t: cache.get(t, math.log(1 + N)) for t in terms} - - -def _trigrams(text: str) -> set[str]: - """Character trigrams of `text`; for <3-char text the whole string is the key.""" - if len(text) < 3: - return {text} if text else set() - return {text[i:i + 3] for i in range(len(text) - 2)} - - -def _node_search_text(data: dict, nid: str) -> str: - """Concatenate every field _score_nodes / _find_node match a query against, so - one trigram index over this text is a complete candidate generator for both. - - - `norm_label` and `source_file` feed _score_nodes' per-term substring tiers. - - `label_tokens` (the space-joined token form) feeds _find_node's - `term in label_tokens` branch, where a multi-word `term` can span a token - boundary that punctuation hides in `norm_label` (e.g. query "foo bar" matches - label "foo.bar" only via its tokenized form). - - `source_tokens` feeds _find_node's exact source-file path lookup, where a - query like "app/api/example/route.ts" tokenizes to "app api example route ts". - - `nid` feeds the whole-query `joined == nid_lower` tier. - - NUL separators stop a trigram from spanning two fields (a query never contains - NUL, so a cross-field trigram can never be a real match). - """ - norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() - label_tokens = " ".join(_search_tokens(data.get("label") or "")) - source = (data.get("source_file") or "").lower() - source_tokens = " ".join(_search_tokens(data.get("source_file") or "")) - return "\x00".join((norm_label, label_tokens, str(nid).lower(), source, source_tokens)) - - -def _get_trigram_index(G: nx.Graph) -> dict: - """Lazily build and cache a trigram -> node-position postings map on the graph. - - Cached on `G.graph` so it auto-invalidates when a hot-reload swaps in a - fresh graph object, exactly like `_idf_cache`. `set_cache` memoizes per-trigram - id-sets across queries within one graph generation. - """ - idx = G.graph.get("_trigram_index") - if idx is not None: - return idx - ids = list(G.nodes()) - postings: dict[str, array] = {} - for i, nid in enumerate(ids): - for g in _trigrams(_node_search_text(G.nodes[nid], nid)): - bucket = postings.get(g) - if bucket is None: - bucket = array("i") - postings[g] = bucket - bucket.append(i) - idx = {"ids": ids, "postings": postings, "set_cache": {}} - G.graph["_trigram_index"] = idx - return idx - - -def _trigram_candidates(G: nx.Graph, needles: list[str], *, guard_frac: float = 0.10) -> list[str] | None: - """Node IDs whose text could contain any `needle` as a substring, via the - trigram index — a *superset* the caller then re-scores with the exact predicates. - - Returns candidates in graph-iteration order (so order-sensitive callers like - _find_node stay byte-identical to a full scan), or **None** when the index isn't - worth it — a needle is too short to trigram, or its rarest trigram is still - common enough that the candidate set would approach the whole graph. The caller - falls back to the full scan, preserving the never-worse contract. The guard is - cheap: postings-length lookups only, no set intersection. - """ - idx = _get_trigram_index(G) - ids, postings, set_cache = idx["ids"], idx["postings"], idx["set_cache"] - n = len(ids) - if n == 0: - return [] - needles = [s for s in needles if s] - thresh = int(n * guard_frac) - for s in needles: - tgs = _trigrams(s) - if not tgs or any(len(g) < 3 for g in tgs): - return None # too short to trigram-filter - present = [len(postings[g]) for g in tgs if g in postings] - if not present: - continue # this needle matches nothing — contributes no candidates - if min(present) > thresh: - return None # rarest trigram still too common -> not worth the index - cand: set[int] = set() - for s in needles: - sets: list[set] | None = [] - for g in _trigrams(s): - bucket = postings.get(g) - if bucket is None: - sets = None # a trigram absent everywhere -> needle matches nothing - break - cached = set_cache.get(g) - if cached is None: - cached = set(bucket) - set_cache[g] = cached - sets.append(cached) - if not sets: - continue - sets.sort(key=len) # intersect smallest-first - hit = set(sets[0]) - for other in sets[1:]: - hit &= other - if not hit: - break - cand |= hit - return [ids[i] for i in sorted(cand)] +def _compute_idf(G: Any, terms: list[str], native_query: Any) -> dict[str, float]: + """Compute exact label document frequencies with public Helix predicates.""" + frequencies = native_query.document_frequencies(terms) + node_count = G.node_count or 1 + return { + term: math.log(1 + node_count / (1 + frequencies.get(term, 0))) + for term in terms + } class _QueryScores(NamedTuple): @@ -337,7 +180,9 @@ class _QueryScores(NamedTuple): best_seed_by_term: dict[str, str] -def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: +def _score_nodes( + G: Any, terms: list[str], *, native_query: Any +) -> list[tuple[float, str]]: """Combined query scorer returning the existing ranked `(score, node_id)` list. Backwards-compatible thin wrapper around `_score_query` for path, explain, @@ -345,11 +190,17 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: per-term seed metadata computed by `_score_query` (when requested) is discarded here so existing callers see no API or runtime-cost change. """ - return _score_query(G, terms, collect_per_term_seeds=False).ranked + return _score_query( + G, terms, collect_per_term_seeds=False, native_query=native_query + ).ranked def _score_query( - G: nx.Graph, terms: list[str], *, collect_per_term_seeds: bool + G: Any, + terms: list[str], + *, + collect_per_term_seeds: bool, + native_query: Any, ) -> _QueryScores: """Single-pass combined scorer that optionally also records the best seed for each normalized query token. @@ -383,21 +234,16 @@ def _score_query( # below it would also inflate the matched-term ratio (#1602). norm_terms = list(dict.fromkeys(tok for t in terms for tok in _search_tokens(t))) n_terms = len(norm_terms) - idf = _compute_idf(G, norm_terms) + idf = _compute_idf(G, norm_terms, native_query) # Whole-query string for full-label matching (mirrors _find_node's `term`). joined = " ".join(norm_terms) # Weight the full-query bonus by the rarest constituent term so a specific # multi-word label still outweighs common-token noise; floor at 1.0. joined_w = max((idf.get(t, 1.0) for t in norm_terms), default=1.0) - # Trigram prefilter: score only nodes whose text could match a term, falling - # back to the whole graph when the index isn't selective. The result is - # identical either way — the per-node scoring below is unchanged and a - # non-candidate node always scores 0. (IDF above stays a whole-graph statistic.) - candidate_ids = _trigram_candidates(G, norm_terms + ([joined] if joined else [])) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) + candidate_ids = native_query.candidate_ids( + norm_terms + ([joined] if joined else []) ) + node_iter = ((nid, node_attributes(G, nid)) for nid in candidate_ids) # Per-token best tracking, only when the caller (the query path) wants the # seed metadata. The key tuple is the full multi-key tie-break # (`(-singleton_score, -degree, label_len, nid)`), so `min` over the @@ -422,7 +268,7 @@ def _score_query( # the per-token singleton tier (joined-singlet exact-match check). When # neither runs (`joined` empty AND not collecting seeds) skip the call; # this preserves the single-query-time perf where nid_lower was lazy. - nid_lower = nid.lower() if (joined or collect_per_term_seeds) else "" + nid_lower = str(nid).lower() if (joined or collect_per_term_seeds) else "" score = 0.0 # Full-query tier: a multi-word query that equals (or prefixes) the whole # label must dominate the per-token bag-of-words sums below, so `path`/ @@ -501,7 +347,7 @@ def _score_query( # (-singleton, -degree, label_len, nid) — the minimum # tuple wins, exactly matching max(tied, key=degree) # over (label_len asc, nid asc)-sorted ties. - key = (-singleton, -G.degree(nid), len(data.get("label") or nid), nid) + key = (-singleton, -G.degree(nid).degree, len(data.get("label") or str(nid)), str(nid)) cur = best_by_term.get(t) if cur is None or key < cur[0]: best_by_term[t] = (key, nid) @@ -511,14 +357,14 @@ def _score_query( scored.append((score, nid)) # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. - scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) + scored.sort(key=lambda s: (-s[0], len(node_attributes(G, s[1]).get("label") or str(s[1])), str(s[1]))) best_seed_by_term: dict[str, str] = {} if collect_per_term_seeds and best_by_term: best_seed_by_term = {t: nid for t, (_key, nid) in best_by_term.items()} return _QueryScores(ranked=scored, best_seed_by_term=best_seed_by_term) -def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: str) -> str: +def _pick_scored_endpoint(G: Any, scored: list[tuple[float, str]], query: str) -> str: """Pick a path endpoint from a _score_nodes result, preferring full-token matches. The full-query tier in _score_nodes only fires when the query equals or @@ -537,7 +383,7 @@ def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: s if not qtokens: return scored[0][1] for _score, nid in scored: - if qtokens <= set(_search_tokens(G.nodes[nid].get("label") or nid)): + if qtokens <= set(_search_tokens(node_attributes(G, nid).get("label") or str(nid))): return nid return scored[0][1] @@ -547,7 +393,7 @@ def _pick_seeds( max_k: int = 3, gap_ratio: float = 0.2, *, - G: "nx.Graph | None" = None, + G: Any | None = None, best_seed_by_term: dict[str, str] | None = None, ) -> list[str]: """Select BFS seed nodes, stopping when score drops too far below the top. @@ -593,7 +439,7 @@ def _pick_seeds( def _seed_label_key(nid: str) -> str: if G is None: return nid - data = G.nodes[nid] + data = node_attributes(G, nid) return (data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower()) or nid @@ -720,80 +566,113 @@ def _resolve_context_filters(question: str, explicit_filters: list[str] | None = return [], None -def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> nx.Graph: - filters = set(_normalize_context_filters(context_filters)) - if not filters: - return G - H = G.__class__() - H.add_nodes_from(G.nodes(data=True)) - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - for u, v, key, data in G.edges(keys=True, data=True): - if data.get("context") in filters: - H.add_edge(u, v, key=key, **data) - else: - for u, v, data in G.edges(data=True): - if data.get("context") in filters: - H.add_edge(u, v, **data) - return H +def _filter_graph_by_context(G: Any, context_filters: list[str] | None) -> tuple[Any, set[str]]: + """Return the native snapshot plus normalized traversal edge filters.""" + return G, set(_normalize_context_filters(context_filters)) -def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: +def _traverse( + G: Any, + start_nodes: list[str], + depth: int, + *, + strategy: str, + context_filters: set[str] | None = None, + native_query: Any | None = None, +) -> tuple[set[str], list[Any]]: # Compute hub threshold: nodes above this degree are not expanded as transit. # p99 of degree distribution, floored at 50 to avoid over-blocking small graphs. - degrees = [G.degree(n) for n in G.nodes()] + degrees = [int(row.degree) for row in G.degrees()] if degrees: degrees_sorted = sorted(degrees) p99_idx = int(len(degrees_sorted) * 0.99) hub_threshold = max(50, degrees_sorted[p99_idx]) else: hub_threshold = 50 - seed_set = set(start_nodes) - visited: set[str] = set(start_nodes) - frontier = set(start_nodes) - edges_seen: list[tuple] = [] - for _ in range(depth): - next_frontier: set[str] = set() - for n in frontier: - # Don't expand through high-degree hubs (except seeds - a hub that - # is the starting node should still be explored). - if n not in seed_set and G.degree(n) >= hub_threshold: + if not context_filters: + from helixdb import TraversalOptions + + result = G.traverse(TraversalOptions( + seeds=tuple(start_nodes), + max_depth=depth, + strategy=strategy, + direction="both", + stop_non_seed_at_or_above_degree=hub_threshold, + )) + return ( + {visit.node_id for visit in result.visits}, + [G.edge(item.edge_id) for item in result.discovery_edges if G.edge(item.edge_id) is not None], + ) + + if native_query is None: + raise RuntimeError("context-filtered traversal requires a native Helix query") + visited = set(native_query.traverse_ids( + start_nodes, depth, contexts=context_filters + )) + edges_seen: list[Any] = [] + seen_edge_ids: set[Any] = set() + for node in visited: + for edge_id in G.incident_edge_ids(node): + if edge_id in seen_edge_ids: + continue + edge = G.edge(edge_id) + if ( + edge is None + or edge.source not in visited + or edge.target not in visited + or edge_attributes(edge).get("context") not in context_filters + ): continue - for neighbor in G.neighbors(n): - if neighbor not in visited: - next_frontier.add(neighbor) - edges_seen.append((n, neighbor)) - visited.update(next_frontier) - frontier = next_frontier + seen_edge_ids.add(edge_id) + edges_seen.append(edge) return visited, edges_seen -def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: - degrees = [G.degree(n) for n in G.nodes()] - if degrees: - degrees_sorted = sorted(degrees) - p99_idx = int(len(degrees_sorted) * 0.99) - hub_threshold = max(50, degrees_sorted[p99_idx]) - else: - hub_threshold = 50 - seed_set = set(start_nodes) - visited: set[str] = set() - edges_seen: list[tuple] = [] - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, d = stack.pop() - if node in visited or d > depth: - continue - visited.add(node) - if node not in seed_set and G.degree(node) >= hub_threshold: - continue - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, d + 1)) - edges_seen.append((node, neighbor)) - return visited, edges_seen +def _bfs( + G: Any, + start_nodes: list[str], + depth: int, + context_filters: set[str] | None = None, + *, + native_query: Any | None = None, +): + return _traverse( + G, + start_nodes, + depth, + strategy="breadth_first", + context_filters=context_filters, + native_query=native_query, + ) + + +def _dfs( + G: Any, + start_nodes: list[str], + depth: int, + context_filters: set[str] | None = None, + *, + native_query: Any | None = None, +): + return _traverse( + G, + start_nodes, + depth, + strategy="depth_first", + context_filters=context_filters, + native_query=native_query, + ) -def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000, *, seeds: list[str] | None = None) -> str: +def _subgraph_to_text( + G: Any, + nodes: set[str], + edges: list[Any], + token_budget: int = 2000, + *, + seeds: list[str] | None = None, + learning_overlay: dict[str, Any] | None = None, +) -> str: """Render subgraph as text, cutting at token_budget (approx 3 chars/token). seeds: exact-match nodes rendered first before the degree-sorted expansion, @@ -801,9 +680,7 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu """ char_budget = token_budget * 3 lines = [] - # Work-memory overlay (derived sidecar) stashed on the graph at load time. - # Empty when no sidecar exists, so un-annotated output stays byte-identical. - overlay = getattr(G, "graph", {}).get("_learning_overlay", {}) or {} + overlay = learning_overlay or {} seed_set = set(seeds or []) seed_hits = [n for n in (seeds or []) if n in nodes] # Rank non-seed nodes by hop distance from the seeds so the node that answers @@ -813,7 +690,7 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu # layers here over BOTH edge directions. Deterministic: neighbor iteration is # insertion-ordered and the sort key ends in str(n) (no hash-order). def _adj(n): - if G.is_directed(): + if G.directed: yield from G.successors(n) yield from G.predecessors(n) else: @@ -831,10 +708,10 @@ def _adj(n): frontier = nxt ordered = seed_hits + sorted( nodes - seed_set, - key=lambda n: (dist.get(n, 1 << 30), -G.degree(n), str(n)), + key=lambda n: (dist.get(n, 1 << 30), -G.degree(n).degree, str(n)), ) for nid in ordered: - d = G.nodes[nid] + d = node_attributes(G, nid) # Every LLM-derived field passes through sanitize_label before being # concatenated into MCP tool output (F-010): an attacker who controls a # corpus document can otherwise inject ANSI escapes, fake graphify-out @@ -856,24 +733,10 @@ def _adj(n): f"{learning_suffix}]" ) lines.append(line) - for u, v in edges: + for traversed in edges: + u, v = traversed.source, traversed.target if u in nodes and v in nodes: - raw = G[u][v] - d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw - # (u, v) is BFS/DFS visit order, not necessarily the true edge - # direction: on an undirected graph G.neighbors() walks callers - # and callees alike, so a caller->callee edge renders backwards - # whenever the callee is visited first. _src/_tgt (stashed on the - # edge data by the `query` CLI loader) carry the real direction; - # fall back to (u, v) for graphs/edges that don't set them. - src = d.get("_src", u) - tgt = d.get("_tgt", v) - # Guard against a stray/dangling _src/_tgt (hand-edited or adversarial - # graph.json): only trust them when they name exactly this edge's - # endpoints, else fall back to (u, v). Without this, G.nodes[src] - # would KeyError on an unknown id (#2080 review). - if {src, tgt} != {u, v}: - src, tgt = u, v + d = edge_attributes(traversed) context = d.get("context") context_suffix = f" context={sanitize_label(str(context))}" if context else "" # The relation SITE (call/import/reference line in the source's @@ -885,10 +748,10 @@ def _adj(n): if _loc else "" ) line = ( - f"EDGE {sanitize_label(G.nodes[src].get('label', src))} " + f"EDGE {sanitize_label(node_attributes(G, u).get('label', u))} " f"--{sanitize_label(str(d.get('relation', '')))} " f"[{sanitize_label(str(d.get('confidence', '')))}{context_suffix}]--> " - f"{sanitize_label(G.nodes[tgt].get('label', tgt))}{at_suffix}" + f"{sanitize_label(node_attributes(G, v).get('label', v))}{at_suffix}" ) lines.append(line) output = "\n".join(lines) @@ -949,13 +812,15 @@ def _cut_lines_to_budget(lines: list[str], token_budget: int, narrow_hint: str) def _query_graph_text( - G: nx.Graph, + G: Any, question: str, *, + native_query: Any, mode: str = "bfs", depth: int = 3, token_budget: int = 2000, context_filters: list[str] | None = None, + learning_overlay: dict[str, Any] | None = None, ) -> str: terms = _query_terms(question) # One graph scoring pass produces both the combined ranking (used to drive @@ -964,16 +829,37 @@ def _query_graph_text( # — one combined + one per query token — re-walking the whole graph each # time; on a 100k-node, three-term benchmark ~71% of scoring time was # spent in those redundant per-term passes. - qs = _score_query(G, terms, collect_per_term_seeds=True) + qs = _score_query( + G, + terms, + collect_per_term_seeds=True, + native_query=native_query, + ) start_nodes = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) - traversal_graph = _filter_graph_by_context(G, resolved_filters) - nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + traversal_graph, edge_filters = _filter_graph_by_context(G, resolved_filters) + nodes, edges = ( + _dfs( + traversal_graph, + start_nodes, + depth, + edge_filters, + native_query=native_query, + ) + if mode == "dfs" + else _bfs( + traversal_graph, + start_nodes, + depth, + edge_filters, + native_query=native_query, + ) + ) header_parts = [ f"Traversal: {mode.upper()} depth={depth}", - f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", + f"Start: {[node_attributes(G, n).get('label', n) for n in start_nodes]}", ] if resolved_filters: header_parts.append(f"Context: {', '.join(resolved_filters)} ({filter_source})") @@ -982,10 +868,17 @@ def _query_graph_text( # Pass the seeds so the queried symbol renders first and survives truncation # (#BUG2): a branch merge had silently dropped this argument, leaving the # seed-first ordering as dead code. - return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget, seeds=start_nodes) + return header + _subgraph_to_text( + traversal_graph, + nodes, + edges, + token_budget, + seeds=start_nodes, + learning_overlay=learning_overlay, + ) -def _find_node(G: nx.Graph, label: str) -> list[str]: +def _find_node(G: Any, label: str, *, native_query: Any) -> list[str]: """Return node IDs whose label or ID matches the search term (diacritic-insensitive). Results are ordered by precedence: exact source-file path match first, then @@ -1006,19 +899,14 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: exact: list[str] = [] prefix: list[str] = [] substring: list[str] = [] - # Trigram prefilter (graph-iteration order preserved so exact/prefix/substring - # ordering — and thus matches[0] — is byte-identical to the full scan). - candidate_ids = _trigram_candidates(G, [term, norm_query]) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) - ) + candidate_ids = native_query.candidate_ids([term, norm_query]) + node_iter = ((nid, node_attributes(G, nid)) for nid in candidate_ids) for nid, d in node_iter: norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower() bare_label = norm_label.rstrip("()") label_tokens = " ".join(_search_tokens(d.get("label") or "")) source_tokens = " ".join(_search_tokens(d.get("source_file") or "")) - nid_lower = nid.lower() + nid_lower = str(nid).lower() if term == source_tokens: source_exact.append(nid) elif ( @@ -1042,11 +930,12 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: query_basename = _strip_diacritics(Path(label).name).lower() preferred = [] for nid in source_exact: - if str(G.nodes[nid].get("source_location", "")) != "L1": + attrs = node_attributes(G, nid) + if str(attrs.get("source_location", "")) != "L1": continue # File-node label is the bare basename OR a directory-qualified form # from the #2032 disambiguation pass (e.g. "process-order/index.ts"). - lbl = _strip_diacritics(str(G.nodes[nid].get("label") or "")).lower() + lbl = _strip_diacritics(str(attrs.get("label") or "")).lower() if lbl == query_basename or lbl.endswith("/" + query_basename): preferred.append(nid) if len(preferred) == 1: @@ -1087,7 +976,7 @@ def _relay() -> None: def _community_header(cid: int, community_name) -> str: # Header for get_community: "Community N — Name", matching get_node / query - # output which read the community_name attribute to_json writes onto nodes. + # output which reads community names from native generation state. # Skip the name when it is just the "Community N" placeholder (written for # unnamed communities) so the header never reads "Community 12 — Community 12"; # also falls back to the bare id when there is no name. Name is sanitised @@ -1105,7 +994,7 @@ def _build_server(graph_path: str): All graph query tools and resources are registered here over a single ``mcp.server.Server`` instance; the caller picks the transport (stdio or - Streamable HTTP) and runs it. Hot-reload of graph.json works the same way + Streamable HTTP) and runs it. Helix generation hot reload works the same way regardless of transport, since reloads happen inside the tool handlers. """ import threading @@ -1119,53 +1008,82 @@ def _build_server(graph_path: str): from graphify import paths as _paths - # Per-graph context cache: resolved graph.json path -> {key, G, communities}. - # The server's default graph is just the first entry; a tool call carrying a - # project_path adds its own. Routing every graph through one cache means the - # eager trigram index and the mtime+size hot-reload behave identically for - # the default graph and for any project graph. + # Per-store context: each reader retains one immutable native snapshot and + # its long-lived public predicate client until the generation changes. _default_graph_path = graph_path _ctx_lock = threading.Lock() + _tool_lock = threading.Lock() _ctx_cache: dict[str, dict] = {} def _load_ctx(path: str): - """Return (G, communities) for a graph.json path, reusing a cached - context until the file's (mtime, size) changes and then transparently - rebuilding it. Unlike ``_load_graph`` it never exits the process on a - missing/corrupt file — it raises, so a bad project_path surfaces as a - tool error instead of killing a server that is happily serving other - projects.""" - try: - s = Path(path).stat() - key = (s.st_mtime_ns, s.st_size) - except FileNotFoundError: - raise FileNotFoundError(f"graph.json not found: {path}") - ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: - return ent["G"], ent["communities"] + """Return the current native graph and communities for a Helix store.""" with _ctx_lock: ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: - return ent["G"], ent["communities"] # another thread built it - try: - new_G = _load_graph(path) - except SystemExit as e: # _load_graph exits on missing/corrupt file - raise RuntimeError(f"could not load graph.json at {path}") from e - # Warm the trigram index before exposing the graph so the first query - # against it is fast (same rationale as the original startup warm-up). - _get_trigram_index(new_G) - comm = _communities_from_graph(new_G) - _ctx_cache[path] = {"key": key, "G": new_G, "communities": comm} - return new_G, comm + if ent is None: + ent = {"reader": HelixGraphReader(validate_store_path(path))} + _ctx_cache[path] = ent + loaded = ent["reader"].get() + if ent.get("generation") == loaded.generation: + return ( + ent["G"], + ent["native_query"], + ent["communities"], + ent["community_labels"], + ent["learning_overlay"], + ent["confidence_counts"], + ) + new_G = loaded.graph + learning = loaded.state.get("learning", {}) + learning_overlay = ( + dict(learning.get("nodes", {})) if isinstance(learning, dict) else {} + ) + community_labels = { + int(record["id"]): str(record.get("name") or f"Community {record['id']}") + for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + } + comm = _communities_from_graph(loaded) + analysis = loaded.state.get("analysis", {}) + raw_counts = analysis.get("confidence_counts", {}) if isinstance(analysis, dict) else {} + confidence_counts = { + name: int(raw_counts.get(name, 0)) + for name in ("EXTRACTED", "INFERRED", "AMBIGUOUS") + } if isinstance(raw_counts, dict) else {} + if not confidence_counts or sum(confidence_counts.values()) != new_G.edge_count: + confidence_counts = { + "EXTRACTED": new_G.edge_count, + "INFERRED": 0, + "AMBIGUOUS": 0, + } + ent.update({ + "generation": loaded.generation, + "G": new_G, + "native_query": loaded.query, + "communities": comm, + "community_labels": community_labels, + "learning_overlay": learning_overlay, + "confidence_counts": confidence_counts, + }) + return ( + new_G, + loaded.query, + comm, + community_labels, + learning_overlay, + confidence_counts, + ) def _resolve_graph_path(project_path) -> str: - """Map an optional project_path to a concrete graph.json path. ``None`` + """Map an optional project_path to a concrete Helix store. ``None`` keeps the server's default graph (backward-compatible); a project_path - resolves to ``//graph.json``, honouring the + resolves to ``//graph.helix``, honouring the GRAPHIFY_OUT override so worktree/shared-output setups keep working.""" if not project_path: return _default_graph_path - return str(Path(project_path) / _paths.GRAPHIFY_OUT / "graph.json") + candidate = Path(project_path) + if candidate.name == "graph.helix": + return str(candidate) + return str(candidate / _paths.GRAPHIFY_OUT / "graph.helix") # Active per-request context, rebound by _select_graph() and read by the tool # handlers below. No lock needed on the hot path: _select_graph and the @@ -1174,17 +1092,35 @@ def _resolve_graph_path(project_path) -> str: # swap. active_graph_path = _default_graph_path try: - G, communities = _load_ctx(_default_graph_path) + ( + G, + native_query, + communities, + community_labels, + learning_overlay, + confidence_counts, + ) = _load_ctx(_default_graph_path) except (FileNotFoundError, RuntimeError): # No default graph at startup → run as a pure multi-project server. Tools # then require project_path; a call without one gets a clear error rather # than the process refusing to start (which is what _load_graph would do). - G, communities = None, {} + G, native_query, communities, community_labels, learning_overlay, confidence_counts = ( + None, None, {}, {}, {}, {} + ) def _select_graph(project_path) -> None: - nonlocal G, communities, active_graph_path + nonlocal G, native_query, communities, community_labels, learning_overlay + nonlocal confidence_counts + nonlocal active_graph_path path = _resolve_graph_path(project_path) - G, communities = _load_ctx(path) + ( + G, + native_query, + communities, + community_labels, + learning_overlay, + confidence_counts, + ) = _load_ctx(path) active_graph_path = path server = Server("graphify") @@ -1325,7 +1261,7 @@ async def list_tools() -> list[types.Tool]: "type": "string", "description": ( "Absolute path to a project directory containing " - "graphify-out/graph.json. Optional — defaults to the graph " + "graphify-out/graph.helix. Optional — defaults to the graph " "this server was started with." ), } @@ -1343,10 +1279,12 @@ def _tool_query_graph(arguments: dict) -> str: result = _query_graph_text( G, question, + native_query=native_query, mode=mode, depth=depth, token_budget=budget, context_filters=context_filter, + learning_overlay=learning_overlay, ) querylog.log_query( kind="mcp_query", @@ -1362,29 +1300,29 @@ def _tool_query_graph(arguments: dict) -> str: def _tool_get_node(arguments: dict) -> str: label = arguments["label"].lower() - matches = [(nid, d) for nid, d in G.nodes(data=True) - if label in (d.get("label") or "").lower() or label == nid.lower()] + matches = _find_node(G, label, native_query=native_query) if not matches: return f"No node matching '{label}' found." - nid, d = matches[0] + nid = matches[0] + d = node_attributes(G, nid) # Sanitise every LLM-derived field before concatenation (F-010). return "\n".join([ f"Node: {sanitize_label(d.get('label', nid))}", - f" ID: {sanitize_label(nid)}", + f" ID: {sanitize_label(str(nid))}", f" Source: {sanitize_label(str(d.get('source_file', '')))} {sanitize_label(str(d.get('source_location', '')))}", f" Type: {sanitize_label(str(d.get('file_type', '')))}", f" Community: {sanitize_label(str(d.get('community_name') or d.get('community', '')))}", - f" Degree: {G.degree(nid)}", + f" Degree: {G.degree(nid).degree}", ]) def _tool_get_neighbors(arguments: dict) -> str: label = arguments["label"].lower() rel_filter = arguments.get("relation_filter", "").lower() - matches = _find_node(G, label) + matches = _find_node(G, label, native_query=native_query) if not matches: return f"No node matching '{label}' found." nid = matches[0] - lines = [f"Neighbors of {sanitize_label(G.nodes[nid].get('label', nid))}:"] + lines = [f"Neighbors of {sanitize_label(node_attributes(G, nid).get('label', nid))}:"] def _edge_at(d: dict) -> str: # Edge location = the relation SITE (call/import line) in the source # node's file, not a def line (#BUG1). @@ -1396,10 +1334,10 @@ def _edge_at(d: dict) -> str: for nb in G.successors(nid): d = edge_data(G, nid, nb) rel = d.get("relation", "") - if rel_filter and rel_filter not in rel.lower(): + if rel_filter and rel_filter not in str(rel).lower(): continue lines.append( - f" --> {sanitize_label(G.nodes[nb].get('label', nb))} " + f" --> {sanitize_label(node_attributes(G, nb).get('label', nb))} " f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" ) for nb in G.predecessors(nid): @@ -1408,7 +1346,7 @@ def _edge_at(d: dict) -> str: if rel_filter and rel_filter not in rel.lower(): continue lines.append( - f" <-- {sanitize_label(G.nodes[nb].get('label', nb))} " + f" <-- {sanitize_label(node_attributes(G, nb).get('label', nb))} " f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]{_edge_at(d)}" ) budget = int(arguments.get("token_budget", 2000)) @@ -1421,10 +1359,10 @@ def _tool_get_community(arguments: dict) -> str: nodes = communities.get(cid, []) if not nodes: return f"Community {cid} not found." - header = _community_header(cid, G.nodes[nodes[0]].get("community_name")) + header = _community_header(cid, node_attributes(G, nodes[0]).get("community_name")) lines = [f"{header} ({len(nodes)} nodes):"] for n in nodes: - d = G.nodes[n] + d = node_attributes(G, n) # Sanitise label and source_file (F-010). lines.append( f" {sanitize_label(d.get('label', n))} " @@ -1443,20 +1381,27 @@ def _tool_god_nodes(arguments: dict) -> str: return "\n".join(lines) def _tool_graph_stats(_: dict) -> str: - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + total = G.edge_count or 1 return ( - f"Nodes: {G.number_of_nodes()}\n" - f"Edges: {G.number_of_edges()}\n" + f"Nodes: {G.node_count}\n" + f"Edges: {G.edge_count}\n" f"Communities: {len(communities)}\n" - f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" - f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" - f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" + f"EXTRACTED: {round(confidence_counts['EXTRACTED']/total*100)}%\n" + f"INFERRED: {round(confidence_counts['INFERRED']/total*100)}%\n" + f"AMBIGUOUS: {round(confidence_counts['AMBIGUOUS']/total*100)}%\n" ) def _tool_shortest_path(arguments: dict) -> str: - src_scored = _score_nodes(G, [t.lower() for t in arguments["source"].split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in arguments["target"].split()]) + src_scored = _score_nodes( + G, + [t.lower() for t in arguments["source"].split()], + native_query=native_query, + ) + tgt_scored = _score_nodes( + G, + [t.lower() for t in arguments["target"].split()], + native_query=native_query, + ) if not src_scored: return f"No node matching source '{arguments['source']}' found." if not tgt_scored: @@ -1487,16 +1432,13 @@ def _tool_shortest_path(arguments: dict) -> str: ) max_hops = int(arguments.get("max_hops", 8)) try: - # Deterministic path (#2074): the hash-seeded undirected view picked an - # arbitrary route among equal-length paths. Build a sorted, materialized - # undirected graph so the chosen path is canonical. Serve's shared G is - # left untouched (its degree feeds query-seed tie-breaks). - _und = nx.Graph() - _und.add_nodes_from(sorted(G.nodes)) - _und.add_edges_from(sorted((min(u, v), max(u, v)) for u, v in G.edges())) - path_nodes = nx.shortest_path(_und, src_nid, tgt_nid) - except (nx.NetworkXNoPath, nx.NodeNotFound): - return f"No path found between '{G.nodes[src_nid].get('label', src_nid)}' and '{G.nodes[tgt_nid].get('label', tgt_nid)}'." + # Use undirected view for path-finding (works regardless of query src/tgt order) + path_result = G.shortest_path(src_nid, tgt_nid, direction="both") + path_nodes = list(path_result.node_ids) + if not path_nodes: + raise ValueError("no path") + except (KeyError, ValueError, RuntimeError): + return f"No path found between '{node_attributes(G, src_nid).get('label', src_nid)}' and '{node_attributes(G, tgt_nid).get('label', tgt_nid)}'." hops = len(path_nodes) - 1 if hops > max_hops: return f"Path exceeds max_hops={max_hops} ({hops} hops found)." @@ -1505,7 +1447,7 @@ def _tool_shortest_path(arguments: dict) -> str: u, v = path_nodes[i], path_nodes[i + 1] # Report the actual stored relation(s), never a fabricated `calls`; # fall back to an honest "related" when the edge has no relation (#2074). - if G.has_edge(u, v): + if G.has_edge_between(u, v): datas = edge_datas(G, u, v) forward = True else: @@ -1516,11 +1458,11 @@ def _tool_shortest_path(arguments: dict) -> str: confs = sorted({d.get("confidence") for d in datas if d.get("confidence")}) conf_str = f" [{'/'.join(confs)}]" if confs else "" if i == 0: - segments.append(G.nodes[u].get("label", u)) + segments.append(node_attributes(G, u).get("label", u)) if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + segments.append(f"--{rel}{conf_str}--> {node_attributes(G, v).get('label', v)}") else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") + segments.append(f"<--{rel}{conf_str}-- {node_attributes(G, v).get('label', v)}") prefix = ("\n".join(warnings) + "\n") if warnings else "" return prefix + f"Shortest path ({hops} hops):\n " + " ".join(segments) @@ -1552,7 +1494,9 @@ def _tool_get_pr_impact(arguments: dict) -> str: files = fetch_pr_files(number, repo) if not files: return f"PR #{number}: no changed files found (may require gh auth)." - comms, nodes = compute_pr_impact(files, G) + comms, nodes = compute_pr_impact( + files, G, communities, native_query=native_query + ) ci = _parse_ci(pr_data.get("statusCheckRollup") or []) lines = [ f"PR #{number}: {pr_data['title']}", @@ -1594,7 +1538,9 @@ def _tool_triage_prs(arguments: dict) -> str: files = [] if files: pr.files_changed = files - pr.communities_touched, pr.nodes_affected = compute_pr_impact(files, G) + pr.communities_touched, pr.nodes_affected = compute_pr_impact( + files, G, communities, native_query=native_query + ) header = ( f"Actionable PRs targeting {base}: {len(actionable)}\n" "Rank these by review priority. Higher blast_radius = more graph communities affected = higher merge risk.\n" @@ -1623,13 +1569,9 @@ def _tool_triage_prs(arguments: dict) -> str: } def _load_community_labels() -> dict[int, str]: - labels_path = Path(active_graph_path).parent / ".graphify_labels.json" - if labels_path.exists(): - try: - return {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - except Exception: - pass - return {cid: f"Community {cid}" for cid in communities} + return dict(community_labels) or { + cid: f"Community {cid}" for cid in communities + } @server.list_resources() async def list_resources() -> list[types.Resource]: @@ -1644,66 +1586,92 @@ async def list_resources() -> list[types.Resource]: @server.read_resource() async def read_resource(uri: AnyUrl) -> str: - _select_graph(None) # resources read the server's default graph - uri_str = str(uri) - if uri_str == "graphify://report": - report_path = Path(active_graph_path).parent / "GRAPH_REPORT.md" - if report_path.exists(): - return report_path.read_text(encoding="utf-8") - return "GRAPH_REPORT.md not found. Run graphify extract first." - if uri_str == "graphify://stats": - return _tool_graph_stats({}) - if uri_str == "graphify://god-nodes": - return _tool_god_nodes({"top_n": 10}) - if uri_str == "graphify://surprises": - try: - from graphify.analyze import surprising_connections - surprises = surprising_connections(G, communities, top_n=10) - if not surprises: - return "No surprising connections found." - lines = ["Surprising cross-community connections:"] - for s in surprises: - lines.append(f" {s.get('source', '')} <-> {s.get('target', '')} [{s.get('relation', '')}]") - return "\n".join(lines) - except Exception as exc: - return f"Could not compute surprising connections: {exc}" - if uri_str == "graphify://audit": - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 - return ( - f"Total edges: {total}\n" - f"EXTRACTED: {confs.count('EXTRACTED')} ({round(confs.count('EXTRACTED')/total*100)}%)\n" - f"INFERRED: {confs.count('INFERRED')} ({round(confs.count('INFERRED')/total*100)}%)\n" - f"AMBIGUOUS: {confs.count('AMBIGUOUS')} ({round(confs.count('AMBIGUOUS')/total*100)}%)\n" - ) - if uri_str == "graphify://questions": - try: - from graphify.analyze import suggest_questions - community_labels = _load_community_labels() - questions = suggest_questions(G, communities, community_labels, top_n=10) - if not questions: - return "No suggested questions available." - lines = ["Suggested questions:"] - for q in questions: - if isinstance(q, dict): - lines.append(f" - {q.get('question', '')}") - else: - lines.append(f" - {q}") - return "\n".join(lines) - except Exception as exc: - return f"Could not generate questions: {exc}" - raise ValueError(f"Unknown resource: {uri_str}") + import asyncio + + def read() -> str: + with _tool_lock: + _select_graph(None) # resources read the server's default graph + uri_str = str(uri) + if uri_str == "graphify://report": + report_path = Path(active_graph_path).parent / "GRAPH_REPORT.md" + if report_path.exists(): + return report_path.read_text(encoding="utf-8") + return "GRAPH_REPORT.md not found. Run graphify extract first." + if uri_str == "graphify://stats": + return _tool_graph_stats({}) + if uri_str == "graphify://god-nodes": + return _tool_god_nodes({"top_n": 10}) + if uri_str == "graphify://surprises": + try: + from graphify.analyze import surprising_connections + + surprises = surprising_connections(G, communities, top_n=10) + if not surprises: + return "No surprising connections found." + lines = ["Surprising cross-community connections:"] + for item in surprises: + lines.append( + f" {item.get('source', '')} <-> " + f"{item.get('target', '')} [{item.get('relation', '')}]" + ) + return "\n".join(lines) + except Exception as exc: + return f"Could not compute surprising connections: {exc}" + if uri_str == "graphify://audit": + total = G.edge_count or 1 + return ( + f"Total edges: {total}\n" + f"EXTRACTED: {confidence_counts['EXTRACTED']} " + f"({round(confidence_counts['EXTRACTED'] / total * 100)}%)\n" + f"INFERRED: {confidence_counts['INFERRED']} " + f"({round(confidence_counts['INFERRED'] / total * 100)}%)\n" + f"AMBIGUOUS: {confidence_counts['AMBIGUOUS']} " + f"({round(confidence_counts['AMBIGUOUS'] / total * 100)}%)\n" + ) + if uri_str == "graphify://questions": + try: + from graphify.analyze import suggest_questions + + labels = _load_community_labels() + questions = suggest_questions( + G, communities, labels, top_n=10 + ) + if not questions: + return "No suggested questions available." + lines = ["Suggested questions:"] + for question in questions: + if isinstance(question, dict): + lines.append(f" - {question.get('question', '')}") + else: + lines.append(f" - {question}") + return "\n".join(lines) + except Exception as exc: + return f"Could not generate questions: {exc}" + raise ValueError(f"Unknown resource: {uri_str}") + + return await asyncio.to_thread(read) @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: + import asyncio + arguments = dict(arguments or {}) project_path = arguments.pop("project_path", None) handler = _handlers.get(name) if not handler: return [types.TextContent(type="text", text=f"Unknown tool: {name}")] + + def _execute() -> str: + # Helix's embedded client is synchronous. Run selection and the tool + # as one serialized worker-thread operation so HTTP event loops are + # never blocked and closure-scoped active context cannot cross calls. + with _tool_lock: + _select_graph(project_path) + return handler(arguments) + try: - _select_graph(project_path) # bind G/communities to the target graph - return [types.TextContent(type="text", text=handler(arguments))] + result = await asyncio.to_thread(_execute) + return [types.TextContent(type="text", text=result)] except Exception as exc: return [types.TextContent(type="text", text=f"Error executing {name}: {exc}")] @@ -1712,7 +1680,7 @@ async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: def serve(graph_path: str | None = None) -> None: """Start the MCP server over stdio (the default, per-developer transport).""" - graph_path = graph_path or _default_graph_json() + graph_path = graph_path or str(DEFAULT_PROJECT_STORE) try: from mcp.server.stdio import stdio_server except ImportError as e: @@ -1832,7 +1800,7 @@ def _build_http_app( # are intentionally exposing the server, so accept any Host header; for a # loopback/specific bind, restrict Host to that address (with and without # the port) plus the localhost aliases. - if host in ("0.0.0.0", "::", ""): + if host in ("0.0.0.0", "::", ""): # nosec B104 - explicit operator-selected bind security = TransportSecuritySettings(enable_dns_rebinding_protection=False) else: allowed = {host, "localhost", "127.0.0.1"} @@ -1890,7 +1858,7 @@ def serve_http( deliberate follow-up. Binding ``0.0.0.0`` exposes the server beyond localhost — set an api_key when you do. """ - graph_path = graph_path or _default_graph_json() + graph_path = graph_path or str(DEFAULT_PROJECT_STORE) try: import uvicorn except ImportError as e: @@ -1917,9 +1885,9 @@ def serve_http( f"graphify MCP server (streamable-http) on http://{host}:{port}{path} - {auth_note}", file=sys.stderr, ) - if host in ("0.0.0.0", "::", "") and not api_key: + if host in ("0.0.0.0", "::", "") and not api_key: # nosec B104 - emits warning print( - f"WARNING: binding {host or '0.0.0.0'} with no api-key exposes the graph " + f"WARNING: binding {host or '0.0.0.0'} with no api-key exposes the graph " # nosec B104 "unauthenticated on the network. Set --api-key (or GRAPHIFY_API_KEY).", file=sys.stderr, ) @@ -1938,14 +1906,14 @@ def _main(argv: list[str] | None = None) -> None: "graph_path", nargs="?", default=None, - help="Path to graph.json (default: graphify-out/graph.json)", + help="Path to graph.helix store (default: graphify-out/graph.helix)", ) parser.add_argument( "--graph", dest="graph_flag", default=None, metavar="PATH", - help="Path to graph.json — alias for the positional argument", + help="Path to graph.helix — alias for the positional argument", ) parser.add_argument( "--transport", @@ -1978,7 +1946,7 @@ def _main(argv: list[str] | None = None) -> None: help="Reap stateful sessions idle this many seconds (default: 3600; 0 disables)", ) args = parser.parse_args(argv) - graph_path = args.graph_flag or args.graph_path or _default_graph_json() + graph_path = args.graph_flag or args.graph_path or str(DEFAULT_PROJECT_STORE) if args.transport == "http": serve_http( diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index afb4ecc12..9f5c1b887 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,430 +163,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 4f03ccbae..c1984542e 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -5,110 +5,34 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run `graphify extract INPUT_PATH`. The production CLI performs corpus detection +and sensitive-file exclusion. Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -129,1146 +53,98 @@ Then act on it: ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. - -Print: `"Semantic extraction: N files (sequential — Aider)"` - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('.graphify_cached.json').unlink(missing_ok=True) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Sequential extraction (Aider)** - -Process each file one at a time. For each file: - -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files - -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. - -**Step B3 - Cache and merge** - -For the accumulated result: - -If more than half the chunks failed, stop and tell the user. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text()) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -extract = json.loads(Path('.graphify_extract.json').read_text()) -# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- +Run `graphify hook install`. ## For native CLAUDE.md integration -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` - ---- +Run `graphify claude install`. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index afb4ecc12..9f5c1b887 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,430 +163,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index d98865cc8..2b529034a 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,430 +166,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 0c821a278..65adee8bf 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (Codex)** @@ -270,430 +163,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index d98865cc8..2b529034a 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,430 +166,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index e3e6d2dec..2f2d31ae8 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -3,125 +3,40 @@ name: graphify description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." argument-hint: "[path|query|subcommand]" model: sonnet -allowed-tools: - - read - - grep - - glob - - exec -triggers: - - user - - model +allowed-tools: [read, grep, glob, exec] +triggers: [user, model] --- # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes -> one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) -export PYTHONUTF8=1 -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > graphify-out/.graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run `graphify extract INPUT_PATH`. The production CLI performs corpus detection +and sensitive-file exclusion. Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -142,1260 +57,106 @@ Then act on it: ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Dispatch subagents** - -Dispatch ALL subagents in the same response so they run in parallel. - -Concrete example for 3 chunks: -``` -[Subagent 1: files 1-15] -[Subagent 2: files 16-30] -[Subagent 3: files 31-45] -``` -All three in one message. Not three separate messages. - -For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): - -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} - -Extraction rules: -- EXTRACTED: relationship explicit in source (import, call, citation) -- INFERRED: reasonable inference (shared structure, implied dependency) -- AMBIGUOUS: uncertain — flag it, do not omit -- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. -- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. -- Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. -- DEEP_MODE (if set): be aggressive with INFERRED edges -- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. -- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. -- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. -- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Schema: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -Files: -FILE_LIST -``` - -**Step B3 - Cache and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -NODE_LIMIT = 5000 -if G.number_of_nodes() > NODE_LIMIT: - from collections import Counter - print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - import networkx as nx_meta - meta = nx_meta.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) - edge_counts = Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') - if meta.number_of_nodes() > 1: - meta_communities = {cid: [str(cid)] for cid in communities} - member_counts = {cid: len(members) for cid, members in communities.items()} - to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) - print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') - print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') - else: - print('Single community — aggregated view not useful. Skipping graph.html.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. - -Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.wiki import to_wiki -from graphify.analyze import god_nodes -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} -gods = god_nodes(G) - -n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) -print(f'Wiki: {n} articles written to graphify-out/wiki/') -print(' graphify-out/wiki/index.md -> agent entry point') -" -``` +Run `graphify export wiki graphify-out/graph.helix`. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) - wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" -fi -``` - ---- +Prefer `graphify`; otherwise use `python3 -m graphify`. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4-8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv -> abstract + metadata saved as `.md` -- PDF -> downloaded as `.pdf` -- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build -- Any webpage -> converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - ---- +Run `graphify hook install`. ## For always-on context in Devin sessions -Run once per project to make graphify always-on in Devin sessions: - -```bash -graphify devin install --project -``` - -This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. - -```bash -graphify devin uninstall --project # remove -``` - ---- +Tell Devin to query `graphify-out/graph.helix` before broad source search. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index c3815d556..cf088d837 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,430 +163,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index dbb4658ca..6035c49f3 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,423 +166,89 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- @@ -705,7 +264,6 @@ When the user asks to install the post-commit auto-rebuild hook or wire graphify ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index d98865cc8..2b529034a 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,430 +166,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index cf5dae440..6758f0985 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** @@ -265,430 +158,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index d98865cc8..2b529034a 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,430 +166,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index b0cbeb122..21aa35b71 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -271,430 +164,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 3e6bc6b7b..720f99c4d 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch subagents and paste their responses** @@ -269,430 +162,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 574384576..694757a68 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -126,19 +104,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -155,7 +136,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -165,109 +146,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -295,443 +188,107 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Troubleshooting -### PowerShell 5.1: Vertical scrolling stops working +### Windows support -If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: +Use CPython 3.10 or 3.12 on native Windows x86_64 and install Graphify normally with pip or uv. The exact public `helix-db-embedded` version must provide a `win_amd64` wheel; do not substitute WSL, a source build, a downloaded DLL, or a compatibility graph library. + +### PowerShell 5.1: Vertical scrolling stops working -1. **Upgrade graphify**: `pip install --upgrade graphifyy` -2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly -3. **Reset your terminal**: close and reopen PowerShell -4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output +Use Windows Terminal or PowerShell 7 when possible. Graphify does not patch terminal modes or load a helper DLL. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill.md b/graphify/skill.md index d98865cc8..2b529034a 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,19 +82,22 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. + ### Step 2 - Detect files +Run the production CLI and let it perform the supported-file and sensitive-file checks: + ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +graphify extract INPUT_PATH ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Use `--out +OUTPUT_PATH` when output must live outside the source tree. A successful build +activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. + +The production CLI performs corpus detection and sensitive-file exclusion. +Present its result as a clean summary: ``` Corpus: X files · ~Y words @@ -133,7 +114,7 @@ Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). + - Use the resolved INPUT_PATH as `scan_root`. - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. @@ -143,109 +124,21 @@ Then act on it: ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,430 +166,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +**Step B3 - Collect and validate results** -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: -# a detected file whose chunk failed or was omitted must stay unstamped so the -# next --update re-queues it, otherwise it is marked done and its content is lost -# forever (#2015). This mirrors the library extract path exactly -# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the -# raw corpus. Code files are always stamped (AST is deterministic); only semantic -# types are gated on output. -from graphify.cli import _stamped_manifest_files -_corpus = detect.get('all_files') or detect['files'] -_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) -# Files dispatched this run (the changed subset) but NOT stamped above still carry -# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues -# them instead of reading them as unchanged (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root -# files newly excluded since last run are dropped rather than masquerading as -# deletions; untouched files' prior rows are still preserved (#1908). -_scan = {f for fl in _corpus.values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) - -# Update cumulative cost tracker -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skills/agents/references/add-watch.md b/graphify/skills/agents/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/agents/references/add-watch.md +++ b/graphify/skills/agents/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/agents/references/exports.md b/graphify/skills/agents/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/agents/references/exports.md +++ b/graphify/skills/agents/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/agents/references/github-and-merge.md b/graphify/skills/agents/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/agents/references/github-and-merge.md +++ b/graphify/skills/agents/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/agents/references/hooks.md b/graphify/skills/agents/references/hooks.md index 3fb74d154..4f22e262c 100644 --- a/graphify/skills/agents/references/hooks.md +++ b/graphify/skills/agents/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/agents/references/query.md b/graphify/skills/agents/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/agents/references/query.md +++ b/graphify/skills/agents/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/amp/references/add-watch.md b/graphify/skills/amp/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/amp/references/add-watch.md +++ b/graphify/skills/amp/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/amp/references/exports.md b/graphify/skills/amp/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/amp/references/exports.md +++ b/graphify/skills/amp/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/amp/references/github-and-merge.md b/graphify/skills/amp/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/amp/references/github-and-merge.md +++ b/graphify/skills/amp/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/amp/references/hooks.md b/graphify/skills/amp/references/hooks.md index af1ac7e72..9e318be21 100644 --- a/graphify/skills/amp/references/hooks.md +++ b/graphify/skills/amp/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/claude/references/add-watch.md b/graphify/skills/claude/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/claude/references/add-watch.md +++ b/graphify/skills/claude/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/claude/references/exports.md b/graphify/skills/claude/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/claude/references/exports.md +++ b/graphify/skills/claude/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/claude/references/github-and-merge.md b/graphify/skills/claude/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/claude/references/github-and-merge.md +++ b/graphify/skills/claude/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/claude/references/hooks.md b/graphify/skills/claude/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/claude/references/hooks.md +++ b/graphify/skills/claude/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/claw/references/add-watch.md b/graphify/skills/claw/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/claw/references/add-watch.md +++ b/graphify/skills/claw/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/claw/references/exports.md b/graphify/skills/claw/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/claw/references/exports.md +++ b/graphify/skills/claw/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/claw/references/github-and-merge.md b/graphify/skills/claw/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/claw/references/github-and-merge.md +++ b/graphify/skills/claw/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/claw/references/hooks.md b/graphify/skills/claw/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/claw/references/hooks.md +++ b/graphify/skills/claw/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/codex/references/add-watch.md b/graphify/skills/codex/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/codex/references/add-watch.md +++ b/graphify/skills/codex/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/codex/references/exports.md b/graphify/skills/codex/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/codex/references/exports.md +++ b/graphify/skills/codex/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/codex/references/github-and-merge.md b/graphify/skills/codex/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/codex/references/github-and-merge.md +++ b/graphify/skills/codex/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/codex/references/hooks.md b/graphify/skills/codex/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/codex/references/hooks.md +++ b/graphify/skills/codex/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/copilot/references/add-watch.md b/graphify/skills/copilot/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/copilot/references/add-watch.md +++ b/graphify/skills/copilot/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/copilot/references/exports.md b/graphify/skills/copilot/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/copilot/references/exports.md +++ b/graphify/skills/copilot/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/copilot/references/github-and-merge.md b/graphify/skills/copilot/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/copilot/references/github-and-merge.md +++ b/graphify/skills/copilot/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/copilot/references/hooks.md b/graphify/skills/copilot/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/copilot/references/hooks.md +++ b/graphify/skills/copilot/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/droid/references/add-watch.md b/graphify/skills/droid/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/droid/references/add-watch.md +++ b/graphify/skills/droid/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/droid/references/exports.md b/graphify/skills/droid/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/droid/references/exports.md +++ b/graphify/skills/droid/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/droid/references/github-and-merge.md b/graphify/skills/droid/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/droid/references/github-and-merge.md +++ b/graphify/skills/droid/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/droid/references/hooks.md b/graphify/skills/droid/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/droid/references/hooks.md +++ b/graphify/skills/droid/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/kilo/references/add-watch.md b/graphify/skills/kilo/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/kilo/references/add-watch.md +++ b/graphify/skills/kilo/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/kilo/references/exports.md b/graphify/skills/kilo/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/kilo/references/exports.md +++ b/graphify/skills/kilo/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/kilo/references/github-and-merge.md b/graphify/skills/kilo/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/kilo/references/github-and-merge.md +++ b/graphify/skills/kilo/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/kilo/references/hooks.md b/graphify/skills/kilo/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/kilo/references/hooks.md +++ b/graphify/skills/kilo/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/kiro/references/add-watch.md b/graphify/skills/kiro/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/kiro/references/add-watch.md +++ b/graphify/skills/kiro/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/kiro/references/exports.md b/graphify/skills/kiro/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/kiro/references/exports.md +++ b/graphify/skills/kiro/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/kiro/references/github-and-merge.md b/graphify/skills/kiro/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/kiro/references/github-and-merge.md +++ b/graphify/skills/kiro/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/kiro/references/hooks.md b/graphify/skills/kiro/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/kiro/references/hooks.md +++ b/graphify/skills/kiro/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/opencode/references/add-watch.md b/graphify/skills/opencode/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/opencode/references/add-watch.md +++ b/graphify/skills/opencode/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/opencode/references/exports.md b/graphify/skills/opencode/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/opencode/references/exports.md +++ b/graphify/skills/opencode/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/opencode/references/github-and-merge.md b/graphify/skills/opencode/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/opencode/references/github-and-merge.md +++ b/graphify/skills/opencode/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/opencode/references/hooks.md b/graphify/skills/opencode/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/opencode/references/hooks.md +++ b/graphify/skills/opencode/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/pi/references/add-watch.md b/graphify/skills/pi/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/pi/references/add-watch.md +++ b/graphify/skills/pi/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/pi/references/exports.md b/graphify/skills/pi/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/pi/references/exports.md +++ b/graphify/skills/pi/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/pi/references/github-and-merge.md b/graphify/skills/pi/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/pi/references/github-and-merge.md +++ b/graphify/skills/pi/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/pi/references/hooks.md b/graphify/skills/pi/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/pi/references/hooks.md +++ b/graphify/skills/pi/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/trae/references/add-watch.md b/graphify/skills/trae/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/trae/references/add-watch.md +++ b/graphify/skills/trae/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/trae/references/exports.md b/graphify/skills/trae/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/trae/references/exports.md +++ b/graphify/skills/trae/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/trae/references/github-and-merge.md b/graphify/skills/trae/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/trae/references/github-and-merge.md +++ b/graphify/skills/trae/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/trae/references/hooks.md b/graphify/skills/trae/references/hooks.md index 7c04d5b0b..04fbab01a 100644 --- a/graphify/skills/trae/references/hooks.md +++ b/graphify/skills/trae/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/vscode/references/add-watch.md b/graphify/skills/vscode/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/vscode/references/add-watch.md +++ b/graphify/skills/vscode/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/vscode/references/exports.md b/graphify/skills/vscode/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/vscode/references/exports.md +++ b/graphify/skills/vscode/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/vscode/references/github-and-merge.md b/graphify/skills/vscode/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/vscode/references/github-and-merge.md +++ b/graphify/skills/vscode/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/vscode/references/hooks.md b/graphify/skills/vscode/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/vscode/references/hooks.md +++ b/graphify/skills/vscode/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/windows/references/add-watch.md b/graphify/skills/windows/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/windows/references/add-watch.md +++ b/graphify/skills/windows/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/windows/references/exports.md b/graphify/skills/windows/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/windows/references/exports.md +++ b/graphify/skills/windows/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/windows/references/github-and-merge.md b/graphify/skills/windows/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/windows/references/github-and-merge.md +++ b/graphify/skills/windows/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/windows/references/hooks.md b/graphify/skills/windows/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/windows/references/hooks.md +++ b/graphify/skills/windows/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd412..740e6b139 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -1,210 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -# -# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output -# THIS run (new_extraction is this run's fresh extraction, read above before the -# merge overwrote the file): a changed doc whose chunk failed must stay unstamped -# so the next --update re-queues it, otherwise it is marked done and its content -# is lost forever (#2015). Mirrors the library extract path -# (cli._stamped_manifest_files + clear_semantic + scan_corpus). -from graphify.cli import _stamped_manifest_files -_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) -# Changed semantic files dispatched this run but NOT stamped had their chunk fail -# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). -_sem_types = ('document', 'paper', 'image') -_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} -_stamped = {f for fl in _manifest_files.values() for f in fl} -_cleared = _dispatched - _stamped -# scan_corpus = the RAW full corpus so in-root files newly excluded since last run -# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). -_scan = {f for fl in incremental['files'].values() for f in fl} -save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/tree_html.py b/graphify/tree_html.py index 1dc658ff1..85ba312a0 100644 --- a/graphify/tree_html.py +++ b/graphify/tree_html.py @@ -572,11 +572,22 @@ def write_tree_html( # kept for CLI compatibility with the older signature; ignored now top_k_edges: int = 0, ) -> Path: - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - graph = json.loads(graph_path.read_text(encoding="utf-8")) - tree = build_tree(graph, root=root, max_children=max_children, - project_label=project_label) + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path + + loaded = load_graph(validate_store_path(graph_path)) + tree = build_tree( + { + "nodes": [ + {"id": node.id, **graphify_attributes(node.attributes)} + for node in loaded.graph.nodes() + ] + }, + root=root, + max_children=max_children, + project_label=project_label, + ) title = f"{tree['name']} — graphify tree viewer" header = f"{tree['name']} — Knowledge Graph" html = emit_html(tree, title=title, header=header) diff --git a/graphify/validate.py b/graphify/validate.py index bab3ddc7c..b20d9ea52 100644 --- a/graphify/validate.py +++ b/graphify/validate.py @@ -51,7 +51,7 @@ def validate_extraction(data: dict) -> list[str]: f"'{node['file_type']}' - must be one of {sorted(VALID_FILE_TYPES)}" ) - # Edges - accept "links" (NetworkX <= 3.1) as fallback for "edges" + # Edges - accept the standard node-link spelling as a fallback for "edges". edge_list = data.get("edges") if "edges" in data else data.get("links") if edge_list is None: errors.append("Missing required key 'edges'") diff --git a/graphify/watch.py b/graphify/watch.py index 1ef1ebd4d..0c8a3bea1 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -1,773 +1,259 @@ -# monitor a folder and auto-trigger --update when files change +"""Watch a project and atomically activate native Helix graph generations.""" + from __future__ import annotations + +import copy import contextlib +import hashlib import json import os -import posixpath import re +import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path -# Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from .helix.model import node_attributes +from .paths import GRAPHIFY_OUT as _GRAPHIFY_OUT + + _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 -def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: - """Append ``changed_paths`` to ``out_dir/.pending_changes`` (one per line). +from graphify.detect import ( # noqa: E402 + CODE_EXTENSIONS, + DOC_EXTENSIONS, + IMAGE_EXTENSIONS, + PAPER_EXTENSIONS, + _is_ignored, + _load_graphifyignore, +) - Used by a post-commit hook process that cannot acquire ``_rebuild_lock`` - so its change set is not silently dropped (#1059). The lock-holding - process drains this file before and after its rebuild and merges the - contents with its own change set. - Opened in append mode so concurrent writers do not clobber each other on - POSIX; each ``write()`` of a small payload is effectively atomic. A - trailing newline is always written so partial-line corruption stays - confined to the offending entry and is skipped on drain. - """ +_WATCHED_EXTENSIONS = CODE_EXTENSIONS | DOC_EXTENSIONS | PAPER_EXTENSIONS | IMAGE_EXTENSIONS +_CODE_EXTENSIONS = CODE_EXTENSIONS + + +_REMOTE_SOURCE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]+://?") + + +def _is_remote_source(source_file: str) -> bool: + """Return whether source_file is a URL/virtual scheme, not a local path.""" + return bool(_REMOTE_SOURCE_RE.match(source_file)) + + +def _topology_sources(build_data) -> list[str]: + """Return normalized source files that contributed native topology.""" + sources: set[str] = set() + + def add(attributes) -> None: + source = attributes.get("source_file") + if source: + sources.add(str(source).replace("\\", "/")) + + for node in build_data.nodes: + add(node.attributes) + for edge in build_data.edges: + add(edge.attributes) + hyperedges = build_data.attributes.get("hyperedges", []) + if isinstance(hyperedges, list): + for hyperedge in hyperedges: + if isinstance(hyperedge, dict): + add(hyperedge) + return sorted(sources) + + +def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: + """Append an incremental change set for the active lock holder to drain.""" if not changed_paths: return out_dir.mkdir(parents=True, exist_ok=True) - pending = out_dir / _PENDING_FILENAME - payload = "".join(f"{os.fspath(p)}\n" for p in changed_paths) - with open(pending, "a", encoding="utf-8") as fh: - fh.write(payload) + payload = "".join(f"{os.fspath(path)}\n" for path in changed_paths) + with (out_dir / _PENDING_FILENAME).open("a", encoding="utf-8") as handle: + handle.write(payload) def _drain_pending(out_dir: Path) -> list[Path]: - """Read + unlink ``out_dir/.pending_changes`` and return deduplicated paths. - - Returns an empty list if the file does not exist. Empty/whitespace lines - are silently skipped so a partial concurrent write that left only a - fragment cannot poison the merge. - """ + """Atomically consume and de-duplicate queued incremental paths.""" pending = out_dir / _PENDING_FILENAME - if not pending.exists(): - return [] try: raw = pending.read_text(encoding="utf-8") - except OSError: + except (FileNotFoundError, OSError): return [] - # Unlink BEFORE returning so a crash between read and process retains the - # data in the next caller's view via the lines we are about to return — - # i.e. losing the file after reading is fine, losing it before would be a - # bug. Use missing_ok to tolerate a racing drain on platforms where - # rename/unlink may interleave. with contextlib.suppress(FileNotFoundError): pending.unlink() seen: set[str] = set() - out: list[Path] = [] + paths: list[Path] = [] for line in raw.splitlines(): - s = line.strip() - if not s or s in seen: - continue - seen.add(s) - out.append(Path(s)) - return out + value = line.strip() + if value and value not in seen: + seen.add(value) + paths.append(Path(value)) + return paths -# Build options that must survive into later rebuilds. The initial `extract` -# scan honours `--exclude`, but `update`/`watch`/hook rebuilds re-run detect() -# and would silently re-include excluded paths unless the patterns are persisted -# (#1886). We store them beside the graph so any rebuild driver can re-apply them. -_BUILD_CONFIG_FILENAME = ".graphify_build.json" +def _merge_changed_paths(*sources: list[Path] | None) -> list[Path]: + """Merge path lists in first-seen order.""" + seen: set[str] = set() + merged: list[Path] = [] + for source in sources: + for path in source or []: + key = os.fspath(path) + if key not in seen: + seen.add(key) + merged.append(path) + return merged def _write_build_config( out_dir: Path, *, - excludes: "list[str] | None", + excludes: list[str] | None, gitignore: bool | None = None, ) -> None: - """Persist corpus-shaping options under ``out_dir``. - - Best effort and non clobbering: omitted options retain their existing values. - """ + """Persist corpus-shaping scan options used by watch and update.""" if not excludes and gitignore is None: return + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / ".graphify_build.json" try: - out_dir.mkdir(parents=True, exist_ok=True) - path = out_dir / _BUILD_CONFIG_FILENAME - try: - config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} - except (OSError, json.JSONDecodeError): - config = {} - if not isinstance(config, dict): - config = {} - if excludes: - config["excludes"] = list(excludes) - if gitignore is not None: - config["gitignore"] = gitignore - path.write_text(json.dumps(config), encoding="utf-8") - except OSError: - pass + payload = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + except (OSError, ValueError): + payload = {} + if not isinstance(payload, dict): + payload = {} + if excludes: + payload["excludes"] = list(excludes) + if gitignore is not None: + payload["gitignore"] = gitignore + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") def _read_build_excludes(out_dir: Path) -> list[str]: - """Return the persisted ``--exclude`` patterns for this graph, or [].""" try: - path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) - ex = cfg.get("excludes") if isinstance(cfg, dict) else None - if isinstance(ex, list): - return [str(x) for x in ex if isinstance(x, str) and x] - except (OSError, json.JSONDecodeError): - pass - return [] + data = json.loads((out_dir / ".graphify_build.json").read_text(encoding="utf-8")) + values = data.get("excludes", []) + return [str(item) for item in values] if isinstance(values, list) else [] + except (OSError, ValueError): + return [] def _read_build_gitignore(out_dir: Path) -> bool: - """Return whether rebuilds should honor VCS ignore files (default True).""" try: - path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) - if isinstance(cfg, dict) and isinstance(cfg.get("gitignore"), bool): - return cfg["gitignore"] - except (OSError, json.JSONDecodeError): - pass - return True - + data = json.loads((out_dir / ".graphify_build.json").read_text(encoding="utf-8")) + value = data.get("gitignore") + return value if isinstance(value, bool) else True + except (OSError, ValueError): + return True -def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: - """Concatenate path lists, preserving order and dropping duplicates. - Used to combine a hook process's own ``changed_paths`` with the drained - contents of ``.pending_changes`` so the lock-holding rebuild covers - every queued commit's worth of files (#1059). - """ - seen: set[str] = set() - out: list[Path] = [] - for src in sources: - if not src: - continue - for p in src: - key = os.fspath(p) - if key in seen: - continue - seen.add(key) - out.append(p) - return out +def _stabilize_rebuild_cwd(watch_path: Path) -> bool: + """Recover detached hooks whose inherited working directory was removed.""" + if watch_path.is_absolute(): + return True + repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() + if repo_root and Path(repo_root).is_dir(): + try: + os.chdir(repo_root) + return True + except OSError: + pass + try: + Path.cwd() + return True + except FileNotFoundError: + print( + "[graphify watch] Rebuild failed: current working directory no longer " + "exists and GRAPHIFY_REPO_ROOT is not set." + ) + return False -@contextlib.contextmanager +@contextmanager def _rebuild_lock(out_dir: Path, *, blocking: bool = False): - """Per-repo advisory lock around a rebuild. - - Yields True if acquired, False if another rebuild is already running and - ``blocking`` is False. Uses fcntl.flock so the lock is released - automatically if the process is killed (no stale-lock cleanup needed). - - While the lock is held, ``.rebuild.lock`` contains the owning PID followed - by a newline so external pollers (publish scripts, etc.) can read it. - On successful release the file is unlinked so downstream tooling that - waits for the lock to clear by polling for its absence unblocks promptly. - - Falls back to a no-op yield(True) on platforms without fcntl (Windows). - """ - try: - import fcntl - except ImportError: - yield True - return + """Serialize local rebuild preparation; Helix also enforces writer exclusion.""" + from graphify.helix.persistence import _StoreLock out_dir.mkdir(parents=True, exist_ok=True) lock_path = out_dir / ".rebuild.lock" - # "a+" creates the file if missing without truncating an existing holder's - # PID payload — important because another process may have already written - # its PID before we attempt the flock. - fh = open(lock_path, "a+", encoding="utf-8") + lock = _StoreLock( + lock_path, + shared=False, + timeout=120.0 if blocking else 0.0, + ) acquired = False try: - flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) try: - fcntl.flock(fh.fileno(), flags) - except BlockingIOError: + lock.acquire() + except TimeoutError: yield False - return - acquired = True - # Replace any prior owner's PID with ours so external readers see a - # single parseable line, not a digit-concatenation across rebuilds. - try: - fh.seek(0) - fh.truncate() - fh.write(f"{os.getpid()}\n") - fh.flush() - except OSError: - pass - yield True + else: + acquired = True + yield True finally: - if acquired: - try: - fcntl.flock(fh.fileno(), fcntl.LOCK_UN) - except OSError: - pass - fh.close() - # Signal "rebuild done" by removing the lock file. Only the holder - # unlinks; a non-acquiring caller leaves the existing lock in place. + lock.release() if acquired: with contextlib.suppress(OSError): lock_path.unlink() -def _apply_resource_limits() -> None: - """Best-effort nice + memory cap. Called from inline hook scripts. - - GRAPHIFY_REBUILD_MEMORY_LIMIT_MB caps RSS-ish memory. Uses RLIMIT_DATA on - macOS (RLIMIT_AS is unreliable under Apple's libmalloc) and RLIMIT_AS on - Linux. Silently skips if the platform doesn't support it. - """ - try: - os.nice(10) - except (OSError, AttributeError): - pass - mb = os.environ.get("GRAPHIFY_REBUILD_MEMORY_LIMIT_MB", "").strip() - if not mb: - return - try: - limit = int(mb) * 1024 * 1024 - except ValueError: - return - try: - import resource - which = resource.RLIMIT_DATA if sys.platform == "darwin" else resource.RLIMIT_AS - soft, hard = resource.getrlimit(which) - new_hard = hard if hard != resource.RLIM_INFINITY and hard < limit else limit - resource.setrlimit(which, (limit, new_hard)) - except (ImportError, ValueError, OSError): - pass - - -def _git_head() -> str | None: - """Return current git HEAD commit hash, or None outside a repo.""" - import subprocess as _sp - try: - r = _sp.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=3) - return r.stdout.strip() if r.returncode == 0 else None - except Exception: - return None - - -from graphify.detect import ( - CODE_EXTENSIONS, - DOC_EXTENSIONS, - PAPER_EXTENSIONS, - IMAGE_EXTENSIONS, - _load_graphifyignore, - _is_ignored, -) - -_WATCHED_EXTENSIONS = CODE_EXTENSIONS | DOC_EXTENSIONS | PAPER_EXTENSIONS | IMAGE_EXTENSIONS -_CODE_EXTENSIONS = CODE_EXTENSIONS - - -def _report_root_label(watch_path: Path) -> str: - if watch_path.is_absolute(): - return watch_path.name or str(watch_path) - return Path.cwd().name if watch_path == Path(".") else str(watch_path) - - -def _is_relative_to(path: Path, root: Path) -> bool: - try: - path.relative_to(root) - return True - except ValueError: - return False - - -def _changed_path_candidates(raw: Path, *, change_root: Path, watch_root: Path) -> list[Path]: - """Return plausible absolute locations for a hook-provided changed path. - - Git hooks pass paths relative to the repository root. Watch callers may - also pass paths relative to the watched root. Keep both interpretations so - a graph rooted at ``src`` accepts ``src/app.py`` and ``app.py``. - """ - if raw.is_absolute(): - lexical = Path(os.path.abspath(raw)) - resolved = raw.resolve() - return [lexical] if lexical == resolved else [lexical, resolved] - - candidates: list[Path] = [] - seen: set[str] = set() - for base in (change_root, watch_root): - lexical = Path(os.path.abspath(base / raw)) - for cand in (lexical, lexical.resolve()): - key = os.fspath(cand) - if key in seen: - continue - seen.add(key) - candidates.append(cand) - return candidates - - -def _relativize_source_files(payload: dict, root: Path, *, scope: Path | None = None) -> None: - for bucket in ("nodes", "edges", "hyperedges"): - for item in payload.get(bucket, []): - source = item.get("source_file") - if not source: - continue - source_path = Path(source) - if not source_path.is_absolute(): - continue - try: - resolved = source_path.resolve() - if scope is not None and not _is_relative_to(resolved, scope): - continue - item["source_file"] = resolved.relative_to(root).as_posix() - except ValueError: - continue - - -def _rebase_relative_source_files(payload: dict, source_root: Path, target_root: Path) -> None: - """Rebase cache-root-relative source paths onto the project root.""" - if source_root == target_root: - return - for bucket in ("nodes", "edges", "hyperedges"): - for item in payload.get(bucket, []): - source = item.get("source_file") - if not source or Path(source).is_absolute(): - continue - try: - item["source_file"] = (source_root / source).relative_to(target_root).as_posix() - except ValueError: - continue - - -class _StoredSourcePaths: - """Resolve source_file values across current and legacy graph roots.""" - - def __init__( - self, - existing: dict, - *, - out: Path, - project_root: Path, - watch_root: Path, - normalize_source, - ) -> None: - self.project_root = project_root - self.watch_root = watch_root - self._normalize_source = normalize_source - self.existing_source_root = project_root - relative_marker_prefix: str | None = None - - root_marker = out / ".graphify_root" - if root_marker.exists(): - try: - saved_root = Path(root_marker.read_text(encoding="utf-8").strip()) - if saved_root.is_absolute(): - self.existing_source_root = saved_root.resolve() - else: - invocation_root = Path.cwd().resolve() - if (invocation_root / saved_root).resolve() == watch_root: - self.existing_source_root = invocation_root - relative_marker_prefix = posixpath.normpath(saved_root.as_posix()) - except (OSError, ValueError): - pass - - self.legacy_watch_relative = False - if relative_marker_prefix not in (None, "."): - has_project_relative_source = False - for bucket in ("nodes", "links", "edges", "hyperedges"): - for item in existing.get(bucket, []): - stored = normalize_source(item.get("source_file")) - if not stored or Path(stored).is_absolute(): - continue - normalized = posixpath.normpath(stored) - if ( - normalized == relative_marker_prefix - or normalized.startswith(relative_marker_prefix + "/") - ): - has_project_relative_source = True - break - if has_project_relative_source: - break - self.legacy_watch_relative = not has_project_relative_source - - def normalize(self, source_file: str | None) -> str | None: - normalized = self._normalize_source(source_file, str(self.project_root)) - return posixpath.normpath(normalized) if normalized else normalized - - def absolute_identity(self, source_file: str | None, root: Path) -> str | None: - normalized = self._normalize_source(source_file) - if not normalized: - return normalized - source_path = Path(posixpath.normpath(normalized)) - if not source_path.is_absolute(): - source_path = root / source_path - return Path(os.path.abspath(source_path)).as_posix() - - def identity(self, source_file: str | None) -> str | None: - normalized = self._normalize_source(source_file) - if normalized and not Path(normalized).is_absolute() and self.legacy_watch_relative: - return self.absolute_identity(normalized, self.watch_root) - return self.absolute_identity(normalized, self.existing_source_root) - - def in_watch_root(self, source_file: str | None) -> bool: - identity = self.identity(source_file) - return bool(identity) and _is_relative_to(Path(identity), self.watch_root) - - def is_evicted(self, item: dict, identities: set[str]) -> bool: - return self.identity(item.get("source_file")) in identities - - def rebase_preserved(self, item: dict) -> None: - identity = self.identity(item.get("source_file")) - if not identity: - return - identity_path = Path(identity) - if not _is_relative_to(identity_path, self.watch_root): - normalized = self.normalize(item.get("source_file")) - if normalized: - item["source_file"] = normalized - return - try: - item["source_file"] = identity_path.relative_to(self.project_root).as_posix() - except ValueError: - item["source_file"] = identity - - -# A source_file that is a URL/virtual scheme (gdoc://, s3://, http://, ...) rather -# than a filesystem path: its on-disk existence is meaningless, so it must never be -# evicted by the disk-absence sweep. Matched with a regex, NOT a literal "://", -# because path normalization on the write side (Path.as_posix) collapses the double -# slash to one — a stored "gdoc://x" reads back as "gdoc:/x" on the next update, and -# a literal "://" check would then miss it and wrongly evict the node (#2051 follow-up). -# The scheme is required to be 2+ chars so a Windows drive letter (C:/...) is not -# misread as a remote source. -_REMOTE_SOURCE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]+://?") - - -def _is_remote_source(source_file: str) -> bool: - return bool(_REMOTE_SOURCE_RE.match(source_file)) - - -def _reconcile_existing_graph( - existing_graph: Path, - result: dict, - *, - out: Path, - project_root: Path, - watch_root: Path, - code_files: list[Path], - extract_targets: list[Path], - full_rebuild: bool, - deleted_paths: set[str], - deleted_source_identities: set[str], -) -> tuple[dict, dict]: - """Merge fresh extraction with preserved graph entries and evict stale sources.""" - existing_graph_data: dict = {} - if not existing_graph.exists(): - return result, existing_graph_data - - try: - from graphify.build import _norm_source_file as _nsf - from graphify.extract import _get_extractor - from graphify.security import check_graph_file_size_cap - - check_graph_file_size_cap(existing_graph) - existing = json.loads(existing_graph.read_text(encoding="utf-8")) - existing_graph_data = existing - source_paths = _StoredSourcePaths( - existing, - out=out, - project_root=project_root, - watch_root=watch_root, - normalize_source=_nsf, +def _git_head(root: Path) -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, + check=False, + ) + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def _community_labels(graph, communities: dict[int, list]) -> dict[int, str]: + labels: dict[int, str] = {} + for community_id, members in communities.items(): + ranked = sorted( + members, + key=lambda node: ( + -graph.degree(node).degree, + str(node_attributes(graph, node).get("label", node)), + ), ) - new_ast_ids = {n["id"] for n in result["nodes"]} - current_sources = { - source_paths.absolute_identity(str(path), project_root) for path in code_files - } - rebuilt_source_identities = { - source_paths.absolute_identity(str(path), project_root) for path in extract_targets - } - node_evicted_source_identities = set(deleted_source_identities) - hyperedge_evicted_source_identities = set(deleted_source_identities) - # Deletion evicts edges regardless of tier; re-extraction only owns a - # source's AST-tier edges (checked per-edge below, #1865). - edge_evicted_source_identities = set(deleted_source_identities) - if not full_rebuild: - node_evicted_source_identities.update(rebuilt_source_identities) - - # Reconcile every rebuild against the current watched corpus. Hook change - # lists can contain only a rename destination, so explicit paths alone - # cannot identify the stale source. Keep the comparison scoped to the - # watched root so subfolder updates preserve records outside that subtree. - # - # Fail-closed eviction: a source identity missing from the corpus is only - # DELETION evidence when the file is actually gone from disk. A file that - # still exists but stopped being collected was *excluded* (ignore rules or - # filters changed — e.g. a .gitignore the scanner newly honors), and - # treating that as deletion silently mass-evicts good nodes. Preserve - # instead and say so; a full re-extraction still purges deliberately - # excluded sources via the AST ownership rule below. - excluded_alive_files: set[str] = set() - excluded_alive_nodes = 0 - _alive_cache: dict[str, bool] = {} - for node in existing.get("nodes", []): - source_file = node.get("source_file") - if not source_file or _is_remote_source(source_file): - continue # sourceless stub or remote/virtual source: never evict - identity = source_paths.identity(source_file) - if not source_paths.in_watch_root(source_file): - continue - if _get_extractor(Path(source_file)) is None: - # Non-AST source (semantic doc/paper/image — .txt/.pdf/.png/...): - # never present in current_sources (built from AST-extractable - # code_files), so corpus absence is meaningless. Disk absence is - # the ONLY deletion evidence here — otherwise its semantic nodes - # are preserved forever and returned as authoritative even after - # the file is deleted (#2051). A present-but-unextractable file - # stays preserved (alive -> skip). - if identity: - alive = _alive_cache.get(identity) - if alive is None: - alive = Path(identity).exists() - _alive_cache[identity] = alive - if not alive: - normalized = source_paths.normalize(source_file) - if normalized: - deleted_paths.add(normalized) - node_evicted_source_identities.add(identity) - edge_evicted_source_identities.add(identity) - hyperedge_evicted_source_identities.add(identity) - continue - if identity not in current_sources: - if identity: - alive = _alive_cache.get(identity) - if alive is None: - alive = Path(identity).exists() - _alive_cache[identity] = alive - if alive: - excluded_alive_files.add(identity) - excluded_alive_nodes += 1 - continue - normalized = source_paths.normalize(source_file) - if normalized: - deleted_paths.add(normalized) - if identity: - node_evicted_source_identities.add(identity) - edge_evicted_source_identities.add(identity) - hyperedge_evicted_source_identities.add(identity) - if excluded_alive_files: - print( - f"[graphify watch] fail-closed: kept {excluded_alive_nodes} node(s) " - f"from {len(excluded_alive_files)} file(s) that left the scan corpus " - "but still exist on disk (ignore rules or filters changed?). " - "Run a full re-extraction to purge them if the exclusion is intentional." - ) - - # A full re-extraction owns every AST node under watch_root. Incremental - # extraction owns only nodes from rebuilt or deleted sources. Semantic - # nodes lack the AST origin marker and remain preserved. - preserved_nodes = [ - node - for node in existing.get("nodes", []) - if node["id"] not in new_ast_ids - and not ( - node.get("_origin") == "ast" - and ( - ( - not node.get("source_file") - and (full_rebuild or not code_files) - ) - or ( - full_rebuild - and source_paths.in_watch_root(node.get("source_file")) - ) - ) - ) - and not source_paths.is_evicted(node, node_evicted_source_identities) - ] - all_ids = new_ast_ids | {node["id"] for node in preserved_nodes} - - # Edges are owned by source_file, but ownership is tier-scoped: the AST - # pass replaces a re-extracted source's AST edges, while that source's - # semantic/LLM edges — which the AST pass cannot regenerate — survive - # until a semantic re-extraction supersedes them. Same provenance rule - # the node reconciliation above applies via _origin (#1865). Deletion - # eviction stays provenance-blind. - preserved_edges = [ - edge - for edge in existing.get("links", existing.get("edges", [])) - if edge.get("source") in all_ids - and edge.get("target") in all_ids - and not source_paths.is_evicted(edge, edge_evicted_source_identities) - and not ( - edge.get("_origin") == "ast" - and source_paths.is_evicted(edge, rebuilt_source_identities) - ) + names = [ + str(node_attributes(graph, node).get("label", node)) + for node in ranked[:2] ] + labels[community_id] = " & ".join(names) if names else f"Community {community_id}" + return labels - new_hyperedge_ids = { - edge.get("id") for edge in result.get("hyperedges", []) if edge.get("id") - } - preserved_hyperedges = [] - for edge in existing.get("hyperedges", []): - members = edge.get("nodes", edge.get("members", edge.get("node_ids", []))) - if edge.get("id") in new_hyperedge_ids or source_paths.is_evicted( - edge, hyperedge_evicted_source_identities - ): - continue - if isinstance(members, list) and any(member not in all_ids for member in members): - continue - preserved_hyperedges.append(edge) - - for item in preserved_nodes + preserved_edges + preserved_hyperedges: - source_paths.rebase_preserved(item) - - return { - "nodes": result["nodes"] + preserved_nodes, - "edges": result["edges"] + preserved_edges, - "hyperedges": result.get("hyperedges", []) + preserved_hyperedges, - "input_tokens": 0, - "output_tokens": 0, - }, existing_graph_data - except Exception: - return result, existing_graph_data - - -def _node_community_map(graph_data: dict) -> dict[str, int]: - out: dict[str, int] = {} - for node in graph_data.get("nodes", []): - node_id = node.get("id") - cid = node.get("community") - if node_id is None or cid is None: - continue - try: - out[str(node_id)] = int(cid) - except (TypeError, ValueError): - print( - f"[graphify watch] Skipping node with invalid community id: " - f"node_id={node_id!r} community={cid!r}", - file=sys.stderr, - ) - continue - return out - - -def _canonical_graph_for_compare(graph_data: dict) -> dict: - canonical = dict(graph_data) - canonical.pop("built_at_commit", None) - for key in ("nodes", "links", "edges", "hyperedges"): - if key in canonical and isinstance(canonical[key], list): - canonical[key] = sorted( - canonical[key], - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - return canonical +def _content_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() -def _canonical_topology_for_compare(graph_data: dict) -> dict: - canonical = dict(graph_data) - canonical.pop("built_at_commit", None) - nodes = canonical.get("nodes") - if isinstance(nodes, list): - norm_nodes = [] - for node in nodes: - if not isinstance(node, dict): - continue - n = dict(node) - n.pop("community", None) - n.pop("community_name", None) - n.pop("norm_label", None) - norm_nodes.append(n) - canonical["nodes"] = sorted( - norm_nodes, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - - for key in ("links", "edges"): - items = canonical.get(key) - if not isinstance(items, list): - continue - norm_edges = [] - for edge in items: - if not isinstance(edge, dict): - continue - e = dict(edge) - # to_json writes _src/_tgt as the canonical directed endpoints and - # overwrites source/target with them before serialising, so the - # on-disk graph has no _src/_tgt. The candidate topology (fresh from - # node_link_data) still has them. Popping and reassigning here makes - # both sides comparable: existing gets no-op pops (None), candidate - # gets source/target overwritten from _src/_tgt — same result. - true_src = e.pop("_src", None) - true_tgt = e.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - e["source"] = true_src - e["target"] = true_tgt - e.pop("confidence_score", None) - norm_edges.append(e) - canonical[key] = sorted( - norm_edges, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - - hyperedges = canonical.get("hyperedges") - if isinstance(hyperedges, list): - canonical["hyperedges"] = sorted( - hyperedges, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), +def _warn_obsolete_store(out: Path) -> None: + legacy = out / "graph.json" + if legacy.exists(): + print( + "[graphify] warning: graph.json is obsolete and ignored; rebuild from source to create graph.helix.", + file=sys.stderr, ) - return canonical - - -def _topology_from_graph(G) -> dict: - from networkx.readwrite import json_graph - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - return data - def _check_shrink( force: bool, existing_data: dict, new_data: dict, - tmp: "Path | None" = None, + tmp: Path | None = None, *, had_explicit_deletions: bool = False, - rebuilt_sources: "set[str] | None" = None, + rebuilt_sources: set[str] | None = None, ) -> bool: - """Return True (ok to proceed) or False (shrink refused). - - When False, cleans up *tmp* if provided and prints a warning to stderr. - - The shrink-guard exists to catch SILENT shrinkage from failed extraction - chunks (a half-written semantic pass leaving thousands of nodes - unaccounted for). When ``had_explicit_deletions`` is True, the caller - has declared which files were removed (e.g. the post-commit hook saw - a ``D`` in ``git diff --name-only``) and a smaller graph is the expected - outcome — skip the guard so legitimate refactors don't require ``--force``. - - ``rebuilt_sources`` (when given) is the set of source files re-extracted this - run. A net shrink is legitimate — not a failed chunk — when every *lost* node - belonged to one of those files (a symbol removed from a re-extracted file) or - carries no source_file. Only an unexplained loss (a node from a file we did - NOT touch — e.g. a dropped semantic/doc node) refuses the write. This lets a - plain ``graphify update`` after deleting a function refresh the graph without - ``--force`` (#1116 left stale nodes write-blocked even though build dropped them). - """ - if force or not existing_data: - return True - if had_explicit_deletions and rebuilt_sources is None: - # Legacy callers declare deletions but pass no rebuilt_sources, so the - # per-source accounting below can't run — keep the wholesale bypass for - # them. When rebuilt_sources IS given, deleted paths are folded into it - # (see call site), so genuine deletions still pass the _accounted check - # while an unexplained loss (a present-but-unextractable file wrongly - # routed to _add_deleted_source, or a dropped semantic node) is still - # caught rather than being waved through by the mere presence of any - # deletion in the change set (#2056). + """Refuse unexplained node loss before activating a staged generation.""" + if force or not existing_data or had_explicit_deletions: return True existing_nodes = existing_data.get("nodes", []) new_nodes = new_data.get("nodes", []) @@ -775,655 +261,910 @@ def _check_shrink( return True if rebuilt_sources is not None: from graphify.build import _norm_source_file - new_ids = {n.get("id") for n in new_nodes} - lost = [n for n in existing_nodes if n.get("id") not in new_ids] - - def _accounted(n: dict) -> bool: - sf = n.get("source_file") - return (not sf - or sf in rebuilt_sources - or _norm_source_file(sf) in rebuilt_sources) - if all(_accounted(n) for n in lost): + + new_ids = {node.get("id") for node in new_nodes} + lost = [node for node in existing_nodes if node.get("id") not in new_ids] + + def accounted(node: dict) -> bool: + source = node.get("source_file") + return bool( + not source + or source in rebuilt_sources + or _norm_source_file(source) in rebuilt_sources + ) + + if all(accounted(node) for node in lost): return True if tmp is not None: - tmp.unlink(missing_ok=True) + if tmp.is_dir(): + import shutil + + shutil.rmtree(tmp) + else: + tmp.unlink(missing_ok=True) print( - f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but existing " - f"graph.json has {len(existing_nodes)}. Refusing to overwrite — you may be " - f"missing chunk files from a previous session. " - f"Pass --force to override.", + f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but the active " + f"graph.helix generation has {len(existing_nodes)}. Refusing to overwrite " + "because untouched source nodes disappeared; pass --force to override.", file=sys.stderr, ) return False -def _report_for_compare(report_text: str) -> str: - return re.sub(r"^- Built from commit: `[^`]+`\n?", "", report_text, flags=re.MULTILINE) - - -def _json_text(data: dict) -> str: - return json.dumps(data, indent=2, ensure_ascii=False) + "\n" - - -def _stabilize_rebuild_cwd(watch_path: Path) -> bool: - """Ensure relative rebuild paths have a usable CWD before queue/lock setup. - - Detached git hooks can inherit a transient working directory that is deleted - before the background rebuild starts. In that state Path.cwd(), - Path('.').resolve(), and relative graphify-out mkdirs raise FileNotFoundError - before the normal rebuild error handling can run. Hooks that know the repo - root export GRAPHIFY_REPO_ROOT so the rebuild can recover by chdir'ing there. - """ - if watch_path.is_absolute(): - return True - - repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() - if repo_root and Path(repo_root).is_dir(): - try: - os.chdir(repo_root) - return True - except OSError: - pass - - try: - Path.cwd() - return True - except FileNotFoundError: - print( - "[graphify watch] Rebuild failed: current working directory " - "no longer exists and GRAPHIFY_REPO_ROOT is not set." - ) - return False - - def _rebuild_code( watch_path: Path, *, + output_root: Path | None = None, changed_paths: list[Path] | None = None, follow_symlinks: bool = False, force: bool = False, no_cluster: bool = False, acquire_lock: bool = True, block_on_lock: bool = False, + include_semantic: bool = False, + code_only: bool = False, + backend: str | None = None, + model: str | None = None, + deep_mode: bool = False, + token_budget: int = 60_000, + max_concurrency: int = 4, + retain_rollback: bool = False, + external_extraction: dict | None = None, + external_kind: str | None = None, + prune_excluded: bool = False, + raise_on_error: bool = False, + _root_marker: str | None = None, + _timer=None, ) -> bool: - """Re-run AST extraction + build + optional cluster + report for code files. No LLM needed. - - When ``force`` is True the node-count safety check in ``to_json`` is bypassed - so the rebuilt graph overwrites graph.json even if it has fewer nodes. - Use this after refactors that legitimately delete code. + """Extract source and atomically activate topology, analysis, and state. - When ``changed_paths`` is provided, only those files are re-extracted; nodes - for unchanged files are preserved from the existing graph. Deleted paths - in ``changed_paths`` (paths that no longer exist on disk) are dropped from - the preserved set. When ``changed_paths`` is None the full code corpus is - re-extracted (used by the watcher and post-checkout hook). - - ``acquire_lock`` (default True) takes a non-blocking per-repo flock around - the rebuild so concurrent post-commit hooks across multiple repos do not - pile up. Returns False with a log line if the lock is held. Pass - ``block_on_lock=True`` to wait instead of skip (used by the interactive - ``graphify update`` CLI). - - ``no_cluster`` skips community detection and writes raw merged extraction - JSON to graphify-out/graph.json (mirrors ``extract --no-cluster``). - - Returns True on success, False on error or skipped-due-to-lock. + ``update`` and git hooks leave ``include_semantic`` false, so they remain + local AST-only refreshes. The headless ``extract`` command enables it and + feeds documents, papers, and images through the configured LLM backend. """ + watch_path = Path(watch_path) + if _root_marker is None: + _root_marker = os.fspath(watch_path) if not _stabilize_rebuild_cwd(watch_path): return False - - out = watch_path / _GRAPHIFY_OUT + watch_path = watch_path.resolve() + source_root = ( + watch_path + if Path(_root_marker).is_absolute() + else Path.cwd().resolve() + ) + output_root = Path(output_root).resolve() if output_root is not None else watch_path + out = output_root / _GRAPHIFY_OUT if acquire_lock: - # #1059: incremental (changed_paths is not None) hooks must not drop - # their change set when another rebuild is already running. Queue - # before attempting the lock so a non-blocking failure still records - # the work; the lock-holder drains the queue and merges it in. Full- - # corpus rebuilds skip the queue entirely — they already cover every - # file, so there is nothing to merge. - if changed_paths is not None and not block_on_lock: - _queue_pending(out, list(changed_paths)) - with _rebuild_lock(out, blocking=block_on_lock) as got: - if not got: - print("[graphify watch] Rebuild already in progress for " - f"{watch_path.resolve()} - changes queued.") + with _rebuild_lock(out, blocking=block_on_lock) as acquired: + if not acquired: + if changed_paths is not None: + _queue_pending(out, changed_paths) + print( + f"[graphify watch] Rebuild already in progress for {watch_path}; " + f"queued {len(changed_paths)} changed path(s)." + ) + else: + print(f"[graphify watch] Rebuild already in progress for {watch_path}.") return False - # Lock acquired. Drain anything queued by earlier contenders - # (including, importantly, the paths we just queued ourselves) - # and merge with our own change set so a single rebuild covers - # everything outstanding. - if changed_paths is not None: - merged = _merge_changed_paths(changed_paths, _drain_pending(out)) - else: - # Full-corpus rebuild supersedes any queued incremental work. - _drain_pending(out) - merged = None - ok = _rebuild_code( - watch_path, - changed_paths=merged, - follow_symlinks=follow_symlinks, - force=force, - no_cluster=no_cluster, - acquire_lock=False, + + queued = _drain_pending(out) + first_paths = ( + None if changed_paths is None else _merge_changed_paths(changed_paths, queued) ) - # Late-arrival drain: another hook may have queued work while we - # were rebuilding. Loop up to _PENDING_DRAIN_MAX_PASSES times so a - # storm of commits eventually quiesces without livelocking. A full - # rebuild already saw everything, so skip this for changed_paths is None. - if merged is not None: - for _ in range(_PENDING_DRAIN_MAX_PASSES): - late = _drain_pending(out) - if not late: - break - ok = _rebuild_code( - watch_path, - changed_paths=late, - follow_symlinks=follow_symlinks, - force=force, - no_cluster=no_cluster, - acquire_lock=False, - ) and ok - return ok - - watch_root = watch_path.resolve() - project_root = Path.cwd().resolve() if not watch_path.is_absolute() else watch_root - report_root = _report_root_label(watch_path) + + def run_inner(paths: list[Path] | None) -> bool: + return _rebuild_code( + watch_path, + changed_paths=paths, + output_root=output_root, + follow_symlinks=follow_symlinks, + force=force, + no_cluster=no_cluster, + acquire_lock=False, + include_semantic=include_semantic, + code_only=code_only, + backend=backend, + model=model, + deep_mode=deep_mode, + token_budget=token_budget, + max_concurrency=max_concurrency, + retain_rollback=retain_rollback, + external_extraction=external_extraction, + external_kind=external_kind, + prune_excluded=prune_excluded, + raise_on_error=raise_on_error, + _root_marker=_root_marker, + _timer=_timer, + ) + + result = run_inner(first_paths) + if changed_paths is None: + return result + for _ in range(_PENDING_DRAIN_MAX_PASSES): + late = _drain_pending(out) + if not late: + break + result = run_inner(late) and result + return result + try: - from graphify.extract import extract, _get_extractor + from graphify.analyze import god_nodes, suggest_questions, surprising_connections + from graphify.build import build_from_extraction, build_unclustered_extraction + from graphify.cluster import ( + cluster, + remap_communities_to_previous, + score_all, + ) + from graphify.cache import check_semantic_cache, save_semantic_cache from graphify.detect import detect - from graphify.build import build_from_json, _norm_source_file as _nsf - from graphify.cluster import cluster, remap_communities_to_previous, score_all - from graphify.analyze import god_nodes, surprising_connections, suggest_questions + from graphify.export import to_html + from graphify.extract import extract + from graphify.helix.model import GraphBuildData + from graphify.helix.native import HELIX_PYTHON_VERSION + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.helix.state import ( + communities_from_state, + community_records, + community_summaries, + labels_from_state, + new_state, + ) from graphify.report import generate - from graphify.export import to_json, to_html - from graphify.security import check_graph_file_size_cap - # Re-apply the excludes the initial extract recorded, so an update/watch/ - # hook rebuild does not silently re-include deliberately excluded paths - # (#1886). - _persisted_excludes = _read_build_excludes(out) + out.mkdir(parents=True, exist_ok=True) + _warn_obsolete_store(out) + store_path = out / "graph.helix" detected = detect( - watch_path, follow_symlinks=follow_symlinks, - extra_excludes=_persisted_excludes or None, + watch_path, + follow_symlinks=follow_symlinks, + extra_excludes=_read_build_excludes(out) or None, gitignore=_read_build_gitignore(out), ) - code_files = [Path(f) for f in detected['files']['code']] - - # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) - ast_doc_files: list[Path] = [] - for doc_file in detected['files'].get('document', []): - p = Path(doc_file) - if _get_extractor(p) is not None: - code_files.append(p) - ast_doc_files.append(p) - - existing_graph = out / "graph.json" - if not code_files and not existing_graph.exists(): - print("[graphify watch] No code files found - nothing to rebuild.") - return False + skipped = detected.get("skipped_sensitive", []) + sensitive = [ + str(path) + for path in skipped + if isinstance(path, str) and " [" not in path + ] + if sensitive: + names = ", ".join( + sorted({Path(path).name for path in sensitive})[:6] + ) + more = f" (+{len(sensitive) - 6} more)" if len(sensitive) > 6 else "" + print( + f"[graphify extract] {len(sensitive)} file(s) skipped as " + "potentially sensitive (rename or move if wrongly flagged): " + f"{names}{more}" + ) + if _timer is not None: + _timer.mark("detect") + files_by_type = detected.get("files", {}) + code_files = [Path(item) for item in files_by_type.get("code", [])] + semantic_files = [ + Path(item) + for kind in ("document", "paper", "image") + for item in files_by_type.get(kind, []) + ] + detected_files = [*code_files, *semantic_files] + quick_document_files = [ + path + for path in semantic_files + if path.suffix.lower() in {".md", ".mdx", ".qmd", ".skill"} + ] + quick_files = [*code_files, *quick_document_files] + if deep_mode: + semantic_files = [*code_files, *semantic_files] + if code_only: + semantic_files = [] + quick_files = code_files + current_files = [*code_files, *semantic_files] + current_absolute = { + identity + for path in detected_files + for identity in ( + Path(os.path.abspath(path)), + path.resolve(), + ) + } - # #1915: a document that already carries SEMANTIC (LLM) nodes in the - # existing graph must not ALSO be AST-quick-scanned — otherwise every - # rebuild mints heading nodes on top of the preserved semantic nodes - # and the doc is represented twice (~4x graph bloat vs the CLI update - # path, which AST-extracts only code). Semantic supersedes AST per doc - # source: the quick-scan stays as a fallback for docs with no semantic - # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new - # docs the graph has never seen. These docs stay in ``code_files`` so - # corpus membership (#1795 fail-closed deletion evidence) and the - # shrink accounting below still cover them — a previously-bloated - # graph must be allowed to self-heal on a full rebuild without the - # shrink-guard refusing the smaller write. - semantic_doc_files: set[Path] = set() - if ast_doc_files and existing_graph.exists(): + def resolve_changed(path: Path) -> Path: + if path.is_absolute(): + return Path(os.path.abspath(path)) + candidates = [ + Path(os.path.abspath(source_root / path)), + Path(os.path.abspath(watch_path / path)), + ] + for candidate in candidates: + if candidate in current_absolute: + return candidate + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] if candidates[0].is_relative_to(watch_path) else candidates[1] + + deleted_sources: set[str] = set() + excluded_sources: set[str] = set() + if changed_paths is not None: + requested = [resolve_changed(Path(path)) for path in changed_paths] + extract_targets = [ + path for path in requested + if path.resolve() in {item.resolve() for item in quick_files} and path.is_file() + ] + semantic_targets = [ + path for path in requested + if include_semantic + and path.resolve() in {item.resolve() for item in semantic_files} + and path.is_file() + ] + deleted_sources.update( + path.relative_to(source_root).as_posix() + for path in requested + if not path.exists() + and path.is_relative_to(watch_path) + and path.is_relative_to(source_root) + ) + else: + extract_targets = quick_files + semantic_targets = semantic_files if include_semantic else [] + + previous_state = new_state() + loaded = None + had_existing_generation = False + existing_source_root = source_root + marker = out / ".graphify_root" + saved_root = Path(".") + if marker.is_file(): try: - check_graph_file_size_cap(existing_graph) - prior = json.loads(existing_graph.read_text(encoding="utf-8")) - prior_paths = _StoredSourcePaths( - prior, - out=out, - project_root=project_root, - watch_root=watch_root, - normalize_source=_nsf, + saved_root = Path(marker.read_text(encoding="utf-8").strip()) + existing_source_root = ( + saved_root.resolve() + if saved_root.is_absolute() + else source_root ) - # Semantic doc nodes lack the AST origin marker. Gate on the - # doc-shaped subset of the six-value file_type enum - # (document/concept/rationale/paper AND code) rather than - # "document" alone: per the extraction spec, a doc full of named - # concepts may be represented with ONLY concept/rationale - # nodes and no separate "document" node — that's still - # evidence of a semantic layer, not a marker-less AST node - # (#1954). "code" is included too (#2014): the semantic pass - # legitimately mints code-typed nodes for symbols surfaced from - # WITHIN a doc (llm.py `_bind_node_evidence`), and it cannot be - # confused with a pre-#1865 marker-less AST code node — those are - # sourced from code files, which never intersect ast_doc_files - # below, whereas the AST quick-scan of a doc only ever mints - # "document" nodes (extractors/markdown.py). "image" stays out. - semantic_doc_identities: set[str] = set() - for node in prior.get("nodes", []): - if node.get("_origin") == "ast": - continue - if node.get("file_type") not in ( - "document", "concept", "rationale", "paper", "code" + except (OSError, ValueError): + pass + if store_path.is_dir(): + try: + # Updates are already a writer operation. Open through the + # ordinary public writer so Helix can complete any required + # embedded-format migration before Graphify takes a snapshot. + with HelixEmbeddedStore(store_path) as existing_store: + if no_cluster: + previous_state = copy.deepcopy(existing_store.read_state()) + had_existing_generation = True + else: + loaded = existing_store.load() + previous_state = copy.deepcopy(dict(loaded.state)) + had_existing_generation = True + except RuntimeError as exc: + if "no active generation" not in str(exc): + raise + if had_existing_generation: + stored_files = previous_state.get("incremental", {}).get("files", {}) + stored_sources = ( + [str(source) for source in stored_files] + if isinstance(stored_files, dict) + else [] + ) + if marker.is_file() and not saved_root.is_absolute() and saved_root != Path("."): + marker_prefix = saved_root.as_posix().rstrip("/") + "/" + if stored_sources and not any( + source.startswith(marker_prefix) for source in stored_sources ): + existing_source_root = watch_path + for source in stored_sources: + source_path = Path(str(source)) + absolute = ( + Path(os.path.abspath(source_path)) + if source_path.is_absolute() + else Path(os.path.abspath(existing_source_root / source_path)) + ) + if not absolute.is_relative_to(watch_path): continue - identity = prior_paths.identity(node.get("source_file")) - if identity: - semantic_doc_identities.add(identity) - if semantic_doc_identities: - semantic_doc_files = { - p for p in ast_doc_files - if prior_paths.absolute_identity(str(p), project_root) - in semantic_doc_identities - } - except Exception: - semantic_doc_files = set() - - # Incremental path: when the caller passed an explicit change list, - # extract only changed-and-still-existing files. Deleted paths are - # tracked separately so their stale nodes can be evicted below. - deleted_paths: set[str] = set() - deleted_source_identities: set[str] = set() - def _add_deleted_source(path: Path) -> None: - deleted_source_identities.add(Path(os.path.abspath(path)).as_posix()) - for root in (project_root, watch_root): - deleted_paths.add(_nsf(str(path), str(root)) or str(path)) + if not absolute.exists(): + deleted_sources.add(str(source).replace("\\", "/")) + elif absolute not in current_absolute: + excluded_sources.add(str(source).replace("\\", "/")) + if excluded_sources: + action = ( + "pruned native nodes from" + if prune_excluded + else "fail-closed: kept native nodes from" + ) + print( + f"[graphify watch] {action} {len(excluded_sources)} " + "excluded-but-existing source file(s)." + ) - if changed_paths is not None: - code_set = {Path(os.path.abspath(p)) for p in code_files} - # #1915: semantic-backed docs are never AST-quick-scanned; their - # semantic nodes are the sole representation. Mirroring #1865's - # tier-scoped edge rule at the node level, they also must NOT - # enter extract_targets (hence rebuilt/node-evicted identities) on - # an incremental rebuild, or their semantic nodes would be wiped. - semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} - wanted: list[Path] = [] - change_root = Path.cwd().resolve() - for raw in changed_paths: - candidates = _changed_path_candidates( - raw, - change_root=change_root, - watch_root=watch_root, - ) - tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) - if tracked is not None: - if tracked not in wanted and tracked not in semantic_doc_set: - wanted.append(tracked) + semantic_backed_sources: set[str] = set() + previous_cache = previous_state.get("incremental", {}).get( + "extraction_cache", {} + ) + if isinstance(previous_cache, dict): + for entry in previous_cache.values(): + if not isinstance(entry, dict) or not str(entry.get("kind", "")).startswith( + "semantic" + ): continue - - existing_in_root = next( - ( - cand for cand in candidates - if cand.exists() and _is_relative_to(cand, watch_root) - ), - None, - ) - if existing_in_root is not None: - # The path exists under the watched root but detect filtered - # it out of code_set (no AST extractor, excluded, or - # sensitive). Existence is NOT deletion evidence (#2056): the - # file may carry semantic (LLM) nodes an AST rebuild cannot - # regenerate, and mis-routing it to _add_deleted_source both - # evicts those nodes AND sets had_explicit_deletions, which - # disables the shrink guard that would otherwise catch the - # loss. Preserve it — a genuine deletion still evicts via the - # branch below, the corpus sweep evicts a truly-gone non-AST - # source, and a deliberate exclusion is purged by a full - # re-extraction. + result_value = entry.get("result", {}) + if not isinstance(result_value, dict): continue + for node in result_value.get("nodes", []): + if not isinstance(node, dict): + continue + source = node.get("source_file") + if source and Path(str(source)).suffix.lower() in { + ".md", ".mdx", ".qmd", ".skill", + }: + semantic_backed_sources.add(str(source).replace("\\", "/")) + if semantic_backed_sources: + extract_targets = [ + path + for path in extract_targets + if Path(os.path.abspath(path)).relative_to(source_root).as_posix() + not in semantic_backed_sources + ] + + cache_state = copy.deepcopy( + previous_state.get("incremental", {}).get("extraction_cache", {}) + ) + if not isinstance(cache_state, dict): + cache_state = {} + if force or os.environ.get("GRAPHIFY_FORCE", "").strip() == "1": + cache_state.clear() + # Watch/update preserve a live source fail-closed when detector or ignore + # behavior changes. An explicit full ``extract`` opts into adopting the + # newly detected corpus boundary, while deletion and ``--force`` remain + # explicit removal signals for incremental callers. + pruned_sources = deleted_sources | (excluded_sources if prune_excluded else set()) + if pruned_sources: + deleted_suffixes = {":" + source.replace("\\", "/") for source in pruned_sources} + for key in list(cache_state): + if any(key.endswith(suffix) for suffix in deleted_suffixes): + del cache_state[key] + + build_root = existing_source_root if had_existing_generation else source_root + + def cached_source_paths(kind_prefix: str) -> list[Path]: + paths: dict[str, Path] = {} + for entry in cache_state.values(): + if not isinstance(entry, dict) or not str(entry.get("kind", "")).startswith( + kind_prefix + ): + continue + cached_result = entry.get("result", {}) + if not isinstance(cached_result, dict): + continue + for bucket in ("nodes", "edges", "hyperedges"): + for item in cached_result.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): + continue + source_path = Path(str(item["source_file"])) + unresolved = ( + source_path + if source_path.is_absolute() + else build_root / source_path + ) + absolute = Path(os.path.abspath(unresolved)) + if absolute.is_file(): + paths[os.fspath(absolute)] = absolute + return list(paths.values()) + + def cached_virtual_semantic() -> dict[str, list[dict]]: + """Reuse durable extraction DTOs whose sources are virtual URLs.""" + result: dict[str, list[dict]] = { + "nodes": [], "edges": [], "hyperedges": [], + } + for entry in cache_state.values(): + if not isinstance(entry, dict) or not str(entry.get("kind", "")).startswith( + "semantic" + ): + continue + cached_result = entry.get("result", {}) + if not isinstance(cached_result, dict): + continue + for bucket in result: + for item in cached_result.get(bucket, []): + if ( + isinstance(item, dict) + and _is_remote_source(str(item.get("source_file", ""))) + ): + result[bucket].append(dict(item)) + return result + + if ( + not extract_targets + and not semantic_targets + and not pruned_sources + and external_extraction is None + and not store_path.is_dir() + ): + print("[graphify watch] No supported source files found - nothing to rebuild.") + return False - deleted_in_root = next( - (cand for cand in candidates if _is_relative_to(cand, watch_root)), - None, - ) - if deleted_in_root is not None: - # File was deleted or renamed away inside the watched root. - # Evict preserved nodes that still claim this source path. - _add_deleted_source(deleted_in_root) - if not wanted and not deleted_paths: - print("[graphify watch] No tracked code files in change set - skipping rebuild.") - return True - extract_targets = wanted - else: - # Full rebuild: skip the AST quick-scan for semantic-backed docs - # (#1915). They remain in code_files, so stale _origin=="ast" - # heading nodes from a previously-bloated graph are dropped by the - # full-rebuild AST ownership rule while the shrink accounting - # below still counts the doc as a rebuilt source. - extract_targets = [p for p in code_files if p not in semantic_doc_files] - - commit = _git_head() - result = extract(extract_targets, cache_root=watch_root) if extract_targets else { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, + # Build from extraction DTOs, never by projecting the active native + # topology back into Python. ``extract`` receives the complete live file + # set; unchanged files are served from the cache stored in Helix state, + # while only invalidated files run an extractor. + ast_candidates = { + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*quick_files, *cached_source_paths("ast")] + if Path(path).is_file() } - _rebase_relative_source_files(result, watch_root, project_root) - - # Preserve semantic nodes/edges from a previous full run. - # AST-only rebuild replaces nodes for changed files; everything else is kept. - # Filter by node ID membership in the new AST output, not by file_type — - # INFERRED/AMBIGUOUS nodes extracted from code files also carry file_type="code" - # and would be wrongly dropped by a file_type-based filter. - # When the caller supplied changed_paths, also evict preserved nodes whose - # source_file matches a path that was changed (re-extracted) or deleted — - # otherwise the old nodes for those files would survive forever. - result, existing_graph_data = _reconcile_existing_graph( - existing_graph, - result, - out=out, - project_root=project_root, - watch_root=watch_root, - code_files=code_files, - extract_targets=extract_targets, - full_rebuild=changed_paths is None, - deleted_paths=deleted_paths, - deleted_source_identities=deleted_source_identities, + ast_build_files = [] + for path in ast_candidates.values(): + try: + relative = path.relative_to(build_root).as_posix() + except ValueError: + continue + if relative not in semantic_backed_sources: + ast_build_files.append(path) + ast_result = extract( + ast_build_files, root=build_root, cache=cache_state + ) if ast_build_files else { + "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, + } + selected_backend = backend + semantic_result = { + "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, + "output_tokens": 0, + } + previous_semantic = previous_state.get("semantic", {}) + semantic_was_used = ( + isinstance(previous_semantic, dict) + and bool(previous_semantic.get("used")) ) - - _relativize_source_files(result, project_root, scope=watch_root) - # Source files re-extracted this run — their symbol sets may legitimately - # shrink (a removed function), so the shrink-guard should not block the - # write when every lost node belongs to one of them (or a deleted file). - _rebuilt_root = str(project_root) - if changed_paths is None: - rebuilt_sources = { - _nsf(str(p.relative_to(project_root)), _rebuilt_root) - for p in code_files if p.is_relative_to(project_root) + semantic_build_files = ( + list({ + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*semantic_files, *cached_source_paths("semantic")] + if Path(path).is_file() + }.values()) + if include_semantic or semantic_was_used + else [] + ) + if semantic_build_files: + from graphify.llm import _extraction_system, detect_backend, extract_corpus_parallel + + previous_mode = previous_state.get("incremental", {}).get( + "extractor_state", {} + ).get("mode") + effective_deep = deep_mode or (not include_semantic and previous_mode == "deep") + semantic_mode = "deep" if effective_deep else None + prompt = _extraction_system(deep=effective_deep) + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( + [str(path) for path in semantic_build_files], + cache_state, + root=build_root, + mode=semantic_mode, + prompt=prompt, + allow_stale=not include_semantic, + ) + requested_semantic = { + Path(os.path.abspath(path)) for path in semantic_targets } - else: - rebuilt_sources = {(_nsf(str(p), _rebuilt_root) or str(p)) for p in extract_targets} - rebuilt_sources |= set(deleted_paths) - out.mkdir(exist_ok=True) - - if no_cluster: - # Normalise to "links" key so schema is consistent with the full clustered path. - # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); - # without it, --no-cluster + repeated `update` accumulate duplicates and edge - # counts diverge across build modes (#1317). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - candidate_graph_data = { - **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, - "nodes": _dedupe_nodes(result.get("nodes", [])), - "links": _dedupe_edges(result.get("edges", [])), + uncached_targets = [ + Path(path) + for path in uncached + if include_semantic and Path(os.path.abspath(path)) in requested_semantic + ] + fresh = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, } - candidate_graph_text = _json_text(candidate_graph_data) - same_graph = False - if existing_graph.exists(): - try: - check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) - same_graph = ( - json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + if uncached_targets: + selected_backend = backend or detect_backend() + if selected_backend is None: + raise RuntimeError( + "semantic sources were detected but no LLM backend is configured; " + "pass --backend, configure an API key, or use --code-only" ) - except Exception: - same_graph = False - if not same_graph: - if not _check_shrink( - force, existing_graph_data, candidate_graph_data, - had_explicit_deletions=bool(deleted_paths), - rebuilt_sources=rebuilt_sources, - ): - return False - existing_graph.write_text(candidate_graph_text, encoding="utf-8") - - # Write the user-supplied path only after the candidate graph is - # accepted, so a refused shrink cannot mismatch graph and marker. - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") - - try: - from graphify.detect import save_manifest - # detected["files"] is a FULL detect of the watched root, so - # pass it as the scan corpus too: rows for files that left the - # scan but still exist on disk (newly excluded) are pruned - # instead of surviving as phantom "deleted" entries (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + fresh = extract_corpus_parallel( + uncached_targets, + backend=selected_backend, + model=model, + root=build_root, + token_budget=token_budget, + max_concurrency=max_concurrency, + deep_mode=effective_deep, + cache=cache_state, + cache_root=output_root, ) - except Exception: - pass - - # clear stale needs_update flag if present - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - - if same_graph: - print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.") - else: + if fresh.get("failed_chunks"): + raise RuntimeError( + f"semantic extraction failed for {fresh['failed_chunks']} chunk(s); " + "the active generation was left unchanged" + ) + save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=build_root, + cache_root=output_root, + allowed_source_files=uncached_targets, + mode=semantic_mode, + prompt=prompt, + cache=cache_state, + ) + elif not uncached: + selected_backend = ( + backend or previous_state.get("semantic", {}).get("backend") + ) + semantic_result = { + "nodes": [*cached_nodes, *fresh.get("nodes", [])], + "edges": [*cached_edges, *fresh.get("edges", [])], + "hyperedges": [*cached_hyperedges, *fresh.get("hyperedges", [])], + "input_tokens": fresh.get("input_tokens", 0), + "output_tokens": fresh.get("output_tokens", 0), + } + virtual_semantic = cached_virtual_semantic() + for bucket in ("nodes", "edges", "hyperedges"): + semantic_result[bucket].extend(virtual_semantic[bucket]) + external_extractions = copy.deepcopy( + previous_state.get("incremental", {}).get( + "external_extractions", {} + ) + ) + if not isinstance(external_extractions, dict): + external_extractions = {} + if external_extraction is not None: + external_extractions[external_kind or "external"] = copy.deepcopy( + external_extraction + ) + external_results = [ + value + for value in external_extractions.values() + if isinstance(value, dict) + ] + current_files = list({ + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*current_files, *ast_build_files, *semantic_build_files] + if Path(path).is_file() + }.values()) + result = { + "nodes": [ + *ast_result.get("nodes", []), + *semantic_result.get("nodes", []), + *(item for value in external_results for item in value.get("nodes", [])), + ], + "edges": [ + *ast_result.get("edges", []), + *semantic_result.get("edges", []), + *(item for value in external_results for item in value.get("edges", [])), + ], + "hyperedges": [ + *ast_result.get("hyperedges", []), + *semantic_result.get("hyperedges", []), + *( + item + for value in external_results + for item in value.get("hyperedges", []) + ), + ], + "input_tokens": ast_result.get("input_tokens", 0) + semantic_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + semantic_result.get("output_tokens", 0), + } + if _timer is not None: + _timer.mark("extract") + build_data = ( + build_unclustered_extraction(result, root=build_root) + if no_cluster + else build_from_extraction(result, root=build_root) + ) + if _timer is not None: + _timer.mark("build") + if had_existing_generation: + previous_topology_sources = previous_state.get("incremental", {}).get( + "topology_sources", [] + ) + expected_sources = { + str(source).replace("\\", "/") + for source in previous_topology_sources + if isinstance(source, str) + and (build_root / source).is_file() + and ( + not prune_excluded + or str(source).replace("\\", "/") not in excluded_sources + ) + } if isinstance(previous_topology_sources, list) else set() + candidate_sources = set(_topology_sources(build_data)) + missing_live_sources = expected_sources - candidate_sources + if missing_live_sources and not force: + sample = ", ".join(sorted(missing_live_sources)[:5]) print( - "[graphify watch] Rebuilt (no clustering): " - f"{len(candidate_graph_data.get('nodes', []))} nodes, " - f"{len(candidate_graph_data.get('links', []))} edges" + "[graphify] WARNING: extraction omitted live source file(s) " + f"present in the active generation ({sample}); pass --force " + "after confirming the extractor change.", + file=sys.stderr, ) - print(f"[graphify watch] graph.json updated in {out}") - return True + return False detection = { - "files": {"code": [str(f) for f in code_files], "document": [], "paper": [], "image": []}, - "total_files": len(code_files), + "files": {key: list(value) for key, value in files_by_type.items()}, + "total_files": detected.get("total_files", len(current_files)), "total_words": detected.get("total_words", 0), } - G = build_from_json(result) - candidate_topology = _topology_from_graph(G) - if existing_graph_data: - try: - same_topology = ( - json.dumps(_canonical_topology_for_compare(existing_graph_data), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_topology_for_compare(candidate_topology), sort_keys=True, ensure_ascii=False) - ) - except Exception: - same_topology = False - if same_topology: - try: - from graphify.detect import save_manifest - # Full-scan save: prune excluded-but-alive rows (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + def analyze_generation(graph, *, reuse_communities: bool = False): + if no_cluster: + communities = {} + cohesion = {} + labels = {} + elif reuse_communities: + communities = communities_from_state(previous_state) + labels = labels_from_state(previous_state) + cohesion = { + int(record["id"]): float(record["cohesion"]) + for record in previous_state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("cohesion"), (int, float)) + } + else: + communities = cluster(graph) + previous_membership = { + member: community_id + for community_id, members in communities_from_state( + previous_state + ).items() + for member in members + } + if previous_membership: + communities = remap_communities_to_previous( + communities, previous_membership ) - except Exception: - pass - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - print("[graphify watch] No code-graph topology changes detected; outputs left untouched.") - return True - - communities = cluster(G) - previous_node_community = _node_community_map(existing_graph_data) - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - labels_file = out / ".graphify_labels.json" - try: - raw = json.loads(labels_file.read_text(encoding="utf-8")) if labels_file.exists() else {} - # Skip persisted "Community N" placeholders so the hub-fill below - # replaces them instead of perpetuating them on every rebuild (#2073). - labels = { - int(k): v for k, v in raw.items() - if int(k) in communities and v != f"Community {int(k)}" + cohesion = score_all(graph, communities) + labels = _community_labels(graph, communities) + if no_cluster: + gods = [] + surprises = [] + questions = [] + else: + gods = god_nodes(graph) + surprises = surprising_connections(graph, communities) + questions = suggest_questions(graph, communities, labels) + state = copy.deepcopy(previous_state) + state["build"] = { + "helix_python_version": HELIX_PYTHON_VERSION, + "node_count": graph.node_count, + "edge_count": graph.edge_count, + "directed": graph.directed, + "multigraph": graph.multigraph, + "semantic": bool(semantic_targets) + or bool(previous_state.get("semantic", {}).get("used")), + "source_root": str(source_root), + "source_commit": _git_head(watch_path), } - except Exception: - raw = {} - labels = {} - missing = {cid: members for cid, members in communities.items() if cid not in labels} - if missing: - # Deterministic hub name (highest-degree member) beats a bare "Community N" - # placeholder for any community without a saved label. - from graphify.cluster import label_communities_by_hub - labels.update(label_communities_by_hub(G, missing)) - questions = suggest_questions(G, communities, labels) - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, detection, - {"input": 0, "output": 0}, report_root, suggested_questions=questions, - built_at_commit=commit, learning=_llfr(out / "graph.json")) - report_path = out / "GRAPH_REPORT.md" - labels_json = json.dumps({str(k): v for k, v in sorted(labels.items())}, ensure_ascii=False, indent=2) + "\n" - graph_tmp = out / ".graph.tmp.json" - json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit, community_labels=labels) - if not json_written: - return False - candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8")) - same_graph = False - same_report = False - if existing_graph.exists(): - try: - check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) - same_graph = ( - json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source="generated", + ) + state["analysis"] = { + "god_nodes": gods, + "surprises": surprises, + "suggested_questions": questions, + "confidence_counts": { + confidence: sum( + 1 + for edge in build_data.edges + if str(edge.attributes.get("confidence", "EXTRACTED")) + == confidence + ) + for confidence in ("EXTRACTED", "INFERRED", "AMBIGUOUS") + }, + "community_summaries": ( + [] + if no_cluster + else community_summaries(graph, communities, labels) + ), + "report_inputs": { + "detection": detection, + "tokens": { + "input": result.get("input_tokens", 0), + "output": result.get("output_tokens", 0), + }, + "source": str(watch_path), + }, + } + state["incremental"] = { + "files": { + path.relative_to(build_root).as_posix(): { + "content_hash": _content_hash(path), + "semantic_hash": "", + "extractor_state": "ast", + } + for path in current_files + if path.is_file() and path.is_relative_to(build_root) + }, + "extractor_state": { + "mode": "deep" if deep_mode else "semantic" if semantic_targets else "ast", + "backend": selected_backend if semantic_targets else None, + }, + "topology_sources": _topology_sources(build_data), + "extraction_cache": cache_state, + "external_extractions": external_extractions, + } + state["semantic"] = { + "used": bool(semantic_targets) + or bool(previous_state.get("semantic", {}).get("used")), + "backend": selected_backend + if semantic_targets + else previous_state.get("semantic", {}).get("backend"), + "input_tokens": result.get("input_tokens", 0), + "output_tokens": result.get("output_tokens", 0), + } + return communities, cohesion, labels, gods, surprises, questions, state + + with HelixEmbeddedStore( + store_path, retain_rollback=retain_rollback + ) as store: + topology_unchanged = ( + had_existing_generation and store.topology_matches(build_data) + ) + if no_cluster: + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation( + build_data, reuse_communities=topology_unchanged ) - except Exception: - same_graph = False - if report_path.exists(): - old_report = report_path.read_text(encoding="utf-8") - same_report = _report_for_compare(old_report) == _report_for_compare(report) - no_change = same_graph and same_report - if no_change: - graph_tmp.unlink(missing_ok=True) - print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.") - else: - if not _check_shrink( - force, existing_graph_data, candidate_graph_data, - tmp=graph_tmp, - had_explicit_deletions=bool(deleted_paths), - rebuilt_sources=rebuilt_sources, - ): - return False - from graphify.export import backup_if_protected as _backup - _backup(out) - graph_tmp.replace(existing_graph) - report_path.write_text(report, encoding="utf-8") - labels_file.write_text(labels_json, encoding="utf-8") + if topology_unchanged: + store.replace_state( + state, + previous_state=previous_state, + ) + print("[graphify watch] No code-graph topology changes detected.") + else: + store.save_generation(build_data, state) + graph_node_count = build_data.node_count + graph_edge_count = build_data.edge_count + elif topology_unchanged: + assert loaded is not None + graph = loaded.graph + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation(graph, reuse_communities=True) + store.replace_state( + state, + previous_state=previous_state, + snapshot=loaded, + ) + print("[graphify watch] No code-graph topology changes detected.") + else: + with store.staged_graph(build_data) as staged: + graph = staged.graph + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation(graph) + del graph + activated = store.activate_staged(staged, state) + graph = activated.graph + if _timer is not None: + _timer.mark("activate") - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + if no_cluster: + # Reopen the activated store through the ordinary public reader and + # verify its generation counts before reporting success. + with HelixEmbeddedStore(store_path, read_only=True) as reopened: + reopened_counts = reopened.verify_counts() + if ( + reopened_counts["nodes"] != graph_node_count + or reopened_counts["edges"] != graph_edge_count + ): + raise RuntimeError( + "activated Helix generation failed reopen verification" + ) + (out / ".graphify_root").write_text(_root_marker, encoding="utf-8") + (out / "needs_update").unlink(missing_ok=True) + print( + f"[graphify watch] Rebuilt (no clustering): {graph_node_count} nodes, " + f"{graph_edge_count} edges" + ) + print(f"[graphify watch] graph.helix updated in {out}") + return True + report = generate( + graph, communities, cohesion, labels, gods, surprises, detection, + { + "input": result.get("input_tokens", 0), + "output": result.get("output_tokens", 0), + }, str(watch_path), + suggested_questions=questions, built_at_commit=_git_head(watch_path), + learning=state.get("learning", {}), + ) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") try: - from graphify.detect import save_manifest - # Full-scan save: prune excluded-but-alive rows (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, - ) - except Exception: - pass + to_html(graph, communities, str(out / "graph.html"), community_labels=labels) + except ValueError as exc: + print(f"[graphify watch] Skipped graph.html: {exc}") + (out / "graph.html").unlink(missing_ok=True) - # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000). - # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land. - html_written = False - if not no_change: - try: - to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) - html_written = True - except ValueError as viz_err: - print(f"[graphify watch] Skipped graph.html: {viz_err}") - stale = out / "graph.html" - if stale.exists(): - stale.unlink() - - # Regenerate callflow HTML if the user previously generated one — - # opt-in by existence so users who never ran callflow-html aren't affected. - callflow_files = list(out.glob("*-callflow.html")) - if callflow_files and not no_change: + for callflow in out.glob("*-callflow.html"): try: from graphify.callflow_html import write_callflow_html - for cf in callflow_files: - write_callflow_html( - graph=out / "graph.json", - report=out / "GRAPH_REPORT.md", - labels=out / ".graphify_labels.json", - output=cf, - verbose=False, - ) - except Exception as cf_err: - print(f"[graphify watch] callflow HTML update skipped: {cf_err}") - - # clear stale needs_update flag if present - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - - if not no_change: - print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " - f"{G.number_of_edges()} edges, {len(communities)} communities") - products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md" - if callflow_files: - products += f", {len(callflow_files)} callflow HTML" - print(f"[graphify watch] {products} updated in {out}") - return True + write_callflow_html( + graph=store_path, report=out / "GRAPH_REPORT.md", + output=callflow, + verbose=False, + ) + except Exception as exc: + print(f"[graphify watch] callflow HTML update skipped: {exc}") + + (out / ".graphify_root").write_text(_root_marker, encoding="utf-8") + (out / "needs_update").unlink(missing_ok=True) + print( + f"[graphify watch] Rebuilt: {graph.node_count} nodes, {graph.edge_count} edges, " + f"{len(communities)} communities" + ) + print(f"[graphify watch] graph.helix and GRAPH_REPORT.md updated in {out}") + return True except Exception as exc: + if raise_on_error: + raise print(f"[graphify watch] Rebuild failed: {exc}") return False def check_update(watch_path: Path) -> bool: - """Check for pending semantic update flag and notify the user if set. - - Cron-safe: always returns True so cron jobs do not alarm. - Non-code file changes (docs, papers, images) require LLM-backed - re-extraction via `/graphify --update` — this function only signals - that the update is needed. - """ flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" if flag.exists(): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") - print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") + print("[graphify check-update] Run `graphify --update` to apply semantic re-extraction.") return True def _notify_only(watch_path: Path) -> None: - """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / _GRAPHIFY_OUT / "needs_update" flag.parent.mkdir(parents=True, exist_ok=True) flag.write_text("1", encoding="utf-8") - print(f"\n[graphify watch] New or changed files detected in {watch_path}") - print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") - print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") - print(f"[graphify watch] Flag written to {flag}") + print(f"\n[graphify watch] Non-code files changed in {watch_path}; run `graphify update .`.") def _has_non_code(changed_paths: list[Path]) -> bool: - return any(p.suffix.lower() not in _CODE_EXTENSIONS for p in changed_paths) + return any(path.suffix.lower() not in _CODE_EXTENSIONS for path in changed_paths) -def watch(watch_path: Path, debounce: float = 3.0) -> None: - """ - Watch watch_path for new or modified files and auto-update the graph. - - For code-only changes: re-runs AST extraction + rebuild immediately (no LLM). - For doc/paper/image changes: writes a needs_update flag and notifies the user - to run /graphify --update (LLM extraction required). - - debounce: seconds to wait after the last change before triggering (avoids - running on every keystroke when many files are saved at once). - """ +def watch( + watch_path: Path, + debounce: float = 3.0, + *, + retain_rollback: bool = False, +) -> None: try: + from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver - from watchdog.events import FileSystemEventHandler - except ImportError as e: - raise ImportError("watchdog not installed. Run: pip install watchdog") from e - - last_trigger: float = 0.0 - pending: bool = False - changed: set[Path] = set() - - # Load .graphifyignore patterns ONCE at startup so the handler does not - # re-parse the file on every filesystem event. Watchdog's handler runs on - # the observer thread and is invoked for every event the OS delivers - # (Time Machine writes, Docker/Colima VM I/O, Spotlight indexing, …) — - # without this short-circuit a busy volume can saturate a CPU core - # discarding events one extension at a time. (gh-928) - watch_root_for_ignore = watch_path.resolve() + except ImportError as exc: + raise ImportError("watchdog not installed. Run: pip install watchdog") from exc + + watch_path = Path(watch_path).resolve() + last_trigger = 0.0 + pending = False + changed: list[Path] = [] + ignore_root = watch_path.resolve() ignore_patterns = _load_graphifyignore( - watch_root_for_ignore, + ignore_root, gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), ) @@ -1433,52 +1174,40 @@ def on_any_event(self, event): if event.is_directory: return path = Path(os.fsdecode(event.src_path)) - # Check .graphifyignore BEFORE the extension/dotfile/out filters so - # the cheapest short-circuit for users with broad ignore patterns - # (node_modules/, .venv/, build/, …) fires first. _is_ignored - # tolerates absolute paths outside watch_root via its internal - # relative_to guard, so a stray symlinked event won't raise. - if ignore_patterns and _is_ignored(path, watch_root_for_ignore, ignore_patterns): + if ignore_patterns and _is_ignored(path, ignore_root, ignore_patterns): return if path.suffix.lower() not in _WATCHED_EXTENSIONS: return try: - filter_parts = path.relative_to(watch_root_for_ignore).parts + filter_parts = path.relative_to(ignore_root).parts except ValueError: filter_parts = path.parts - if any(part.startswith(".") for part in filter_parts): - return - if _GRAPHIFY_OUT in filter_parts: + if any(part.startswith(".") for part in filter_parts) or _GRAPHIFY_OUT in filter_parts: return last_trigger = time.monotonic() pending = True - changed.add(path) + if path not in changed: + changed.append(path) - handler = Handler() - # Use polling observer on macOS — FSEvents can miss rapid saves in some editors observer = PollingObserver() if sys.platform == "darwin" else Observer() - observer.schedule(handler, str(watch_path), recursive=True) + observer.schedule(Handler(), str(watch_path), recursive=True) observer.start() - - print(f"[graphify watch] Watching {watch_path.resolve()} - press Ctrl+C to stop") - print(f"[graphify watch] Code changes rebuild graph automatically. " - f"Doc/image changes require /graphify --update.") - print(f"[graphify watch] Debounce: {debounce}s") - + print(f"[graphify watch] Watching {watch_path} - press Ctrl+C to stop") try: while True: time.sleep(0.5) - if pending and (time.monotonic() - last_trigger) >= debounce: + if pending and time.monotonic() - last_trigger >= debounce: pending = False batch = list(changed) changed.clear() - print(f"\n[graphify watch] {len(batch)} file(s) changed") - has_non_code = _has_non_code(batch) - has_code = any(p.suffix.lower() in _CODE_EXTENSIONS for p in batch) - if has_code: - _rebuild_code(watch_path) - if has_non_code: + if _has_non_code(batch): _notify_only(watch_path) + else: + _rebuild_code( + watch_path, + changed_paths=batch, + retain_rollback=retain_rollback, + ) except KeyboardInterrupt: print("\n[graphify watch] Stopped.") finally: @@ -1488,9 +1217,14 @@ def on_any_event(self, event): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(description="Watch a folder and auto-update the graphify graph") - parser.add_argument("path", nargs="?", default=".", help="Folder to watch (default: .)") - parser.add_argument("--debounce", type=float, default=3.0, - help="Seconds to wait after last change before updating (default: 3)") + + parser = argparse.ArgumentParser(description="Watch a folder and update graph.helix") + parser.add_argument("path", nargs="?", default=".") + parser.add_argument("--debounce", type=float, default=3.0) + parser.add_argument("--retain-rollback", action="store_true") args = parser.parse_args() - watch(Path(args.path), debounce=args.debounce) + watch( + Path(args.path), + debounce=args.debounce, + retain_rollback=args.retain_rollback, + ) diff --git a/graphify/wiki.py b/graphify/wiki.py index cb9c6cf35..2aea8e229 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -3,10 +3,11 @@ from __future__ import annotations from collections import Counter from pathlib import Path +from typing import Any from urllib.parse import quote -import networkx as nx from graphify.build import edge_data +from graphify.helix.model import node_attributes def _safe_filename(name: str) -> str: @@ -48,7 +49,7 @@ def _md_link(label: str, resolver: dict[str, str]) -> str: return f"[{text}]({quote(f'{slug}.md')})" -def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str], node_community: dict[str, int]) -> list[tuple[str, int]]: +def _cross_community_links(G: Any, nodes: list[str], own_cid: int, labels: dict[int, str], node_community: dict[str, int]) -> list[tuple[str, int]]: """Return (community_label, edge_count) pairs for cross-community connections, sorted descending.""" counts: dict[str, int] = Counter() for nid in nodes: @@ -60,7 +61,7 @@ def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: def _community_article( - G: nx.Graph, + G: Any, cid: int, nodes: list[str], label: str, @@ -70,7 +71,7 @@ def _community_article( resolver: dict[str, str] | None = None, ) -> str: resolver = resolver or {} - top_nodes = sorted(nodes, key=lambda n: G.degree(n), reverse=True)[:25] + top_nodes = sorted(nodes, key=lambda n: G.degree(n).degree, reverse=True)[:25] cross = _cross_community_links(G, nodes, cid, labels, node_community or {}) # Edge confidence breakdown @@ -81,7 +82,7 @@ def _community_article( conf_counts[ed.get("confidence", "EXTRACTED")] += 1 total_edges = sum(conf_counts.values()) or 1 - sources = sorted({G.nodes[n].get("source_file") or "" for n in nodes} - {""}) + sources = sorted({node_attributes(G, n).get("source_file") or "" for n in nodes} - {""}) lines: list[str] = [] lines += [f"# {label}", ""] @@ -93,10 +94,10 @@ def _community_article( lines += ["## Key Concepts", ""] for nid in top_nodes: - d = G.nodes[nid] + d = node_attributes(G, nid) node_label = d.get("label", nid) src = d.get("source_file", "") - degree = G.degree(nid) + degree = G.degree(nid).degree src_str = f" — `{src}`" if src else "" lines.append(f"- **{node_label}** ({degree} connections){src_str}") remaining = len(nodes) - len(top_nodes) @@ -129,9 +130,9 @@ def _community_article( return "\n".join(lines) -def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str], node_community: dict[str, int] | None = None, resolver: dict[str, str] | None = None) -> str: +def _god_node_article(G: Any, nid: str, labels: dict[int, str], node_community: dict[str, int] | None = None, resolver: dict[str, str] | None = None) -> str: resolver = resolver or {} - d = G.nodes[nid] + d = node_attributes(G, nid) node_label = d.get("label", nid) src = d.get("source_file", "") cid = (node_community or {}).get(nid) @@ -139,15 +140,15 @@ def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str], node_commun lines: list[str] = [] lines += [f"# {node_label}", ""] - lines += [f"> God node · {G.degree(nid)} connections · `{src}`", ""] + lines += [f"> God node · {G.degree(nid).degree} connections · `{src}`", ""] if community_name: lines += [f"**Community:** {_md_link(community_name, resolver)}", ""] # Group neighbors by relation type by_relation: dict[str, list[str]] = {} - for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True): - nd = G.nodes[neighbor] + for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n).degree, reverse=True): + nd = node_attributes(G, neighbor) ed = edge_data(G, nid, neighbor) rel = ed.get("relation", "related") neighbor_label = nd.get("label", neighbor) @@ -203,13 +204,13 @@ def _index_md( lines += [ "---", "", - "*Generated by [graphify](https://github.com/safishamsi/graphify)*", + "*Generated by [graphify](https://github.com/Graphify-Labs/graphify)*", ] return "\n".join(lines) def to_wiki( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_dir: str | Path, community_labels: dict[int, str] | None = None, @@ -234,12 +235,9 @@ def to_wiki( "Run `graphify extract .` or `graphify cluster-only .` first." ) - # Filter stale node IDs that exist in communities but not in G. - # Analysis JSON can drift from the graph after dedup / re-extract / update. - # NetworkX 3.x returns DegreeView({}) for missing nodes instead of raising, - # which crashes sorted() with TypeError; G.neighbors()/G.nodes[] also raise. + # Filter stale community IDs that no longer exist in the active generation. import sys as _sys - _g_nodes = set(G.nodes) + _g_nodes = {node.id for node in G.nodes()} _orig_total = sum(len(ns) for ns in communities.values()) communities = {cid: [n for n in nodes if n in _g_nodes] for cid, nodes in communities.items()} communities = {cid: nodes for cid, nodes in communities.items() if nodes} @@ -254,7 +252,7 @@ def to_wiki( if not communities: raise ValueError( "all community node IDs are stale — none exist in the graph. " - "Re-run `graphify extract .` to regenerate .graphify_analysis.json." + "Re-run `graphify extract .` to regenerate native analysis state." ) # Clear stale .md files from previous runs to prevent orphan accumulation. @@ -269,6 +267,9 @@ def to_wiki( labels = community_labels or {cid: f"Community {cid}" for cid in communities} cohesion = cohesion or {} god_nodes_data = god_nodes_data or [] + membership = { + node: cid for cid, members in communities.items() for node in members + } # Build node->community lookup once; node attrs never carry community (it lives in # the communities dict), so _cross_community_links and _god_node_article need this. @@ -311,7 +312,7 @@ def _unique_slug(base: str) -> str: god_articles: list[tuple[str, str]] = [] # (node_id, slug) for node_data in god_nodes_data: nid = node_data.get("id") - if nid and nid in G: + if nid and G.contains_node(nid): slug = _unique_slug(_safe_filename(node_data['label'])) god_articles.append((nid, slug)) resolver.setdefault(node_data['label'], slug) @@ -330,7 +331,7 @@ def _unique_slug(base: str) -> str: # Index (out / "index.md").write_text( - _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges(), resolver), + _index_md(communities, labels, god_nodes_data, G.node_count, G.edge_count, resolver), encoding="utf-8", ) diff --git a/pyproject.toml b/pyproject.toml index 2d2abafb6..a9f0d7b44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=77"] +requires = ["setuptools>=83.0.0"] build-backend = "setuptools.build_meta" [project] @@ -12,7 +12,9 @@ license-files = ["LICENSE", "LICENSE-MIT", "NOTICE"] keywords = ["claude", "claude-code", "codex", "opencode", "kilo", "cursor", "gemini", "aider", "kiro", "pi", "devin", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ - "networkx>=3.4", + "helix-db==0.2.0b4", + "helix-db-embedded==0.2.0b4", + "defusedxml>=0.7.1", "numpy>=1.21", "rapidfuzz>=3.0", "tree-sitter>=0.23.0,<0.26", @@ -52,13 +54,12 @@ Issues = "https://github.com/Graphify-Labs/graphify/issues" # starlette is pulled in transitively by mcp, but graphify/serve.py imports it # directly for the HTTP transport, so declare it here and floor it above the # CVE-2026-48818 / CVE-2026-54283 fixes (both resolved by 1.3.1) (#1391, #1396). -mcp = ["mcp", "starlette>=1.3.1"] +mcp = ["mcp>=1.28.1", "starlette>=1.3.1"] neo4j = ["neo4j"] falkordb = ["falkordb"] pdf = ["pypdf>=6.12.0", "markdownify"] watch = ["watchdog"] svg = ["matplotlib", "numpy>=2.0; python_version >= '3.13'"] -leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] google = ["openpyxl"] postgres = ["psycopg[binary]"] @@ -81,7 +82,7 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp>=1.28.1", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] [project.scripts] graphify = "graphify.__main__:main" @@ -100,7 +101,7 @@ dev = [ "pytest>=9.0.3", "pytest-cov>=7.1.0", "ruff>=0.15.13", - "setuptools>=82.0.1", + "setuptools>=83.0.0", "wheel>=0.47.0", "tomli>=2.0 ; python_version < '3.11'", "tree-sitter-hcl>=1.2.0", @@ -112,7 +113,7 @@ dev = [ package = true [tool.setuptools] -packages = ["graphify", "graphify.extractors", "graphify.exporters"] +packages = ["graphify", "graphify.extractors", "graphify.exporters", "graphify.helix"] include-package-data = false [tool.setuptools.package-data] @@ -146,3 +147,8 @@ select = ["E9", "F63", "F7", "F82"] include = ["graphify", "tests"] pythonVersion = "3.10" typeCheckingMode = "basic" + +[tool.graphify.helix] +index = "https://pypi.org/project/helix-db/0.2.0b4/" +package = "helix-db==0.2.0b4" +embedded_package = "helix-db-embedded==0.2.0b4" diff --git a/requirements-runtime.txt b/requirements-runtime.txt new file mode 100644 index 000000000..1f4c36f27 --- /dev/null +++ b/requirements-runtime.txt @@ -0,0 +1,33 @@ +# Production dependencies. Keep synchronized with pyproject.toml. +# Public Helix SDK and its matching embedded runtime. +helix-db==0.2.0b4 +helix-db-embedded==0.2.0b4 +defusedxml>=0.7.1 +numpy>=1.21 +rapidfuzz>=3.0 +tree-sitter>=0.23.0,<0.26 +tree-sitter-python>=0.23,<0.26 +tree-sitter-javascript>=0.23,<0.26 +tree-sitter-typescript>=0.23,<0.25 +tree-sitter-go>=0.23,<0.26 +tree-sitter-rust>=0.23,<0.25 +tree-sitter-java>=0.23,<0.25 +tree-sitter-groovy>=0.1,<0.3 +tree-sitter-c>=0.23,<0.25 +tree-sitter-cpp>=0.23,<0.25 +tree-sitter-ruby>=0.23,<0.25 +tree-sitter-c-sharp>=0.23,<0.25 +tree-sitter-kotlin>=1.0,<2.0 +tree-sitter-scala>=0.23,<0.27 +tree-sitter-php>=0.23,<0.25 +tree-sitter-swift>=0.7,<0.9 +tree-sitter-lua>=0.2,<0.6 +tree-sitter-zig>=1.0,<2.0 +tree-sitter-powershell>=0.26,<0.28 +tree-sitter-elixir>=0.3,<0.5 +tree-sitter-objc>=3.0,<4.0 +tree-sitter-julia>=0.23,<0.25 +tree-sitter-verilog>=1.0,<2.0 +tree-sitter-fortran>=0.6,<0.8 +tree-sitter-bash>=0.23,<0.27 +tree-sitter-json>=0.23,<0.26 diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..420c66ee8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Graphify test package.""" diff --git a/tests/bench_query_scoring.py b/tests/bench_query_scoring.py deleted file mode 100644 index 68e6ce4be..000000000 --- a/tests/bench_query_scoring.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python3 -"""Non-CI microbenchmark: single-pass scoring vs legacy per-term rescoring. - -Verifies the single-pass refactor eliminates the T+1 graph-scoring passes a -T-term query used to run, without changing the result. Reports median/min -latency for: - - * legacy — one combined `_score_nodes(G, terms)` call PLUS one - `_score_nodes(G, [token])` call per distinct query token - (the old `_pick_seeds(terms=...)` per-term guarantee loop). - * optimized — one `_score_query(G, terms, collect_per_term_seeds=True)` - call, with the per-token best computed inline in the same - traversal. `_pick_seeds` then consumes `best_seed_by_term` - directly — no rescoring. - -Equality is asserted (ranked, best_seed_by_term, seeds) before timing. The -optimized path's traversal count is invariant in the number of query terms. - -Run it manually; do NOT wire this into CI (wall-clock assertions are flaky): - - uv run python tests/bench_query_scoring.py \\ - --nodes 100000 --term-counts 3,10 --repeats 5 - - uv run python tests/bench_query_scoring.py \\ - --graph graphify-out/graph.json \\ - --query "what calls extract" --query "symbol resolution" \\ - --repeats 10 -""" -from __future__ import annotations - -import argparse -import json -import random -import statistics -import sys -import time -from pathlib import Path - -import networkx as nx - -from graphify.serve import ( - _get_trigram_index, - _pick_seeds, - _query_terms, - _score_nodes, - _score_query, - _search_tokens, -) - - -SYLLABLES = [ - "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", - "build", "report", "extract", "router", "config", "service", - "handler", "token", "auth", "rate", "limit", "widget", "model", -] - -QUERIES_BY_TERM_COUNT: dict[int, list[str]] = { - 1: ["foo"], - 2: ["foo", "bar"], - 3: ["router", "service", "handler"], - 5: ["get", "user", "run", "name", "path"], - 10: ["extract", "build", "report", "router", "config", - "service", "token", "rate", "limit", "widget"], -} - - -def _build_random_graph(n: int, *, seed: int) -> nx.DiGraph: - """Reproducible broad-match DiGraph: short constructed labels + edge noise. - - Labels draw from a small syllable pool so tokens collide across nodes, - forcing the trigram prefilter to be selective and exercising score ties - on common tokens. Edge noise provides degree variance so the legacy - tie-break (`max(tied, key=degree)`) is actually exercised against the - new `(-singleton, -degree, label_len, nid)` key tuple.""" - rng = random.Random(seed) - G: nx.DiGraph = nx.DiGraph() - for i in range(n): - label = "_".join(rng.sample(SYLLABLES, rng.randint(1, 3))) - G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") - for _ in range(n * 2): - a, b = rng.randrange(n), rng.randrange(n) - if a != b: - G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") - return G - - -def _load_real_graph(path: str) -> nx.Graph: - """Light wrapper that just builds a NetworkX graph from a real - `graphify-out/graph.json`, skipping the size cap and work-memory overlay - that `serve._load_graph` enforces (the bench is read-only).""" - data = json.loads(Path(path).read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} - try: - return nx.readwrite.json_graph.node_link_graph(data, edges="links") - except TypeError: - return nx.readwrite.json_graph.node_link_graph(data) - - -def _legacy_score_and_pick(G, terms): - """Recreates the pre-refactor flow: combined scoring plus one - `_score_nodes([token])` call per distinct query token, with the legacy - tie-break (`max(tied, key=degree)` over a (-score, label_len, nid)-sorted - list) to derive `best_seed_by_term`. Returns the same triple as the - optimized path so they can be compared for equality.""" - ranked = _score_nodes(G, terms) - norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) - best_seed_by_term: dict[str, str] = {} - for term in norm_terms: - term_scored = _score_nodes(G, [term]) - if not term_scored: - continue - best_score = term_scored[0][0] - tied = [nid for s, nid in term_scored if s == best_score] - best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] - best_seed_by_term[term] = best_nid - seeds = _pick_seeds(ranked, G=G, best_seed_by_term=best_seed_by_term) - return ranked, best_seed_by_term, seeds - - -def _optimized_score_and_pick(G, terms): - qs = _score_query(G, terms, collect_per_term_seeds=True) - seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) - return qs.ranked, qs.best_seed_by_term, seeds - - -def _warm_caches(G, terms): - """Pre-populate the trigram index and IDF cache so the first timed - iteration doesn't pay the amortized build cost on either path. Both - legacy and optimized share these caches via the graph object, so warming - once is fair to both.""" - _get_trigram_index(G) - # Touch the idf cache for the combined terms and every per-token singleton, - # matching exactly the calls the legacy path will make. - _score_nodes(G, terms) - for term in {tok for t in terms for tok in _search_tokens(t)}: - _score_nodes(G, [term]) - - -def _bench(fn, *, repeats: int) -> list[float]: - # One uncounted warm-up — `_warm_caches` already populated caches, but - # this also amortizes any per-call code-path setup unique to `fn`. - fn() - times: list[float] = [] - for _ in range(repeats): - t0 = time.perf_counter() - fn() - times.append(time.perf_counter() - t0) - return times - - -def _verify_equality(G, terms) -> tuple[int, int]: - leg_rank, leg_best, leg_seeds = _legacy_score_and_pick(G, terms) - opt_rank, opt_best, opt_seeds = _optimized_score_and_pick(G, terms) - assert leg_rank == opt_rank, ( - f"ranked diverged for terms={terms}: legacy[:5]={leg_rank[:5]} opt[:5]={opt_rank[:5]}" - ) - assert leg_best == opt_best, ( - f"best_seed_by_term diverged for terms={terms}: legacy={leg_best} opt={opt_best}" - ) - assert leg_seeds == opt_seeds, ( - f"seeds diverged for terms={terms}: legacy={leg_seeds} opt={opt_seeds}" - ) - return len(leg_rank), len(leg_seeds) - - -def _row(label: str, n_nodes: int, n_terms: int, times: list[float], - traversal_count: int, n_ranked: int, n_seeds: int) -> str: - med = statistics.median(times) * 1000 - mn = min(times) * 1000 - return (f"{label:<10} | n={n_nodes:<7} | terms={n_terms:<3} | " - f"median={med:7.2f}ms | min={mn:7.2f}ms | " - f"passes={traversal_count:<3} | ranked={n_ranked:<6} seeds={n_seeds}") - - -def _legacy_traversal_count(terms) -> int: - # 1 combined pass + one per-token singleton pass. - return 1 + len({tok for t in terms for tok in _search_tokens(t)}) - - -def _run_scenario(G, terms, *, repeats: int) -> tuple[float, float]: - _warm_caches(G, terms) - n_ranked, n_seeds = _verify_equality(G, terms) - - legacy_times = _bench(lambda: _legacy_score_and_pick(G, terms), repeats=repeats) - opt_times = _bench(lambda: _optimized_score_and_pick(G, terms), repeats=repeats) - - n_nodes = G.number_of_nodes() - n_terms = len(set(tok for t in terms for tok in _search_tokens(t))) - print(_row("legacy", n_nodes, n_terms, legacy_times, - _legacy_traversal_count(terms), n_ranked, n_seeds)) - print(_row("optimized", n_nodes, n_terms, opt_times, - 1, n_ranked, n_seeds)) - - med_legacy = statistics.median(legacy_times) - med_opt = statistics.median(opt_times) - speedup = med_legacy / med_opt if med_opt > 0 else float("inf") - print(f"speedup | median: {speedup:.2f}x | " - f"min: {min(legacy_times) / min(opt_times):.2f}x") - return med_legacy, med_opt - - -def _resolve_scenarios(args) -> list[list[str]]: - if args.graph: - # Real-graph mode: each --query is a natural-language sentence, tokenized - # using the same helper the production path uses. - sentences = args.query or ["what calls extract"] - scenarios = [_query_terms(s) for s in sentences] - # Dedupe identical token sets (multiple --query args may tokenize the same). - seen: list[list[str]] = [] - for q in scenarios: - if q not in seen: - seen.append(q) - return seen - term_counts = [int(s.strip()) for s in args.term_counts.split(",") if s.strip()] - scenarios: list[list[str]] = [] - for tc in term_counts: - if tc in QUERIES_BY_TERM_COUNT: - scenarios.append(QUERIES_BY_TERM_COUNT[tc]) - elif tc > 0: - scenarios.append([SYLLABLES[i % len(SYLLABLES)] for i in range(tc)]) - return scenarios - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - description="Microbenchmark single-pass query scoring vs legacy per-term rescoring.", - ) - p.add_argument("--nodes", type=int, default=100_000, - help="node count for the synthetic benchmark graph (default: 100000)") - p.add_argument("--seed", type=int, default=20260714, help="RNG seed for the synthetic graph") - p.add_argument("--term-counts", default="3,10", - help="comma-separated list of term counts to benchmark (synthetic mode)") - p.add_argument("--repeats", type=int, default=5, - help="timed iterations per scenario (after one warm-up)") - p.add_argument("--graph", default=None, - help="optional path to a real graphify-out/graph.json; overrides --nodes") - p.add_argument("--query", action="append", default=None, - help="natural-language query sentence (real-graph mode; repeat for multiple)") - args = p.parse_args(argv) - - if args.graph: - print(f"loading real graph: {args.graph} ...", file=sys.stderr) - t0 = time.perf_counter() - G = _load_real_graph(args.graph) - print(f" loaded in {time.perf_counter() - t0:.2f}s", file=sys.stderr) - else: - print(f"building synthetic graph: n={args.nodes} seed={args.seed} ...", - file=sys.stderr) - t0 = time.perf_counter() - G = _build_random_graph(args.nodes, seed=args.seed) - print(f" built in {time.perf_counter() - t0:.2f}s", file=sys.stderr) - - print() - print(f"graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - scenarios = _resolve_scenarios(args) - print(f"scenarios: {len(scenarios)} | repeats per scenario: {args.repeats}") - print("-" * 110) - - summaries = [] - for terms in scenarios: - print() - med_legacy, med_opt = _run_scenario(G, terms, repeats=args.repeats) - summaries.append((terms, med_legacy, med_opt)) - - print() - print("-" * 110) - print("summary:") - for terms, med_legacy, med_opt in summaries: - speedup = med_legacy / med_opt if med_opt > 0 else float("inf") - print(f" terms={len(set(tok for t in terms for tok in _search_tokens(t))):>3} | " - f"median legacy={med_legacy*1000:>8.2f}ms | " - f"median optimized={med_opt*1000:>8.2f}ms | " - f"speedup={speedup:>5.2f}x") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/native_helpers.py b/tests/native_helpers.py new file mode 100644 index 000000000..51ca397db --- /dev/null +++ b/tests/native_helpers.py @@ -0,0 +1,80 @@ +"""Real embedded-Helix fixtures used by Graphify integration tests.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +from typing import Any + +from graphify.helix.model import GraphBuildData +from graphify.helix.persistence import HelixEmbeddedStore +from graphify.helix.state import new_state + + +def make_loaded( + root: Path | None = None, + *, + nodes: list[dict[str, Any]] | None = None, + edges: list[dict[str, Any]] | None = None, + kind: str = "graph", + graph: dict[str, Any] | None = None, + state: dict[str, Any] | None = None, +): + directed = kind in {"digraph", "multidigraph"} + multigraph = kind in {"multigraph", "multidigraph"} + root = Path(tempfile.mkdtemp(prefix="graphify-native-test-")) if root is None else root + payload = { + "directed": directed, + "multigraph": multigraph, + "graph": graph or {}, + "nodes": nodes or [], + "links": edges or [], + } + build = GraphBuildData.from_node_link(payload) + store_path = root / "graph.helix" + with HelixEmbeddedStore(store_path) as store: + store.save_generation(build, state or new_state()) + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load() + + +def triangle(root: Path): + return make_loaded( + root, + nodes=[ + {"id": "a", "label": "A", "source_file": "a.py", "file_type": "code"}, + {"id": "b", "label": "B", "source_file": "b.py", "file_type": "code"}, + {"id": "c", "label": "C", "source_file": "c.py", "file_type": "code"}, + ], + edges=[ + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}, + {"source": "b", "target": "c", "relation": "imports", "confidence": "INFERRED", "weight": 0.8}, + {"source": "c", "target": "a", "relation": "references", "confidence": "AMBIGUOUS", "weight": 0.5}, + ], + ) + + +def graph_from_payload( + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]] | None = None, + *, + kind: str = "graph", + graph: dict[str, Any] | None = None, +): + """Return a real immutable embedded-Helix snapshot for unit tests.""" + return make_loaded(nodes=nodes, edges=edges or [], kind=kind, graph=graph).graph + + +def graph_from_build(build: GraphBuildData): + """Persist transient build data and return a real native snapshot.""" + return loaded_from_build(build).graph + + +def loaded_from_build(build: GraphBuildData): + """Persist transient build data and return its graph plus native query handle.""" + root = Path(tempfile.mkdtemp(prefix="graphify-native-build-test-")) + store_path = root / "graph.helix" + with HelixEmbeddedStore(store_path) as store: + store.save_generation(build, new_state()) + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load() diff --git a/tests/restored/test_affected_cli_restored.py b/tests/restored/test_affected_cli_restored.py new file mode 100644 index 000000000..81700f9f9 --- /dev/null +++ b/tests/restored/test_affected_cli_restored.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import graphify.__main__ as mainmod +from tests.native_helpers import make_loaded + + +def _write_graph(tmp_path): + return make_loaded( + tmp_path, + kind="digraph", + nodes=[ + {"id": "target", "label": "Foo", "source_file": "pkg/foo.py", "source_location": "L1"}, + {"id": "caller", "label": "X()", "source_file": "app.py", "source_location": "L4"}, + {"id": "barrel", "label": "__init__.py", "source_file": "pkg/__init__.py"}, + {"id": "consumer", "label": "app.py", "source_file": "app.py"}, + ], + edges=[ + {"source": "caller", "target": "target", "relation": "calls", "context": "call", "confidence": "EXTRACTED"}, + {"source": "barrel", "target": "target", "relation": "re_exports", "context": "export", "confidence": "EXTRACTED"}, + {"source": "consumer", "target": "target", "relation": "imports", "context": "import", "confidence": "EXTRACTED"}, + ], + ).store_path + + +def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys): + graph_path = _write_graph(tmp_path) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "affected", "Foo", "--graph", str(graph_path)], + ) + + mainmod.main() + + out = capsys.readouterr().out + assert "Affected nodes for Foo" in out + assert "X()" in out + assert "calls" in out + assert "__init__.py" in out + assert "re_exports" in out + assert "app.py" in out + assert "imports" in out + + +def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys): + graph_path = _write_graph(tmp_path) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "affected", "Foo", "--relation", "calls", "--graph", str(graph_path)], + ) + + mainmod.main() + + out = capsys.readouterr().out + assert "Relations: calls" in out + assert "X()" in out + assert "__init__.py" not in out + + +def test_affected_cli_forces_directed_on_undirected_graph(monkeypatch, tmp_path, capsys): + """A graph persisted with directed=false must still recover caller->callee + direction (#1174): affected on the callee returns the caller, not the callee + or nothing. Without forcing directed=True, node_link_graph builds an + undirected Graph, predecessors() collapses, and the reverse traversal breaks. + """ + graph_path = make_loaded( + tmp_path, + kind="graph", + nodes=[ + {"id": "A", "label": "caller_fn", "source_file": "a.py", "source_location": "L1"}, + {"id": "B", "label": "callee_fn", "source_file": "b.py", "source_location": "L2"}, + ], + edges=[{"source": "A", "target": "B", "relation": "calls", "context": "call", "confidence": "EXTRACTED"}], + ).store_path + + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "affected", "B", "--relation", "calls", "--graph", str(graph_path)], + ) + + mainmod.main() + + out = capsys.readouterr().out + # A (the caller) is affected by a change to B (the callee). + assert "caller_fn" in out + assert "calls" in out + # B is the query node, not an affected node, and the result is not empty. + assert "No affected nodes found." not in out + + +def test_resolve_seed_bare_name_matches_callable_label(): + from graphify.affected import resolve_seed + + loaded = make_loaded(nodes=[ + {"id": "a", "label": "classifyProperty()", "source_file": "pkg/entity.py"}, + {"id": "b", "label": "classifyPropertySafe()", "source_file": "app/context.py"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, "classifyProperty", node_query=loaded.query) == "a" + assert resolve_seed(loaded.graph, "classifyPropertySafe", node_query=loaded.query) == "b" + + +def test_resolve_seed_decorated_query_matches_bare_label(): + from graphify.affected import resolve_seed + + loaded = make_loaded(nodes=[ + {"id": "a", "label": "Foo", "source_file": "pkg/foo.py"}, + {"id": "b", "label": "FooBar", "source_file": "pkg/foobar.py"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, "Foo()", node_query=loaded.query) == "a" + + +def test_resolve_seed_matches_unicode_normalized_label(): + import unicodedata + + from graphify.affected import resolve_seed + + loaded = make_loaded(nodes=[{"id": "a", "label": "Auditoría", "source_file": "pkg/auditoria.py"}], kind="digraph") + + assert resolve_seed(loaded.graph, unicodedata.normalize("NFD", "Auditoría"), node_query=loaded.query) == "a" + + +def test_resolve_seed_preserves_distinct_accents(): + from graphify.affected import resolve_seed + + loaded = make_loaded(nodes=[ + {"id": "a", "label": "resume", "source_file": "pkg/resume.py"}, + {"id": "b", "label": "résumé", "source_file": "pkg/resume_accented.py"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, "resume", node_query=loaded.query) == "a" + + +def test_resolve_seed_bare_name_tie_still_returns_none(): + from graphify.affected import resolve_seed + + loaded = make_loaded(nodes=[ + {"id": "a", "label": "dup()", "source_file": "pkg/one.py"}, + {"id": "b", "label": "dup()", "source_file": "pkg/two.py"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, "dup", node_query=loaded.query) is None + + +def test_resolve_seed_source_file_path_prefers_file_level_node(): + from graphify.affected import resolve_seed + + source_file = "app/api/example/route.ts" + loaded = make_loaded(nodes=[ + {"id": "example_route_get", "label": "GET()", "source_file": source_file, "source_location": "L42"}, + {"id": "example_route", "label": "route.ts", "source_file": source_file, "source_location": "L1"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, source_file, node_query=loaded.query) == "example_route" + + +def test_resolve_seed_source_file_trailing_slash_parity(): + """A trailing path separator must not change the match (parity with explain's + _find_node, which tokenizes the path and drops the slash).""" + from graphify.affected import resolve_seed + + source_file = "app/api/example/route.ts" + loaded = make_loaded(nodes=[ + {"id": "get", "label": "GET()", "source_file": source_file, "source_location": "L42"}, + {"id": "file", "label": "route.ts", "source_file": source_file, "source_location": "L1"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, source_file + "/", node_query=loaded.query) == "file" + + +def test_resolve_seed_source_file_ambiguous_no_file_node_returns_none(): + """Several nodes share a source_file but none is the L1 file node and none's + basename matches the path — must not guess; return None.""" + from graphify.affected import resolve_seed + + source_file = "pkg/handlers.py" + loaded = make_loaded(nodes=[ + {"id": "a", "label": "handle_a()", "source_file": source_file, "source_location": "L10"}, + {"id": "b", "label": "handle_b()", "source_file": source_file, "source_location": "L20"}, + ], kind="digraph") + + assert resolve_seed(loaded.graph, source_file, node_query=loaded.query) is None + + +def test_affected_cli_source_file_path_uses_file_level_node(monkeypatch, tmp_path, capsys): + source_file = "app/api/example/route.ts" + graph_path = make_loaded( + tmp_path, + kind="digraph", + nodes=[ + {"id": "example_route_get", "label": "GET()", "source_file": source_file, "source_location": "L42"}, + {"id": "example_route", "label": "route.ts", "source_file": source_file, "source_location": "L1"}, + {"id": "consumer", "label": "consumer.ts", "source_file": "app/consumer.ts", "source_location": "L1"}, + ], + edges=[{"source": "consumer", "target": "example_route", "relation": "imports_from", "context": "import", "confidence": "EXTRACTED"}], + ).store_path + + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "affected", source_file, "--graph", str(graph_path)], + ) + + mainmod.main() + + out = capsys.readouterr().out + assert "Affected nodes for route.ts" in out + assert "consumer.ts" in out + assert "imports_from" in out + assert "No unique node matched" not in out diff --git a/tests/restored/test_affected_member_seed_restored.py b/tests/restored/test_affected_member_seed_restored.py new file mode 100644 index 000000000..9da2ef416 --- /dev/null +++ b/tests/restored/test_affected_member_seed_restored.py @@ -0,0 +1,69 @@ +"""#1669 — affected must reach callers that bind to the class's method +nodes (post-#1634 method-granularity resolution), by seeding the reverse walk +with the root's member nodes (one method/contains hop). method/contains stay out +of the general relation-filtered walk, so no forward noise is added elsewhere. +""" +from __future__ import annotations + +from graphify.affected import affected_nodes +from tests.native_helpers import graph_from_payload + + +def _g(): + return graph_from_payload( + [ + {"id": "proc", "label": "Processor"}, + {"id": "proc_call", "label": ".call()"}, + {"id": "runner", "label": "Runner"}, + {"id": "runner_run", "label": ".run()"}, + ], + [ + {"source": "proc", "target": "proc_call", "relation": "method"}, + {"source": "runner", "target": "runner_run", "relation": "method"}, + {"source": "runner_run", "target": "proc_call", "relation": "calls"}, + ], + kind="digraph", + ) + + +def test_class_affected_reaches_method_bound_caller(): + g = _g() + hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} + assert "runner_run" in hits, "caller of Processor.call must be reachable from Processor" + + +def test_member_method_node_not_reported_as_hit(): + g = _g() + hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} + # the class's own method node is a seed, not an affected node + assert "proc_call" not in hits + + +def test_method_contains_still_excluded_from_general_walk(): + # A node two method-hops away (method of a DIFFERENT class discovered during + # the walk) must NOT be pulled in: only the root's own members are seeded. + g = graph_from_payload( + [{"id": nid, "label": label} for nid, label in + [("a", "A"), ("a_m", ".m()"), ("b", "B"), ("b_m", ".n()")]], + [ + {"source": "a", "target": "a_m", "relation": "method"}, + {"source": "a_m", "target": "b", "relation": "calls"}, + {"source": "b", "target": "b_m", "relation": "method"}, + ], + kind="digraph", + ) + hits = {h.node_id for h in affected_nodes(g, "a", depth=3)} + # We seeded A's members and walk reverse; B and B's method are downstream of A + # (A.m -> B), not reverse-callers of A, so they must not appear. + assert hits == set() or "b_m" not in hits + + +def test_class_level_caller_still_works(): + # A caller bound to the class node itself (not a method) is unaffected. + g = graph_from_payload( + [{"id": "svc", "label": "Svc"}, {"id": "caller", "label": ".use()"}], + [{"source": "caller", "target": "svc", "relation": "references"}], + kind="digraph", + ) + hits = {h.node_id for h in affected_nodes(g, "svc", depth=2)} + assert "caller" in hits diff --git a/tests/restored/test_analyze_restored.py b/tests/restored/test_analyze_restored.py new file mode 100644 index 000000000..46a122047 --- /dev/null +++ b/tests/restored/test_analyze_restored.py @@ -0,0 +1,640 @@ +"""Tests for analyze.py.""" +import json +import pytest +from pathlib import Path +from graphify.build import build_from_extraction +from graphify.cluster import cluster +from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff, _surprise_score, _file_category, _is_json_key_node, find_import_cycles, suggest_questions +from graphify.extract import _make_id +from graphify.helix.access import first_edge_attributes +from tests.native_helpers import graph_from_build, graph_from_payload + +FIXTURES = Path(__file__).parents[1] / "fixtures" + + +def make_graph(): + return graph_from_build(build_from_extraction(json.loads((FIXTURES / "extraction.json").read_text()))) + + +def _edge(graph, source, target): + return first_edge_attributes(graph, source, target) + + +def _degrees(graph): + return {row.node_id: int(row.degree) for row in graph.degrees()} + + +def test_god_nodes_returns_list(): + G = make_graph() + result = god_nodes(G, top_n=3) + assert isinstance(result, list) + assert len(result) <= 3 + + +def test_god_nodes_sorted_by_degree(): + G = make_graph() + result = god_nodes(G, top_n=30) + degrees = [r["degree"] for r in result] + assert degrees == sorted(degrees, reverse=True) + + +def test_god_nodes_have_required_keys(): + G = make_graph() + result = god_nodes(G, top_n=1) + assert "id" in result[0] + assert "label" in result[0] + assert "degree" in result[0] + + +def test_surprising_connections_cross_source_multi_file(): + """Multi-file graph: should find cross-file edges between real entities.""" + G = make_graph() + communities = cluster(G) + surprises = surprising_connections(G, communities) + assert len(surprises) > 0 + for s in surprises: + assert s["source_files"][0] != s["source_files"][1] + + +def test_surprising_connections_excludes_concept_nodes(): + """Concept nodes (empty source_file) must not appear in surprises.""" + extraction = json.loads((FIXTURES / "extraction.json").read_text()) + extraction["nodes"].append({"id": "concept_x", "label": "Abstract Concept", "file_type": "document", "source_file": ""}) + extraction["edges"].append({"source": "n_transformer", "target": "concept_x", "relation": "relates_to", "confidence": "INFERRED", "source_file": "", "weight": 0.5}) + G = graph_from_build(build_from_extraction(extraction)) + communities = cluster(G) + surprises = surprising_connections(G, communities) + labels = [s["source"] for s in surprises] + [s["target"] for s in surprises] + assert "Abstract Concept" not in labels + + +def test_surprising_connections_single_file_uses_community_bridges(): + """Single-file graph: should return cross-community edges, not empty list.""" + nodes = [ + {"id": f"a{i}", "label": f"A{i}", "file_type": "code", "source_file": "single.py", "source_location": f"L{i}"} + for i in range(5) + ] + [ + {"id": f"b{i}", "label": f"B{i}", "file_type": "code", "source_file": "single.py", "source_location": f"L{i+10}"} + for i in range(5) + ] + edges = [ + {"source": f"a{i}", "target": f"a{i+1}", "relation": "calls", "confidence": "EXTRACTED", "source_file": "single.py", "weight": 1.0} + for i in range(4) + ] + [ + {"source": f"b{i}", "target": f"b{i+1}", "relation": "calls", "confidence": "EXTRACTED", "source_file": "single.py", "weight": 1.0} + for i in range(4) + ] + [{"source": "a4", "target": "b0", "relation": "references", "confidence": "INFERRED", "source_file": "single.py", "weight": 0.5}] + G = graph_from_payload(nodes, edges) + + communities = cluster(G) + surprises = surprising_connections(G, communities) + # Should find at least the bridge edge + assert len(surprises) > 0 + + +def test_surprising_connections_ambiguous_scores_higher_than_extracted(): + """AMBIGUOUS edge should score higher than an otherwise identical EXTRACTED edge.""" + nodes = [ + ("a", "Alpha", "repo1/model.py"), + ("b", "Beta", "repo2/train.py"), + ("c", "Gamma", "repo1/data.py"), + ("d", "Delta", "repo2/eval.py"), + ] + G = graph_from_payload( + [{"id": nid, "label": label, "source_file": src, "file_type": "code"} for nid, label, src in nodes], + [{"source": "a", "target": "b", "relation": "calls", "confidence": "AMBIGUOUS", "weight": 1.0, "source_file": "repo1/model.py"}, {"source": "c", "target": "d", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "repo1/data.py"}], + ) + communities = {0: ["a", "c"], 1: ["b", "d"]} + nc = {"a": 0, "c": 0, "b": 1, "d": 1} + score_amb, _ = _surprise_score(G, "a", "b", _edge(G, "a", "b"), nc, "repo1/model.py", "repo2/train.py") + score_ext, _ = _surprise_score(G, "c", "d", _edge(G, "c", "d"), nc, "repo1/data.py", "repo2/eval.py") + assert score_amb > score_ext + + +def test_surprise_score_accepts_precomputed_degrees(): + nodes = [ + ("hub", "Hub", "repo1/hub.py"), + ("leaf", "Leaf", "repo2/leaf.py"), + ("n1", "N1", "repo1/n1.py"), + ("n2", "N2", "repo1/n2.py"), + ("n3", "N3", "repo1/n3.py"), + ("n4", "N4", "repo1/n4.py"), + ] + G = graph_from_payload( + [{"id": nid, "label": label, "source_file": src, "file_type": "code"} for nid, label, src in nodes], + [{"source": "hub", "target": node, "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0} for node in ("leaf", "n1", "n2", "n3", "n4")], + ) + + nc = {"hub": 0, "leaf": 1} + edge = _edge(G, "hub", "leaf") + args = (G, "hub", "leaf", edge, nc, "repo1/hub.py", "repo2/leaf.py") + + assert _surprise_score(*args) == _surprise_score(*args, _degrees(G)) + + +def test_surprising_connections_cross_type_scores_higher(): + """Code↔paper edge should score higher than code↔code edge.""" + nodes = [ + ("a", "Transformer", "code/model.py"), + ("b", "FlashAttn", "papers/flash.pdf"), + ("c", "Trainer", "code/train.py"), + ("d", "Dataset", "code/data.py"), + ] + G = graph_from_payload( + [{"id": nid, "label": label, "source_file": src, "file_type": "code"} for nid, label, src in nodes], + [{"source": "a", "target": "b", "relation": "references", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "code/model.py"}, {"source": "c", "target": "d", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "code/train.py"}], + ) + nc = {"a": 0, "b": 1, "c": 0, "d": 0} + score_cross, reasons_cross = _surprise_score(G, "a", "b", _edge(G, "a", "b"), nc, "code/model.py", "papers/flash.pdf") + score_same, _ = _surprise_score(G, "c", "d", _edge(G, "c", "d"), nc, "code/train.py", "code/data.py") + assert score_cross > score_same + assert any("code" in r and "paper" in r for r in reasons_cross) + + +def _make_cross_lang_graph(relation="calls", confidence="INFERRED"): + """Helper: Python node in backend/, TypeScript node in frontend/, different communities.""" + return graph_from_payload( + [ + {"id": "py_auth", "label": "AuthError", "source_file": "backend/auth.py", "file_type": "code"}, + {"id": "ts_member", "label": "Member", "source_file": "frontend/types.ts", "file_type": "code"}, + {"id": "py_a", "label": "ServiceA", "source_file": "backend/service.py", "file_type": "code"}, + {"id": "py_b", "label": "ServiceB", "source_file": "backend/utils.py", "file_type": "code"}, + ], + [ + {"source": "py_auth", "target": "ts_member", "relation": relation, "confidence": confidence, "weight": 0.85, "source_file": "backend/auth.py"}, + {"source": "py_a", "target": "py_b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "backend/service.py"}, + ], + ) + + +def test_cross_language_inferred_calls_suppressed(): + """Cross-language INFERRED calls edge should score lower than same-language EXTRACTED.""" + G = _make_cross_lang_graph("calls", "INFERRED") + nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} + score_cross, _ = _surprise_score(G, "py_auth", "ts_member", + _edge(G, "py_auth", "ts_member"), nc, + "backend/auth.py", "frontend/types.ts") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "backend/service.py", "backend/utils.py") + assert score_cross <= score_same + + +def test_cross_language_inferred_uses_suppressed(): + """Cross-language INFERRED uses edge (the exact rsl-siege-manager false positive) should be suppressed.""" + G = _make_cross_lang_graph("uses", "INFERRED") + nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} + score_cross, _ = _surprise_score(G, "py_auth", "ts_member", + _edge(G, "py_auth", "ts_member"), nc, + "backend/auth.py", "frontend/types.ts") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "backend/service.py", "backend/utils.py") + assert score_cross <= score_same + + +def test_cross_language_semantically_similar_not_suppressed(): + """`semantically_similar_to` across languages is a genuine insight — must not be suppressed.""" + G = _make_cross_lang_graph("semantically_similar_to", "INFERRED") + nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} + score_sem, _ = _surprise_score(G, "py_auth", "ts_member", + _edge(G, "py_auth", "ts_member"), nc, + "backend/auth.py", "frontend/types.ts") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "backend/service.py", "backend/utils.py") + assert score_sem > score_same + + +def test_same_language_inferred_calls_not_suppressed(): + """INFERRED calls within the same language family must not be affected.""" + G = graph_from_payload( + [{"id": f"py_{name}", "label": f"Module{name.upper()}", "source_file": f"src/{name}.py", "file_type": "code"} for name in "abcd"], + [{"source": "py_a", "target": "py_b", "relation": "calls", "confidence": "INFERRED", "weight": 0.8, "source_file": "src/a.py"}, {"source": "py_c", "target": "py_d", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "src/c.py"}], + ) + nc = {"py_a": 0, "py_b": 1, "py_c": 0, "py_d": 1} + score_inf, _ = _surprise_score(G, "py_a", "py_b", _edge(G, "py_a", "py_b"), nc, + "src/a.py", "src/b.py") + score_ext, _ = _surprise_score(G, "py_c", "py_d", _edge(G, "py_c", "py_d"), nc, + "src/c.py", "src/d.py") + assert score_inf > score_ext + + +def test_cross_language_extracted_calls_not_suppressed(): + """EXTRACTED cross-language edges are real structural facts — must not be penalised.""" + G = _make_cross_lang_graph("calls", "EXTRACTED") + nc = {"py_auth": 0, "ts_member": 1} + score, _ = _surprise_score(G, "py_auth", "ts_member", + _edge(G, "py_auth", "ts_member"), nc, + "backend/auth.py", "frontend/types.ts") + assert score >= 1 + + +def test_surprising_connections_have_why_field(): + G = make_graph() + communities = cluster(G) + for s in surprising_connections(G, communities): + assert "why" in s + assert isinstance(s["why"], str) + assert len(s["why"]) > 0 + + +def test_file_category(): + assert _file_category("model.py") == "code" + assert _file_category("flash.pdf") == "paper" + assert _file_category("diagram.png") == "image" + assert _file_category("notes.md") == "doc" + # Languages added in later releases — would misclassify as "doc" without detect.py import + assert _file_category("app.swift") == "code" + assert _file_category("plugin.lua") == "code" + assert _file_category("build.zig") == "code" + assert _file_category("deploy.ps1") == "code" + assert _file_category("server.ex") == "code" + assert _file_category("component.jsx") == "code" + assert _file_category("analysis.jl") == "code" + assert _file_category("view.m") == "code" + + +def test_is_concept_node_empty_source(): + G = graph_from_payload([{"id": "c1", "source_file": ""}]) + assert _is_concept_node(G, "c1") is True + + +def test_is_concept_node_real_file(): + G = graph_from_payload([{"id": "n1", "source_file": "model.py"}]) + assert _is_concept_node(G, "n1") is False + + +def test_surprising_connections_have_required_keys(): + G = make_graph() + communities = cluster(G) + for s in surprising_connections(G, communities): + assert "source" in s + assert "target" in s + assert "source_files" in s + assert "confidence" in s + + +# --- graph_diff tests --- + +def _make_simple_graph(nodes, edges): + """Build a small immutable native graph from node/edge specs.""" + return graph_from_payload( + [{"id": node_id, "label": label, "source_file": "test.py"} for node_id, label in nodes], + [{"source": src, "target": tgt, "relation": rel, "confidence": conf} for src, tgt, rel, conf in edges], + ) + + +def test_graph_diff_new_nodes(): + G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) + G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) + diff = graph_diff(G_old, G_new) + assert len(diff["new_nodes"]) == 1 + assert diff["new_nodes"][0]["id"] == "n3" + assert diff["new_nodes"][0]["label"] == "Gamma" + assert diff["removed_nodes"] == [] + assert "1 new node" in diff["summary"] + + +def test_graph_diff_removed_nodes(): + G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) + G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) + diff = graph_diff(G_old, G_new) + assert diff["new_nodes"] == [] + assert len(diff["removed_nodes"]) == 1 + assert diff["removed_nodes"][0]["id"] == "n3" + assert "removed" in diff["summary"] + + +def test_graph_diff_new_edges(): + nodes = [("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")] + G_old = _make_simple_graph(nodes, [("n1", "n2", "calls", "EXTRACTED")]) + G_new = _make_simple_graph( + nodes, + [("n1", "n2", "calls", "EXTRACTED"), ("n2", "n3", "uses", "INFERRED")], + ) + diff = graph_diff(G_old, G_new) + assert len(diff["new_edges"]) == 1 + new_edge = diff["new_edges"][0] + assert new_edge["relation"] == "uses" + assert new_edge["confidence"] == "INFERRED" + assert diff["removed_edges"] == [] + assert "new edge" in diff["summary"] + + +def test_graph_diff_empty_diff(): + nodes = [("n1", "Alpha"), ("n2", "Beta")] + edges = [("n1", "n2", "calls", "EXTRACTED")] + G_old = _make_simple_graph(nodes, edges) + G_new = _make_simple_graph(nodes, edges) + diff = graph_diff(G_old, G_new) + assert diff["new_nodes"] == [] + assert diff["removed_nodes"] == [] + assert diff["new_edges"] == [] + assert diff["removed_edges"] == [] + assert diff["summary"] == "no changes" + + +# --- code↔doc INFERRED suppression tests --- + +def _make_code_doc_graph(relation="calls", confidence="INFERRED"): + return graph_from_payload( + [ + {"id": "py_fn", "label": "ProcessData", "source_file": "src/processor.py", "file_type": "code"}, + {"id": "md_doc", "label": "README Section", "source_file": "docs/readme.md", "file_type": "document"}, + {"id": "py_a", "label": "ServiceA", "source_file": "src/service.py", "file_type": "code"}, + {"id": "py_b", "label": "ServiceB", "source_file": "src/utils.py", "file_type": "code"}, + ], + [ + {"source": "py_fn", "target": "md_doc", "relation": relation, "confidence": confidence, "weight": 0.85, "source_file": "src/processor.py"}, + {"source": "py_a", "target": "py_b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "src/service.py"}, + ], + ) + + +def test_code_doc_inferred_calls_suppressed(): + """Code→doc INFERRED calls edge should score lower than same-language EXTRACTED.""" + G = _make_code_doc_graph("calls", "INFERRED") + nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} + score_noise, _ = _surprise_score(G, "py_fn", "md_doc", + _edge(G, "py_fn", "md_doc"), nc, + "src/processor.py", "docs/readme.md") + score_real, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "src/service.py", "src/utils.py") + assert score_noise <= score_real + + +def test_code_doc_inferred_uses_suppressed(): + """Code→doc INFERRED uses edge should score lower than same-language EXTRACTED.""" + G = _make_code_doc_graph("uses", "INFERRED") + nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} + score_noise, _ = _surprise_score(G, "py_fn", "md_doc", + _edge(G, "py_fn", "md_doc"), nc, + "src/processor.py", "docs/readme.md") + score_real, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "src/service.py", "src/utils.py") + assert score_noise <= score_real + + +def test_code_doc_extracted_calls_not_suppressed(): + """EXTRACTED code↔doc edges are real facts — must not be penalised.""" + G = _make_code_doc_graph("calls", "EXTRACTED") + nc = {"py_fn": 0, "md_doc": 1} + score, _ = _surprise_score(G, "py_fn", "md_doc", + _edge(G, "py_fn", "md_doc"), nc, + "src/processor.py", "docs/readme.md") + assert score >= 1 + + +def test_code_doc_inferred_semantically_similar_not_suppressed(): + """`semantically_similar_to` across code↔doc is explicit LLM insight — must not be suppressed.""" + G = _make_code_doc_graph("semantically_similar_to", "INFERRED") + nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} + score_sem, _ = _surprise_score(G, "py_fn", "md_doc", + _edge(G, "py_fn", "md_doc"), nc, + "src/processor.py", "docs/readme.md") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "src/service.py", "src/utils.py") + assert score_sem > score_same + + +def test_code_unknown_extension_inferred_calls_suppressed(): + """_file_category falls back to 'doc' for unknown extensions, so INFERRED + calls/uses to unknown-extension files are suppressed the same as code↔doc.""" + assert _file_category("vendor/random.xyz") == "doc" + G = graph_from_payload( + [{"id": "py_fn", "label": "Handler", "source_file": "src/handler.py", "file_type": "code"}, {"id": "unk", "label": "Handler", "source_file": "vendor/unknown.xyz", "file_type": "document"}, {"id": "py_a", "label": "A", "source_file": "src/a.py", "file_type": "code"}, {"id": "py_b", "label": "B", "source_file": "src/b.py", "file_type": "code"}], + [{"source": "py_fn", "target": "unk", "relation": "calls", "confidence": "INFERRED", "weight": 0.8, "source_file": "src/handler.py"}, {"source": "py_a", "target": "py_b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "src/a.py"}], + ) + nc = {"py_fn": 0, "unk": 1, "py_a": 0, "py_b": 0} + score_unk, _ = _surprise_score(G, "py_fn", "unk", + _edge(G, "py_fn", "unk"), nc, + "src/handler.py", "vendor/unknown.xyz") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "src/a.py", "src/b.py") + assert score_unk <= score_same + + +def test_code_paper_inferred_calls_not_suppressed(): + """Code↔paper INFERRED calls should still surface — it is a meaningful link.""" + G = graph_from_payload( + [{"id": "py_model", "label": "Transformer", "source_file": "src/model.py", "file_type": "code"}, {"id": "pdf_paper", "label": "Attention Is All You Need", "source_file": "papers/vaswani.pdf", "file_type": "paper"}, {"id": "py_a", "label": "ServiceA", "source_file": "src/service.py", "file_type": "code"}, {"id": "py_b", "label": "ServiceB", "source_file": "src/utils.py", "file_type": "code"}], + [{"source": "py_model", "target": "pdf_paper", "relation": "calls", "confidence": "INFERRED", "weight": 0.8, "source_file": "src/model.py"}, {"source": "py_a", "target": "py_b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0, "source_file": "src/service.py"}], + ) + nc = {"py_model": 0, "pdf_paper": 1, "py_a": 0, "py_b": 1} + score_cross, _ = _surprise_score(G, "py_model", "pdf_paper", + _edge(G, "py_model", "pdf_paper"), nc, + "src/model.py", "papers/vaswani.pdf") + score_same, _ = _surprise_score(G, "py_a", "py_b", + _edge(G, "py_a", "py_b"), nc, + "src/service.py", "src/utils.py") + assert score_cross > score_same + + +# --- JSON key node filtering tests --- + +def test_is_json_key_node_noise_label(): + G = graph_from_payload([{"id": "j1", "label": "name", "source_file": "schema.json"}]) + assert _is_json_key_node(G, "j1") is True + + +def test_is_json_key_node_non_json_file(): + G = graph_from_payload([{"id": "n1", "label": "name", "source_file": "model.py"}]) + assert _is_json_key_node(G, "n1") is False + + +# --- npm dep-block key god-node filtering tests --- + +@pytest.mark.parametrize("dep_key", [ + "dependencies", + "devDependencies", + "peerDependencies", + "optionalDependencies", + "bundledDependencies", +]) +def test_god_nodes_excludes_npm_dep_block_keys(dep_key: str) -> None: + """npm package.json dep-block keys must be filtered from god_nodes output. + + Constructs a small graph with one node labelled with an npm dep-block key + (sourced from a .json file) and one real-domain node that has high degree. + Asserts that god_nodes() excludes the dep-block node even when it has the + highest degree, while the real-domain node is included. + + Args: + dep_key: The npm dependency-block key label to test (parametrized). + """ + G = graph_from_payload( + [{"id": "real_node", "label": "AuthService", "source_file": "src/auth.py", "file_type": "code", "source_location": "L1"}, {"id": "dep_node", "label": dep_key, "source_file": "frontend/package.json", "file_type": "code", "source_location": "L1"}] + + [{"id": f"pkg_{i}", "label": f"package-{i}", "source_file": "frontend/package.json", "file_type": "code", "source_location": f"L{i + 2}"} for i in range(20)], + [{"source": "dep_node", "target": f"pkg_{i}", "relation": "contains", "confidence": "EXTRACTED", "source_file": "frontend/package.json", "weight": 1.0} for i in range(20)] + + [{"source": "real_node", "target": "dep_node", "relation": "imports", "confidence": "EXTRACTED", "source_file": "src/auth.py", "weight": 1.0}], + ) + + result = god_nodes(G, top_n=30) + result_ids = [r["id"] for r in result] + + assert "dep_node" not in result_ids, ( + f"god_nodes() should filter npm dep-block key '{dep_key}' " + f"but it appeared in the result: {result}" + ) + assert "real_node" in result_ids, ( + f"god_nodes() should include real-domain node 'AuthService' " + f"but it was absent: {result}" + ) + + +def test_is_json_key_node_real_label(): + G = graph_from_payload([{"id": "j2", "label": "UserProfile", "source_file": "schema.json"}]) + assert _is_json_key_node(G, "j2") is False + + +def test_god_nodes_excludes_json_noise(): + """god_nodes must not return generic JSON key nodes like 'name' or 'id'.""" + G = graph_from_payload( + [{"id": "real", "label": "AuthService", "source_file": "src/auth.py"}, {"id": "json_name", "label": "name", "source_file": "schema.json"}] + + [{"id": f"peer{i}", "label": f"Peer{i}", "source_file": f"src/peer{i}.py"} for i in range(8)], + [{"source": hub, "target": f"peer{i}"} for i in range(8) for hub in ("json_name", "real")], + ) + result = god_nodes(G, top_n=10) + labels = [r["label"] for r in result] + assert "name" not in labels + assert "AuthService" in labels + + +def test_god_nodes_filter_is_case_insensitive(): + """JSON-key filter must match regardless of label casing.""" + variants = ("Start", "START", "Name", "ID") + G = graph_from_payload( + [{"id": "real", "label": "RealAbstraction", "source_file": "libs/real.py"}] + + [{"id": f"peer{i}", "label": f"P{i}", "source_file": f"src/p{i}.py"} for i in range(3)] + + [{"id": f"json_{index}_{variant.lower()}", "label": variant, "source_file": "testhelpers/data.json"} for index, variant in enumerate(variants)] + + [{"id": f"json_{index}_{variant.lower()}_t{i}", "label": f"X{i}", "source_file": "testhelpers/data.json"} for index, variant in enumerate(variants) for i in range(15)], + [{"source": "real", "target": f"peer{i}"} for i in range(3)] + + [{"source": f"json_{index}_{variant.lower()}_t{i}", "target": f"json_{index}_{variant.lower()}"} for index, variant in enumerate(variants) for i in range(15)], + ) + result = god_nodes(G, top_n=10) + labels = [r["label"] for r in result] + for variant in ("Start", "START", "Name", "ID"): + assert variant not in labels, f"`{variant}` should be filtered as JSON-key noise" + + +def test_suggest_questions_excludes_rationale_nodes_from_isolated_count(): + G = graph_from_payload([{"id": "service", "label": "Service", "file_type": "code", "source_file": "service.py"}, {"id": "reason", "label": "Explains service", "file_type": "rationale", "source_file": "service.py"}]) + + questions = suggest_questions(G, communities={}, community_labels={}, top_n=10) + isolated = next(question for question in questions if question["type"] == "isolated_nodes") + + assert isolated["why"].startswith("1 weakly-connected node") + assert "`Service`" in isolated["question"] + assert "Explains service" not in isolated["question"] + + +# ── find_import_cycles tests ────────────────────────────────────────────────── + + +def _make_file_node(path: str) -> tuple[str, dict]: + """Create a graph node resembling real graphify schema.""" + nid = _make_id(path) + return nid, {"label": Path(path).name, "source_file": path, "file_type": "code"} + + +def _cycle_payload(): + a_id, a = _make_file_node("src/a.ts") + b_id, b = _make_file_node("src/b.ts") + c_id, c = _make_file_node("src/c.ts") + d_id, d = _make_file_node("src/d.ts") + ext_id = _make_id("react") + + nodes = [{"id": a_id, **a}, {"id": b_id, **b}, {"id": c_id, **c}, {"id": d_id, **d}, {"id": ext_id, "label": "react", "file_type": "code"}] + edges = [ + {"source": a_id, "target": b_id, "relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}, + {"source": b_id, "target": a_id, "relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}, + {"source": b_id, "target": c_id, "relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}, + {"source": c_id, "target": d_id, "relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}, + {"source": d_id, "target": b_id, "relation": "imports_from", "source_file": "src/d.ts", "confidence": "EXTRACTED"}, + {"source": c_id, "target": c_id, "relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}, + {"source": a_id, "target": ext_id, "relation": "calls", "source_file": "src/a.ts", "confidence": "INFERRED"}, + {"source": a_id, "target": ext_id, "relation": "contains", "source_file": "src/a.ts", "confidence": "EXTRACTED"}, + {"source": a_id, "target": ext_id, "relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}, + ] + return nodes, edges + + +def _make_cycle_graph_directed(): + nodes, edges = _cycle_payload() + return graph_from_payload(nodes, edges, kind="multidigraph") + + +def test_find_import_cycles_returns_structured_records(): + G = _make_cycle_graph_directed() + cycles = find_import_cycles(G) + assert isinstance(cycles, list) + assert cycles + assert isinstance(cycles[0], dict) + assert "cycle" in cycles[0] + assert "length" in cycles[0] + assert "why" in cycles[0] + + +def test_find_import_cycles_detects_2_and_3_cycles(): + G = _make_cycle_graph_directed() + cycles = find_import_cycles(G) + cycle_sets = [set(c["cycle"]) for c in cycles] + assert any({"src/a.ts", "src/b.ts"}.issubset(s) for s in cycle_sets) + assert any({"src/b.ts", "src/c.ts", "src/d.ts"}.issubset(s) for s in cycle_sets) + + +def test_find_import_cycles_includes_self_loop_cycle(): + G = _make_cycle_graph_directed() + cycles = find_import_cycles(G) + assert any(c["cycle"] == ["src/c.ts"] and c["length"] == 1 for c in cycles) + + +def test_find_import_cycles_respects_max_cycle_length(): + G = _make_cycle_graph_directed() + cycles = find_import_cycles(G, max_cycle_length=2) + assert all(c["length"] <= 2 for c in cycles) + + +def test_find_import_cycles_skips_nodes_without_source_file(): + G = _make_cycle_graph_directed() + cycles = find_import_cycles(G) + flat = " ".join(" ".join(c["cycle"]) for c in cycles) + assert "react" not in flat + + +def test_find_import_cycles_handles_undirected_graph_input(): + nodes, edges = _cycle_payload() + Gu = graph_from_payload(nodes, edges, kind="multigraph") + cycles = find_import_cycles(Gu) + assert cycles # should still resolve orientation via edge.source_file + + +def test_find_import_cycles_ignores_non_import_relations(): + a_id, a = _make_file_node("src/a.ts") + b_id, b = _make_file_node("src/b.ts") + G = graph_from_payload( + [{"id": a_id, **a}, {"id": b_id, **b}], + [{"source": a_id, "target": b_id, "relation": "calls", "source_file": "src/a.ts", "confidence": "INFERRED"}, {"source": b_id, "target": a_id, "relation": "contains", "source_file": "src/b.ts", "confidence": "EXTRACTED"}], + kind="digraph", + ) + assert find_import_cycles(G) == [] + + +def test_find_import_cycles_empty_graph(): + assert find_import_cycles(graph_from_payload([], kind="digraph")) == [] + + +def test_find_import_cycles_no_cycles(): + x_id, x = _make_file_node("x.ts") + y_id, y = _make_file_node("y.ts") + G = graph_from_payload( + [{"id": x_id, **x}, {"id": y_id, **y}], + [{"source": x_id, "target": y_id, "relation": "imports_from", "source_file": "x.ts", "confidence": "EXTRACTED"}], + kind="digraph", + ) + assert find_import_cycles(G) == [] diff --git a/tests/restored/test_atomic_writes_restored.py b/tests/restored/test_atomic_writes_restored.py new file mode 100644 index 000000000..f605e8063 --- /dev/null +++ b/tests/restored/test_atomic_writes_restored.py @@ -0,0 +1,120 @@ +"""Tests for generic atomic text/JSON writes and extraction manifests. + +A crash, kill, or disk-full mid-write must not leave a truncated/corrupt file +that a later load chokes on. `write_text_atomic` writes a temp file in the same +directory then `os.replace`s it into place; on failure the original is untouched. +""" +import json +import os + +import pytest + +from graphify.paths import write_text_atomic + + +def test_write_text_atomic_writes_and_leaves_no_tmp(tmp_path): + p = tmp_path / "out" / "payload.json" # parent doesn't exist yet + write_text_atomic(p, '{"a": 1}') + assert json.loads(p.read_text()) == {"a": 1} + # No leftover temp file in the target directory. + assert [x.name for x in p.parent.iterdir()] == ["payload.json"] + + +def test_write_text_atomic_preserves_existing_on_failure(tmp_path, monkeypatch): + p = tmp_path / "payload.json" + p.write_text("original", encoding="utf-8") + + def boom(src, dst): + raise OSError("simulated disk full") + + monkeypatch.setattr(os, "replace", boom) + with pytest.raises(OSError): + write_text_atomic(p, "content-that-must-not-land") + + # The original file is intact and the temp file was cleaned up. + assert p.read_text() == "original" + assert sorted(x.name for x in tmp_path.iterdir()) == ["payload.json"] + + +def test_write_text_atomic_preserves_existing_mode(tmp_path): + # An atomic replace must not tighten a 0644 file to mkstemp's 0600 default. + p = tmp_path / "payload.json" + p.write_text("{}", encoding="utf-8") + os.chmod(p, 0o644) + write_text_atomic(p, '{"x": 1}') + assert (os.stat(p).st_mode & 0o777) == 0o644 + + +def test_write_text_atomic_new_file_respects_umask(tmp_path): + # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's + # 0600 — otherwise every fresh atomic output would be owner-only. + p = tmp_path / "new.json" + write_text_atomic(p, "{}") + umask = os.umask(0) + os.umask(umask) + assert (os.stat(p).st_mode & 0o777) == (0o666 & ~umask) + + +def test_write_text_atomic_writes_through_symlink(tmp_path): + # Shared-output setups may symlink an output to shared storage; the atomic write + # must update the target and keep the link, not replace it with a real file. + target = tmp_path / "real.json" + target.write_text("old", encoding="utf-8") + link = tmp_path / "link.json" + link.symlink_to(target) + write_text_atomic(link, "new") + assert link.is_symlink() + assert target.read_text() == "new" + + +def test_write_json_atomic_roundtrip(tmp_path): + from graphify.paths import write_json_atomic + + p = tmp_path / "g.json" + write_json_atomic(p, {"nodes": [1, 2], "x": "é"}, indent=2) + assert json.loads(p.read_text()) == {"nodes": [1, 2], "x": "é"} + assert not any(name.name.endswith(".tmp") for name in tmp_path.iterdir()) + + +def test_save_manifest_writes_atomically(tmp_path): + from graphify.detect import save_manifest + + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + mpath = tmp_path / "graphify-out" / "manifest.json" + save_manifest({"code": [str(tmp_path / "a.py")]}, manifest_path=str(mpath), + kind="both", root=tmp_path) + assert json.loads(mpath.read_text()) # non-empty, valid JSON + assert not any(x.name.endswith(".tmp") for x in mpath.parent.iterdir()) + + +def test_write_text_atomic_windows_permission_fallback(tmp_path, monkeypatch): + """On Windows os.replace raises PermissionError when the destination is + briefly locked (antivirus, an open reader); the copy-then-delete fallback + must still land the new content and leave no temp file.""" + p = tmp_path / "payload.json" + p.write_text("original", encoding="utf-8") + + real_replace = os.replace + calls = {"n": 0} + + def flaky_replace(src, dst): + calls["n"] += 1 + raise PermissionError("simulated WinError 5") + + monkeypatch.setattr(os, "replace", flaky_replace) + write_text_atomic(p, "new-content") + + assert calls["n"] == 1 # the fallback path was actually exercised + assert p.read_text() == "new-content" + assert sorted(x.name for x in tmp_path.iterdir()) == ["payload.json"] + + +def test_write_json_atomic_ensure_ascii_false_preserves_utf8(tmp_path): + from graphify.paths import write_json_atomic + + p = tmp_path / "g.json" + write_json_atomic(p, {"label": "Wörker 数据"}, ensure_ascii=False) + raw = p.read_text(encoding="utf-8") + assert "Wörker 数据" in raw # raw UTF-8, not \\uXXXX escapes + assert "\\u" not in raw + assert json.loads(raw) == {"label": "Wörker 数据"} diff --git a/tests/restored/test_benchmark_restored.py b/tests/restored/test_benchmark_restored.py new file mode 100644 index 000000000..2b29f8cc8 --- /dev/null +++ b/tests/restored/test_benchmark_restored.py @@ -0,0 +1,169 @@ +"""Tests for graphify/benchmark.py.""" +from __future__ import annotations +from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS, _safe, _hr +from graphify.helix.model import edge_attributes, graphify_attributes +from tests.native_helpers import graph_from_payload, make_loaded + + +def _make_graph(): + return graph_from_payload( + [ + {"id": "n1", "label": "authentication", "source_file": "auth.py", "source_location": "L1", "community": 0}, + {"id": "n2", "label": "api_handler", "source_file": "api.py", "source_location": "L5", "community": 0}, + {"id": "n3", "label": "main_entry", "source_file": "main.py", "source_location": "L1", "community": 1}, + {"id": "n4", "label": "error_handler", "source_file": "errors.py", "source_location": "L1", "community": 1}, + {"id": "n5", "label": "database_layer", "source_file": "db.py", "source_location": "L1", "community": 2}, + ], + [ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "INFERRED"}, + {"source": "n2", "target": "n3", "relation": "imports", "confidence": "EXTRACTED"}, + {"source": "n3", "target": "n4", "relation": "uses", "confidence": "EXTRACTED"}, + {"source": "n5", "target": "n2", "relation": "provides", "confidence": "EXTRACTED"}, + ], + ) + + +def _write_graph(G, path): + loaded = make_loaded( + path.parent, + nodes=[{"id": node.id, **graphify_attributes(node.attributes)} for node in G.nodes()], + edges=[{"source": edge.source, "target": edge.target, **edge_attributes(edge)} for edge in G.edges()], + kind="digraph" if G.directed else "graph", + ) + return loaded.store_path + + +# --- _query_subgraph_tokens --- + +def test_query_returns_positive_for_matching_question(): + G = _make_graph() + tokens = _query_subgraph_tokens(G, "how does authentication work") + assert tokens > 0 + +def test_query_returns_zero_for_no_match(): + G = _make_graph() + tokens = _query_subgraph_tokens(G, "xyzzy plugh zorkmid") + assert tokens == 0 + +def test_query_bfs_expands_neighbors(): + G = _make_graph() + # "authentication" matches n1, BFS depth=3 should reach n2, n3, n4 + tokens_deep = _query_subgraph_tokens(G, "authentication", depth=3) + tokens_shallow = _query_subgraph_tokens(G, "authentication", depth=1) + assert tokens_deep >= tokens_shallow + + +def test_query_keeps_short_non_english_terms(): + G = graph_from_payload([{"id": "frontend", "label": "前端", "source_file": "docs/前端.md", "source_location": "L1", "community": 0}]) + tokens = _query_subgraph_tokens(G, "前端", depth=1) + assert tokens > 0 + + +# --- run_benchmark --- + +def test_run_benchmark_returns_reduction(tmp_path): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=10_000) + assert "reduction_ratio" in result + assert result["reduction_ratio"] > 1.0 + +def test_run_benchmark_corpus_tokens_proportional(tmp_path): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + r1 = run_benchmark(str(graph_file), corpus_words=1_000) + r2 = run_benchmark(str(graph_file), corpus_words=10_000) + # corpus_tokens scales linearly with corpus_words (within integer-division rounding) + assert abs(r2["corpus_tokens"] - r1["corpus_tokens"] * 10) <= r1["corpus_tokens"] + +def test_run_benchmark_per_question_list(tmp_path): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=5_000, + questions=["how does authentication work", "what is the main entry"]) + assert len(result["per_question"]) >= 1 + for p in result["per_question"]: + assert "question" in p + assert "query_tokens" in p + assert "reduction" in p + +def test_run_benchmark_estimates_corpus_if_no_words(tmp_path): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=None) + assert result["corpus_words"] > 0 + +def test_run_benchmark_error_on_empty_graph(tmp_path): + G = graph_from_payload([]) + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=1_000) + assert "error" in result + +def test_run_benchmark_includes_node_edge_counts(tmp_path): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=5_000) + assert result["nodes"] == G.node_count + assert result["edges"] == G.edge_count + + +# --- print_benchmark --- + +def test_print_benchmark_no_crash(tmp_path, capsys): + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=5_000) + print_benchmark(result) + out = capsys.readouterr().out + assert "reduction" in out.lower() + assert "x" in out + +def test_print_benchmark_error_message(capsys): + print_benchmark({"error": "test error message"}) + out = capsys.readouterr().out + assert "test error message" in out + + +# --- cp1252 / Windows-console encoding compatibility (regression for #?) --- +# print_benchmark previously crashed on Windows consoles (cp1252) because it +# unconditionally printed U+2500 and U+2192. _safe() falls back to ASCII when +# stdout cannot encode the glyph. + +def test_safe_returns_unicode_when_encodable(): + import io, sys + real_stdout = sys.stdout + try: + sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") + assert _safe("→", "->") == "→" + assert _hr(5) == "─" * 5 + finally: + sys.stdout = real_stdout + +def test_safe_falls_back_when_unencodable(): + import io, sys + real_stdout = sys.stdout + try: + sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252") + assert _safe("→", "->") == "->" + assert _hr(5) == "-" * 5 + finally: + sys.stdout = real_stdout + +def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): + """Regression: U+2500 / U+2192 used to crash with UnicodeEncodeError on cp1252.""" + import io, sys + G = _make_graph() + graph_file = _write_graph(G, tmp_path / "graph.helix") + result = run_benchmark(str(graph_file), corpus_words=5_000) + + # Replace stdout with a strict cp1252 stream — same behaviour as the + # legacy Windows console that surfaced this bug. + cp1252_stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", cp1252_stdout) + print_benchmark(result) # must not raise UnicodeEncodeError + cp1252_stdout.flush() + written = cp1252_stdout.buffer.getvalue().decode("cp1252") + assert "reduction" in written.lower() + # ASCII fallbacks must be present, fancy glyphs must not. + assert "─" not in written + assert "→" not in written diff --git a/tests/restored/test_build_merge_hyperedges_and_prune_restored.py b/tests/restored/test_build_merge_hyperedges_and_prune_restored.py new file mode 100644 index 000000000..6c4407a7b --- /dev/null +++ b/tests/restored/test_build_merge_hyperedges_and_prune_restored.py @@ -0,0 +1,132 @@ +"""Incremental merge retention and path-normalization regressions on build DTOs.""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from graphify.build import _infer_merge_root, build_from_extraction, build_merge + + +def _seed(): + return build_from_extraction({ + "nodes": [ + {"id": "a1", "label": "a1", "source_file": "a.md", "_origin": "ast"}, + {"id": "b1", "label": "b1", "source_file": "b.md", "_origin": "ast"}, + ], + "edges": [], + "hyperedges": [ + {"id": "he_a", "source_file": "a.md", "nodes": ["a1"], "_origin": "ast"}, + {"id": "he_b", "source_file": "b.md", "nodes": ["b1"], "_origin": "ast"}, + {"id": "he_global", "nodes": ["a1", "b1"]}, + ], + }) + + +def _he_ids(data): + return {item["id"] for item in data.attributes.get("hyperedges", [])} + + +def _labels(data): + return {node.attributes.get("label") for node in data.nodes} + + +def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): + fresh = { + "nodes": [{"id": "b1", "label": "b1", "source_file": "b.md", "_origin": "ast"}], + "edges": [], + "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"], "_origin": "ast"}], + } + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=_seed(), dedup=False, root=tmp_path) + assert _he_ids(data) == {"he_a", "he_global", "he_b_v2"} + + +def test_update_without_root_still_preserves_hyperedges(tmp_path): + fresh = { + "nodes": [{"id": "b1", "label": "b1", "source_file": "b.md", "_origin": "ast"}], + "edges": [], + "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"], "_origin": "ast"}], + } + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=_seed(), dedup=False) + assert _he_ids(data) == {"he_a", "he_global", "he_b_v2"} + + +def test_deleted_file_hyperedges_are_pruned(tmp_path): + data = build_merge( + [], + tmp_path / "graph.helix", + base_graph=_seed(), + prune_sources=[str(tmp_path / "a.md")], + dedup=False, + root=tmp_path, + ) + assert "he_a" not in _he_ids(data) + assert "he_b" in _he_ids(data) + assert "a1" not in {node.id for node in data.nodes} + + +def test_prune_without_root_removes_ghost_nodes_via_grandparent_fallback(tmp_path): + root = tmp_path / "corpus" + path = root / "graphify-out" / "graph.helix" + path.parent.mkdir(parents=True) + data = build_merge( + [], + path, + base_graph=build_from_extraction({ + "nodes": [{"id": "h1", "label": "handoff", "source_file": "HANDOFF.md", "_origin": "ast"}, {"id": "k1", "label": "keep", "source_file": "KEEP.md", "_origin": "ast"}], + "edges": [], + }), + prune_sources=[str(root / "HANDOFF.md")], + dedup=False, + ) + assert _labels(data) == {"keep"} + + +def test_prune_without_root_uses_graphify_root_marker(tmp_path): + out = tmp_path / "out" + out.mkdir() + path = out / "graph.helix" + real_root = tmp_path / "elsewhere" / "repo" + real_root.mkdir(parents=True) + (out / ".graphify_root").write_text(str(real_root)) + assert _infer_merge_root(path) == str(real_root.resolve()) + data = build_merge( + [], + path, + base_graph=build_from_extraction({"nodes": [{"id": "h1", "label": "handoff", "source_file": "HANDOFF.md", "_origin": "ast"}], "edges": []}), + prune_sources=[str(real_root / "HANDOFF.md")], + dedup=False, + ) + assert data.node_count == 0 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") +def test_prune_matches_across_symlinked_root(tmp_path): + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real) + data = build_merge( + [], + real / "graph.helix", + base_graph=build_from_extraction({"nodes": [{"id": "h1", "label": "handoff", "source_file": "HANDOFF.md", "_origin": "ast"}, {"id": "k1", "label": "keep", "source_file": "KEEP.md", "_origin": "ast"}], "edges": []}), + prune_sources=[str(link / "HANDOFF.md")], + root=real, + dedup=False, + ) + assert _labels(data) == {"keep"} + + +def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): + base = build_from_extraction({"nodes": [{"id": "foo", "label": "Widget Cache Design", "source_file": "docs/foo.md", "_origin": "ast"}, {"id": "bar", "label": "Other", "source_file": "docs/bar.md", "_origin": "ast"}], "edges": []}) + fresh = {"nodes": [{"id": "foo", "label": "Widget Cache Design", "source_file": "docs/foo.md", "_origin": "ast"}], "edges": []} + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=base, prune_sources=["docs/foo.md"], root=tmp_path) + assert "Widget Cache Design" in _labels(data) + + +def test_genuine_deletion_still_prunes(tmp_path): + base = build_from_extraction({"nodes": [{"id": "foo", "label": "Widget Cache Design", "source_file": "docs/foo.md", "_origin": "ast"}, {"id": "bar", "label": "Other", "source_file": "docs/bar.md", "_origin": "ast"}], "edges": []}) + fresh = {"nodes": [{"id": "foo", "label": "Widget Cache Design", "source_file": "docs/foo.md", "_origin": "ast"}], "edges": []} + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=base, prune_sources=["docs/bar.md"], root=tmp_path) + assert _labels(data) == {"Widget Cache Design"} diff --git a/tests/restored/test_build_restored.py b/tests/restored/test_build_restored.py new file mode 100644 index 000000000..302db45c3 --- /dev/null +++ b/tests/restored/test_build_restored.py @@ -0,0 +1,486 @@ +"""Retained build regressions adapted to transient DTOs and native snapshots.""" +from __future__ import annotations + +import json +from pathlib import Path + +from graphify.build import ( + _semantic_id_remap, + build, + build_from_extraction, + build_merge, + dedupe_edges, + dedupe_nodes, + edge_data, + edge_datas, + graph_has_legacy_ids, +) +from graphify.cluster import cluster +from tests.native_helpers import graph_from_build, graph_from_payload + +FIXTURES = Path(__file__).parents[1] / "fixtures" + + +def _attrs(data, node_id): + return next(node.attributes for node in data.nodes if node.id == node_id) + + +def _edge(data, source, target, relation=None): + return next( + edge for edge in data.edges + if edge.source == source and edge.target == target + and (relation is None or edge.attributes.get("relation") == relation) + ) + + +def _has_edge(data, source, target): + return any(edge.source == source and edge.target == target for edge in data.edges) + + +def _ids(data): + return {node.id for node in data.nodes} + + +def load_extraction(): + return json.loads((FIXTURES / "extraction.json").read_text()) + + +def test_dedupe_edges_collapses_exact_parallels(): + edges = [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, + {"source": "a", "target": "b", "relation": "imports"}, + {"source": "b", "target": "c", "relation": "calls"}, + ] + out = dedupe_edges(edges) + assert [(e["source"], e["target"], e["relation"]) for e in out] == [ + ("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls") + ] + assert out[0]["source_location"] == "L1" + + +def test_dedupe_edges_is_idempotent(): + edges = [{"source": "a", "target": "b", "relation": "calls"}] * 2 + once = dedupe_edges(edges) + assert len(once) == len(dedupe_edges(once + edges)) == 1 + + +def test_dedupe_nodes_collapses_by_id_last_wins(): + nodes = [ + {"id": "foundation", "source_file": "A.swift"}, + {"id": "akit"}, + {"id": "foundation", "source_file": "B.swift"}, + ] + out = dedupe_nodes(nodes) + assert [node["id"] for node in out] == ["foundation", "akit"] + assert out[0]["source_file"] == "B.swift" + + +def test_build_from_json_node_count(): + assert build_from_extraction(load_extraction()).node_count == 4 + + +def test_build_from_json_edge_count(): + assert build_from_extraction(load_extraction()).edge_count == 4 + + +def test_null_weight_edge_builds_and_clusters(tmp_path): + data = build_from_extraction({ + "nodes": [{"id": item, "label": item, "source_file": f"{item}.py"} for item in "abc"], + "edges": [ + {"source": "a", "target": "b", "relation": "references", "weight": None, "confidence_score": None}, + {"source": "b", "target": "c", "relation": "references", "weight": 2.5}, + ], + }) + assert _edge(data, "a", "b").attributes["weight"] == 1.0 + assert _edge(data, "a", "b").attributes["confidence_score"] == 1.0 + assert _edge(data, "b", "c").attributes["weight"] == 2.5 + cluster(graph_from_build(data)) + + +def test_malformed_weights_normalize(): + data = build_from_extraction({ + "nodes": [{"id": f"n{i}", "source_file": f"{i}.py"} for i in range(4)], + "edges": [ + {"source": "n0", "target": "n1", "weight": "3.5"}, + {"source": "n1", "target": "n2", "weight": float("nan")}, + {"source": "n2", "target": "n3", "weight": -4}, + ], + }) + assert [_edge(data, f"n{i}", f"n{i+1}").attributes["weight"] for i in range(3)] == [3.5, 1.0, 1.0] + + +def test_nodes_have_label(): + assert _attrs(build_from_extraction(load_extraction()), "n_transformer")["label"] == "Transformer" + + +def test_edges_have_confidence(): + data = build_from_extraction(load_extraction()) + assert _edge(data, "n_attention", "n_concept_attn").attributes["confidence"] == "INFERRED" + + +def test_ambiguous_edge_preserved(): + data = build_from_extraction(load_extraction()) + assert _edge(data, "n_layernorm", "n_concept_attn").attributes["confidence"] == "AMBIGUOUS" + + +def test_legacy_node_source_canonicalized(): + data = build_from_extraction({"nodes": [{"id": "n1", "source": "a.py"}], "edges": []}) + assert _attrs(data, "n1")["source_file"] == "a.py" + assert "source" not in _attrs(data, "n1") + + +def test_legacy_edge_from_to_canonicalized(): + data = build_from_extraction({ + "nodes": [{"id": "n1"}, {"id": "n2"}], + "edges": [{"from": "n1", "to": "n2", "relation": "calls"}], + }) + assert data.edge_count == 1 + + +def test_source_file_backslash_normalized(): + data = build_from_extraction({ + "nodes": [ + {"id": "n1", "source_file": r"src\middleware\auth.py"}, + {"id": "n2", "source_file": "src/middleware/auth.py"}, + ], + "edges": [], + }) + assert {_attrs(data, node)["source_file"] for node in _ids(data)} == {"src/middleware/auth.py"} + + +def test_edge_missing_source_file_backfilled_from_node(): + data = build_from_extraction({ + "nodes": [{"id": "n1", "source_file": "docs/a.md"}, {"id": "n2", "source_file": "docs/b.md"}], + "edges": [{"source": "n1", "target": "n2", "relation": "relates_to"}], + }) + assert _edge(data, "n1", "n2").attributes["source_file"] == "docs/a.md" + + +def test_build_merges_multiple_extractions(): + data = build([ + {"nodes": [{"id": "n1", "source_file": "a.py"}], "edges": []}, + {"nodes": [{"id": "n2", "source_file": "b.md"}], "edges": [{"source": "n1", "target": "n2"}]}, + ]) + assert data.node_count == 2 and data.edge_count == 1 + + +def test_none_file_type_defaults_to_concept(capsys): + data = build_from_extraction({"nodes": [{"id": "n1", "file_type": None}], "edges": []}) + assert _attrs(data, "n1")["file_type"] == "concept" + assert "invalid file_type" not in capsys.readouterr().err + + +def test_missing_file_type_defaults_to_concept(capsys): + data = build_from_extraction({"nodes": [{"id": "n1"}], "edges": []}) + assert _attrs(data, "n1")["file_type"] == "concept" + assert "invalid file_type" not in capsys.readouterr().err + + +def test_real_invalid_file_type_coerced_to_concept(): + data = build_from_extraction({"nodes": [{"id": "n1", "file_type": "weird_type"}], "edges": []}) + assert _attrs(data, "n1")["file_type"] == "concept" + + +def test_file_type_synonym_mapping(): + data = build_from_extraction({ + "nodes": [ + {"id": "n1", "file_type": "markdown"}, + {"id": "n2", "file_type": "tool"}, + {"id": "n3", "file_type": "pattern"}, + ], + "edges": [], + }) + assert [_attrs(data, node)["file_type"] for node in ("n1", "n2", "n3")] == ["document", "code", "concept"] + + +def _ghost_nodes(two_ast=False, same_file=False): + nodes = [ + {"id": "ast_render", "label": "render", "source_file": "src/a/index.ts", "source_location": "L10", "_origin": "ast"}, + {"id": "ghost_render", "label": "render", "source_file": "src/a/index.ts", "source_location": "L11"}, + {"id": "caller", "label": "main", "source_file": "src/main.ts", "source_location": "L1", "_origin": "ast"}, + ] + if two_ast: + nodes.append({"id": "other_render", "label": "render", "source_file": "src/b/index.ts", "source_location": "L20", "_origin": "ast"}) + if same_file: + nodes = [ + {"id": "a_foo", "label": "Foo", "source_file": "x/doc.md", "source_location": "L1"}, + {"id": "b_foo", "label": "Foo", "source_file": "x/doc.md", "source_location": "L2"}, + ] + return nodes + + +def test_ghost_merge_unique_located_node_still_merges(): + data = build_from_extraction({ + "nodes": _ghost_nodes(), + "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls"}], + }) + assert "ghost_render" not in _ids(data) + assert _has_edge(data, "caller", "ast_render") + + +def test_ghost_merge_uses_full_path_despite_basename_collision(): + data = build_from_extraction({ + "nodes": _ghost_nodes(two_ast=True), + "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls"}], + }) + assert "ghost_render" not in _ids(data) + assert "other_render" in _ids(data) + assert _has_edge(data, "caller", "ast_render") + assert not _has_edge(data, "caller", "other_render") + + +def test_ghost_merge_non_ast_different_files_both_survive(): + data = build_from_extraction({ + "nodes": [ + {"id": "a", "label": "build_merge() function", "source_file": "dir_a/update.md", "source_location": "L10"}, + {"id": "b", "label": "build_merge() function", "source_file": "dir_b/update.md", "source_location": "L12"}, + ], + "edges": [], + }) + assert _ids(data) == {"a", "b"} + + +def test_ghost_merge_non_ast_same_file_still_merges(): + assert build_from_extraction({"nodes": _ghost_nodes(same_file=True), "edges": []}).node_count == 1 + + +def test_build_merge_preserves_call_edge_direction(tmp_path): + from graphify.extract import extract_js + + source = tmp_path / "x.js" + source.write_text("function b() {}\nfunction a() { b(); }\n") + extraction = extract_js(source) + initial = build([extraction], dedup=False) + call = next(edge for edge in initial.edges if edge.attributes.get("relation") == "calls") + merged = build_merge([], tmp_path / "graph.helix", base_graph=initial, dedup=False) + retained = next(edge for edge in merged.edges if edge.attributes.get("relation") == "calls") + assert (retained.source, retained.target) == (call.source, call.target) + + +def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_path): + data = build_from_extraction({ + "nodes": [{"id": "a_handler"}, {"id": "z_emitter"}], + "edges": [ + {"source": "a_handler", "target": "z_emitter", "relation": "calls"}, + {"source": "z_emitter", "target": "a_handler", "relation": "calls"}, + ], + }) + assert data.edge_count == 1 + assert (data.edges[0].source, data.edges[0].target) == ("a_handler", "z_emitter") + + +def test_edge_data_simple_graph(): + graph = graph_from_payload([{"id": "a"}, {"id": "b"}], [{"source": "a", "target": "b", "relation": "calls"}]) + assert edge_data(graph, "a", "b")["relation"] == "calls" + + +def test_edge_datas_simple_graph_returns_singleton_list(): + graph = graph_from_payload([{"id": "a"}, {"id": "b"}], [{"source": "a", "target": "b", "relation": "calls"}]) + assert len(edge_datas(graph, "a", "b")) == 1 + + +def test_edge_data_multigraph_with_parallel_edges(): + graph = graph_from_payload( + [{"id": "a"}, {"id": "b"}], + [{"source": "a", "target": "b", "key": "one", "relation": "calls"}, {"source": "a", "target": "b", "key": "two", "relation": "uses"}], + kind="multigraph", + ) + assert edge_data(graph, "a", "b")["relation"] in {"calls", "uses"} + + +def test_edge_datas_multigraph_returns_all_parallel_edges(): + graph = graph_from_payload( + [{"id": "a"}, {"id": "b"}], + [{"source": "a", "target": "b", "key": "one", "relation": "calls"}, {"source": "a", "target": "b", "key": "two", "relation": "uses"}], + kind="multigraph", + ) + assert {row["relation"] for row in edge_datas(graph, "a", "b")} == {"calls", "uses"} + + +def test_edge_data_multidigraph(): + graph = graph_from_payload([{"id": "a"}, {"id": "b"}], [{"source": "a", "target": "b", "key": 0, "relation": "calls"}], kind="multidigraph") + assert edge_data(graph, "a", "b")["relation"] == "calls" + + +def test_edge_data_node_link_multigraph_roundtrip(): + data = build_from_extraction({ + "directed": True, + "multigraph": True, + "nodes": [{"id": "a"}, {"id": "b"}], + "links": [{"source": "a", "target": "b", "key": 4, "relation": "calls"}], + }) + graph = graph_from_build(data) + assert edge_datas(graph, "a", "b")[0]["relation"] == "calls" + + +def test_build_from_json_relativizes_absolute_source_file(tmp_path): + root = tmp_path / "repo" + path = root / "src" / "a.py" + data = build_from_extraction({"nodes": [{"id": "a", "source_file": str(path)}], "edges": []}, root=root) + assert _attrs(data, "src_a")["source_file"] == "src/a.py" + + +def test_build_relativizes_absolute_source_file(tmp_path): + root = tmp_path / "repo" + data = build([{"nodes": [{"id": "a", "source_file": str(root / "src/a.py")}], "edges": []}], root=root) + assert next(iter(data.nodes)).attributes["source_file"] == "src/a.py" + + +def test_build_from_json_ambiguous_old_stem_alias_stays_dangling(tmp_path): + data = build_from_extraction({ + "nodes": [ + {"id": "a_utility", "label": "utility.h", "source_file": "A/utility.h"}, + {"id": "b_utility", "label": "utility.h", "source_file": "B/utility.h"}, + {"id": "server", "label": "server", "source_file": "server.cpp"}, + ], + "edges": [{"source": "server", "target": "utility", "relation": "imports"}], + }, root=tmp_path) + assert data.edge_count == 0 + + +def test_build_from_json_ambiguous_alias_detected_despite_header_impl_salting(tmp_path): + data = build_from_extraction({ + "nodes": [ + {"id": "a_utility_h", "label": "utility.h", "source_file": "A/utility.h"}, + {"id": "b_utility_h", "label": "utility.h", "source_file": "B/utility.h"}, + {"id": "server", "source_file": "server.cpp"}, + ], + "edges": [{"source": "server", "target": "utility", "relation": "imports"}], + }, root=tmp_path) + assert data.edge_count == 0 + + +def test_build_from_json_unambiguous_old_stem_alias_still_resolves(tmp_path): + data = build_from_extraction({ + "nodes": [ + {"id": "monitoring_utility", "label": "utility.h", "source_file": "Dev/monitoring/utility.h"}, + {"id": "server", "source_file": "Dev/poker/server.cpp"}, + ], + "edges": [{"source": "server", "target": "utility", "relation": "imports"}], + }, root=tmp_path) + assert data.edge_count == 1 + + +def test_build_from_json_relative_source_file_unchanged(tmp_path): + data = build_from_extraction({"nodes": [{"id": "foo_bar", "label": "bar", "source_file": "src/foo.py"}], "edges": []}, root=tmp_path) + assert next(iter(data.nodes)).attributes["source_file"] == "src/foo.py" + + +def _merge_base(): + return build([{ + "nodes": [ + {"id": "n1", "label": "login", "source_file": "module_a/auth.py", "_origin": "ast"}, + {"id": "n2", "label": "format_date", "source_file": "module_b/utils.py", "_origin": "ast"}, + ], + "edges": [{"source": "n1", "target": "n2", "source_file": "module_b/utils.py", "_origin": "ast"}], + }], dedup=False) + + +def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): + root = tmp_path / "corpus" + data = build_merge([], tmp_path / "graph.helix", base_graph=_merge_base(), prune_sources=[str(root / "module_b/utils.py")], root=root, dedup=False) + assert {node.attributes.get("label") for node in data.nodes} == {"login"} + assert data.edge_count == 0 + + +def test_build_merge_prune_windows_backslash_paths(tmp_path): + root = tmp_path / "corpus" + data = build_merge([], tmp_path / "graph.helix", base_graph=_merge_base(), prune_sources=[str(root / "module_b/utils.py").replace("/", "\\")], root=root, dedup=False) + assert "format_date" not in {node.attributes.get("label") for node in data.nodes} + + +def test_build_merge_replaces_changed_file_stale_edges(tmp_path): + base = build([{ + "nodes": [{"id": "A", "source_file": "changed.md", "_origin": "ast"}, {"id": "B", "source_file": "changed.md", "_origin": "ast"}, {"id": "K", "source_file": "keep.md", "_origin": "ast"}], + "edges": [{"source": "A", "target": "B", "source_file": "changed.md", "_origin": "ast"}, {"source": "K", "target": "A", "source_file": "keep.md", "_origin": "ast"}], + }], dedup=False) + fresh = {"nodes": [{"id": "A", "source_file": "changed.md", "_origin": "ast"}, {"id": "C", "source_file": "changed.md", "_origin": "ast"}], "edges": [{"source": "A", "target": "C", "source_file": "changed.md", "_origin": "ast"}]} + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=base, dedup=False, root=tmp_path) + assert _ids(data) == {"A", "C", "K"} + assert _has_edge(data, "A", "C") and _has_edge(data, "K", "A") + + +def test_build_merge_root_collapses_convention_drift(tmp_path): + base = build([{"nodes": [{"id": "old", "source_file": "docs/wiki/overview.md", "_origin": "ast"}, {"id": "stale", "source_file": "docs/wiki/overview.md", "_origin": "ast"}], "edges": []}], dedup=False) + fresh = {"nodes": [{"id": "new", "source_file": str(tmp_path / "docs/wiki/overview.md"), "_origin": "ast"}], "edges": []} + data = build_merge([fresh], tmp_path / "graph.helix", base_graph=base, dedup=False, root=tmp_path) + assert data.node_count == 1 + assert next(iter(data.nodes)).attributes["source_file"] == "docs/wiki/overview.md" + + +def test_build_from_json_skips_non_hashable_node_id(): + data = build_from_extraction({"nodes": [{"id": "a"}, {"id": ["x"]}, {"label": "missing"}], "edges": []}) + assert _ids(data) == {"a"} + + +def test_build_from_json_skips_edge_with_non_hashable_endpoint(): + data = build_from_extraction({ + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [{"source": "a", "target": ["b"], "relation": "calls"}, {"source": "a", "target": "b", "relation": "imports"}], + }) + assert data.node_count == 2 and data.edge_count == 1 + + +def test_graph_has_legacy_ids_detects_old_scheme(): + old = [{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] + new = [{"id": "docs_v1_api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] + assert graph_has_legacy_ids(old, root=".") + assert not graph_has_legacy_ids(new, root=".") + + +def test_semantic_rekey_relative_vs_absolute_source_file(): + assert _semantic_id_remap([{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document"}], ".") == {"api_readme": "docs_v1_api_readme"} + assert _semantic_id_remap([{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}], None) == {} + + +def test_cross_language_imports_references_are_dropped(): + data = build_from_extraction({ + "nodes": [ + {"id": "py", "source_file": "backend/worker.py", "_origin": "ast"}, + {"id": "ts", "source_file": "src/time.ts", "_origin": "ast"}, + {"id": "util", "source_file": "src/util.ts", "_origin": "ast"}, + ], + "edges": [{"source": "py", "target": "ts", "relation": "imports"}, {"source": "ts", "target": "util", "relation": "imports"}], + }) + assert not _has_edge(data, "py", "ts") + assert _has_edge(data, "ts", "util") + + +def test_cross_family_reference_to_unknown_ext_is_kept(): + data = build_from_extraction({ + "nodes": [{"id": "pkg", "source_file": "package.json", "_origin": "ast"}, {"id": "app", "source_file": "src/app.ts", "_origin": "ast"}], + "edges": [{"source": "pkg", "target": "app", "relation": "references"}], + }) + assert _has_edge(data, "pkg", "app") + + +def test_markdown_doc_twin_merges_into_semantic_doc_node(): + data = build_from_extraction({ + "nodes": [ + {"id": "docs_readme_doc", "file_type": "document", "source_file": "docs/readme.md"}, + {"id": "docs_readme", "file_type": "document", "source_file": "docs/readme.md"}, + {"id": "auth", "source_file": "auth.py"}, + ], + "edges": [{"source": "docs_readme", "target": "auth", "relation": "references"}], + }) + assert "docs_readme" not in _ids(data) + assert _has_edge(data, "docs_readme_doc", "auth") + + +def test_doc_twin_merge_does_not_touch_code_symbols(): + data = build_from_extraction({ + "nodes": [{"id": "m_foo", "file_type": "code", "source_file": "m.py"}, {"id": "m_foo_doc", "file_type": "rationale", "source_file": "m.py"}], + "edges": [], + }) + assert _ids(data) == {"m_foo", "m_foo_doc"} + + +def test_build_from_json_prunes_dangling_hyperedge_members(capsys): + data = build_from_extraction({ + "nodes": [{"id": "alpha"}, {"id": "beta"}], + "edges": [], + "hyperedges": [{"id": "partial", "nodes": ["alpha", "beta", "ghost"]}, {"id": "gone", "nodes": ["ghost"]}], + }) + assert data.attributes["hyperedges"] == [{"id": "partial", "nodes": ["alpha", "beta"]}] + assert "gone" in capsys.readouterr().err diff --git a/tests/restored/test_cli_export_restored.py b/tests/restored/test_cli_export_restored.py new file mode 100644 index 000000000..33708d87b --- /dev/null +++ b/tests/restored/test_cli_export_restored.py @@ -0,0 +1,74 @@ +"""CLI behaviors not subsumed by the bundled native export/query tests.""" +from __future__ import annotations + +import os +import subprocess +import sys + +from graphify.helix.persistence import load_graph + + +def _run(cwd, *args, env=None): + return subprocess.run( + [sys.executable, "-m", "graphify", *args], + cwd=cwd, + capture_output=True, + text=True, + env=env, + ) + + +def test_export_html_error_without_graph(tmp_path): + assert _run(tmp_path, "export", "html").returncode != 0 + + +def test_query_missing_graph_fails(tmp_path): + assert _run(tmp_path, "query", "anything").returncode != 0 + + +def test_path_missing_graph_fails(tmp_path): + assert _run(tmp_path, "path", "a", "b").returncode != 0 + + +def test_explain_missing_graph_fails(tmp_path): + assert _run(tmp_path, "explain", "anything").returncode != 0 + + +def test_export_unknown_format_fails(tmp_path): + assert _run(tmp_path, "export", "pdf").returncode != 0 + + +def test_extract_writes_to_graphify_out_env(tmp_path): + (tmp_path / "m.py").write_text("def a():\n return 1\n") + env = dict(os.environ, GRAPHIFY_OUT="custom-out") + result = _run(tmp_path, "extract", ".", "--code-only", "--no-cluster", env=env) + assert result.returncode == 0, result.stderr + assert (tmp_path / "custom-out" / "graph.helix").is_dir() + assert not (tmp_path / "graphify-out").exists() + + +def test_extract_out_does_not_pollute_corpus(tmp_path): + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text("def main():\n return 1\n") + output = tmp_path / "scratch" + result = _run( + tmp_path, + "extract", + str(corpus), + "--out", + str(output), + "--no-cluster", + "--code-only", + ) + assert result.returncode == 0, result.stderr + assert (output / "graphify-out" / "graph.helix").is_dir() + assert not (corpus / "graphify-out").exists() + + +def test_update_no_cluster_writes_native_store(tmp_path): + (tmp_path / "sample.py").write_text("def f():\n return 1\n") + result = _run(tmp_path, "update", ".", "--no-cluster") + assert result.returncode == 0, result.stderr + loaded = load_graph(tmp_path / "graphify-out" / "graph.helix") + assert loaded.graph.node_count > 0 diff --git a/tests/restored/test_cluster_restored.py b/tests/restored/test_cluster_restored.py new file mode 100644 index 000000000..4ea4630e4 --- /dev/null +++ b/tests/restored/test_cluster_restored.py @@ -0,0 +1,103 @@ +import json +from pathlib import Path +from graphify.build import build_from_extraction +from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all +from tests.native_helpers import graph_from_build, graph_from_payload + +FIXTURES = Path(__file__).parents[1] / "fixtures" + +def make_graph(): + return graph_from_build( + build_from_extraction(json.loads((FIXTURES / "extraction.json").read_text())) + ) + +def test_cluster_returns_dict(): + G = make_graph() + communities = cluster(G) + assert isinstance(communities, dict) + +def test_cluster_covers_all_nodes(): + G = make_graph() + communities = cluster(G) + all_nodes = {n for nodes in communities.values() for n in nodes} + assert all_nodes == {node.id for node in G.nodes()} + +def test_cohesion_score_complete_graph(): + nodes = [{"id": str(index)} for index in range(4)] + edges = [ + {"source": str(left), "target": str(right)} + for left in range(4) for right in range(left + 1, 4) + ] + G = graph_from_payload(nodes, edges) + score = cohesion_score(G, [node["id"] for node in nodes]) + assert score == 1.0 + +def test_cohesion_score_single_node(): + G = graph_from_payload([{"id": "a"}]) + score = cohesion_score(G, ["a"]) + assert score == 1.0 + +def test_cohesion_score_disconnected(): + G = graph_from_payload([{"id": node} for node in ("a", "b", "c")]) + score = cohesion_score(G, ["a", "b", "c"]) + assert score == 0.0 + +def test_cohesion_score_range(): + G = make_graph() + communities = cluster(G) + for cid, nodes in communities.items(): + score = cohesion_score(G, nodes) + assert 0.0 <= score <= 1.0 + +def test_score_all_keys_match_communities(): + G = make_graph() + communities = cluster(G) + scores = score_all(G, communities) + assert set(scores.keys()) == set(communities.keys()) + + +def test_cluster_does_not_write_to_stdout(capsys): + """Clustering should not emit ANSI escape codes or other output. + + Native Leiden diagnostics can emit ANSI escape sequences that break + PowerShell 5.1's scroll buffer on Windows (issue #19). The output + suppression in _partition() should prevent any output from leaking. + """ + G = make_graph() + cluster(G) + captured = capsys.readouterr() + assert captured.out == "", f"cluster() wrote to stdout: {captured.out!r}" + + +def test_cluster_does_not_write_to_stderr(capsys): + """Same as above but for stderr — ANSI codes can go to either stream.""" + G = make_graph() + cluster(G) + captured = capsys.readouterr() + # Allow logging output (starts with [graphify]) but no raw ANSI codes + for line in captured.err.splitlines(): + assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}" + + +def test_remap_communities_to_previous_reuses_old_ids(): + communities = { + 10: ["a", "b", "c"], + 11: ["d", "e"], + } + previous = {"a": 5, "b": 5, "c": 5, "d": 1, "e": 1} + remapped = remap_communities_to_previous(communities, previous) + assert set(remapped.keys()) == {1, 5} + assert remapped[5] == ["a", "b", "c"] + assert remapped[1] == ["d", "e"] + + +def test_remap_communities_to_previous_assigns_deterministic_new_ids(): + communities = { + 7: ["x", "y", "z"], + 8: ["m"], + } + previous = {"a": 3} + remapped = remap_communities_to_previous(communities, previous) + assert list(remapped.keys()) == [0, 1] + assert remapped[0] == ["x", "y", "z"] + assert remapped[1] == ["m"] diff --git a/tests/restored/test_confidence_restored.py b/tests/restored/test_confidence_restored.py new file mode 100644 index 000000000..95e1617e0 --- /dev/null +++ b/tests/restored/test_confidence_restored.py @@ -0,0 +1,174 @@ +"""Tests for confidence_score on native edges.""" +from graphify.build import build_from_extraction +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.helix.access import edge_rows +from tests.native_helpers import graph_from_build + + +def _make_extraction(**edge_overrides): + """Return a minimal extraction dict with one edge of each confidence type.""" + base = { + "nodes": [ + {"id": "n_a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n_b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "n_c", "label": "C", "file_type": "document", "source_file": "c.md"}, + {"id": "n_d", "label": "D", "file_type": "document", "source_file": "d.md"}, + ], + "edges": [ + {"source": "n_a", "target": "n_b", "relation": "calls", "confidence": "EXTRACTED", + "confidence_score": 1.0, "source_file": "a.py", "weight": 1.0}, + {"source": "n_b", "target": "n_c", "relation": "implements", "confidence": "INFERRED", + "confidence_score": 0.75, "source_file": "b.py", "weight": 0.8}, + {"source": "n_c", "target": "n_d", "relation": "references", "confidence": "AMBIGUOUS", + "confidence_score": 0.2, "source_file": "c.md", "weight": 0.5}, + ], + "input_tokens": 100, + "output_tokens": 50, + } + return base + + +def _make_graph(extraction=None): + return graph_from_build(build_from_extraction(extraction or _make_extraction())) + + +def test_extracted_edges_have_score_1(): + """EXTRACTED edges must have confidence_score == 1.0.""" + G = _make_graph() + for u, v, d, _edge in edge_rows(G): + if d.get("confidence") == "EXTRACTED": + assert d.get("confidence_score") == 1.0, ( + f"EXTRACTED edge ({u},{v}) should have confidence_score=1.0, got {d.get('confidence_score')}" + ) + + +def test_inferred_edges_score_in_range(): + """INFERRED edges must have confidence_score between 0.0 and 1.0.""" + G = _make_graph() + found = False + for u, v, d, _edge in edge_rows(G): + if d.get("confidence") == "INFERRED": + found = True + score = d.get("confidence_score") + assert score is not None, f"INFERRED edge ({u},{v}) missing confidence_score" + assert 0.0 <= score <= 1.0, ( + f"INFERRED edge ({u},{v}) confidence_score={score} out of range [0,1]" + ) + assert found, "No INFERRED edges found in test fixture" + + +def test_ambiguous_edges_score_at_most_04(): + """AMBIGUOUS edges must have confidence_score <= 0.4.""" + G = _make_graph() + found = False + for u, v, d, _edge in edge_rows(G): + if d.get("confidence") == "AMBIGUOUS": + found = True + score = d.get("confidence_score") + assert score is not None, f"AMBIGUOUS edge ({u},{v}) missing confidence_score" + assert score <= 0.4, ( + f"AMBIGUOUS edge ({u},{v}) confidence_score={score} should be <= 0.4" + ) + assert found, "No AMBIGUOUS edges found in test fixture" + + +def test_confidence_score_round_trip(): + """confidence_score survives transient build → native generation round-trip.""" + G = _make_graph() + links = [data for _u, _v, data, _edge in edge_rows(G)] + assert links + for link in links: + assert "confidence_score" in link, f"Edge missing confidence_score: {link}" + score = link["confidence_score"] + assert isinstance(score, float), f"confidence_score should be float, got {type(score)}" + assert 0.0 <= score <= 1.0, f"confidence_score={score} out of range" + + +def test_to_json_defaults_missing_confidence_score(): + """Edges lacking confidence_score get sensible native defaults.""" + extraction = { + "nodes": [ + {"id": "n_x", "label": "X", "file_type": "code", "source_file": "x.py"}, + {"id": "n_y", "label": "Y", "file_type": "code", "source_file": "y.py"}, + {"id": "n_z", "label": "Z", "file_type": "code", "source_file": "z.py"}, + ], + "edges": [ + # No confidence_score field on any of these + {"source": "n_x", "target": "n_y", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "x.py", "weight": 1.0}, + {"source": "n_y", "target": "n_z", "relation": "depends_on", + "confidence": "INFERRED", "source_file": "y.py", "weight": 1.0}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = _make_graph(extraction) + + links_by_conf = {} + for _u, _v, link, _edge in edge_rows(G): + conf = link.get("confidence", "EXTRACTED") + links_by_conf[conf] = link.get("confidence_score") + + assert links_by_conf.get("EXTRACTED") == 1.0, "EXTRACTED default should be 1.0" + assert links_by_conf.get("INFERRED") == 0.5, "INFERRED default should be 0.5" + + +def test_report_shows_avg_confidence_for_inferred(): + """Report summary line should include avg confidence for INFERRED edges.""" + extraction = _make_extraction() + G = _make_graph(extraction) + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + surprises = surprising_connections(G) + detection = {"total_files": 2, "total_words": 5000, "needs_graph": True, "warning": None} + tokens = {"input": 100, "output": 50} + + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, ".") + assert "avg confidence" in report, "Report should show avg confidence for INFERRED edges" + # The fixture has one INFERRED edge with score 0.75, so avg should be 0.75 + assert "0.75" in report, f"Expected avg confidence 0.75 in report" + + +def test_report_inferred_tag_with_score(): + """Surprising connections section shows confidence score next to INFERRED edges.""" + # Build a graph where surprising_connections will find an INFERRED cross-file edge + extraction = { + "nodes": [ + {"id": "n_p", "label": "Parser", "file_type": "code", "source_file": "parser.py"}, + {"id": "n_q", "label": "Renderer", "file_type": "code", "source_file": "renderer.py"}, + ], + "edges": [ + {"source": "n_p", "target": "n_q", "relation": "feeds", + "confidence": "INFERRED", "confidence_score": 0.82, + "source_file": "parser.py", "weight": 1.0}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = _make_graph(extraction) + + # Manually construct a surprise entry the way analyze.surprising_connections would + surprise = { + "source": "Parser", + "target": "Renderer", + "relation": "feeds", + "confidence": "INFERRED", + "confidence_score": 0.82, + "source_files": ["parser.py", "renderer.py"], + "note": "", + } + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + detection = {"total_files": 2, "total_words": 1000, "needs_graph": True, "warning": None} + tokens = {"input": 0, "output": 0} + + report = generate(G, communities, cohesion, labels, gods, [surprise], detection, tokens, ".") + assert "INFERRED 0.82" in report, ( + f"Report should show 'INFERRED 0.82' in surprising connections section. Got:\n{report}" + ) diff --git a/tests/restored/test_corrupt_graph_json_restored.py b/tests/restored/test_corrupt_graph_json_restored.py new file mode 100644 index 000000000..0fa365122 --- /dev/null +++ b/tests/restored/test_corrupt_graph_json_restored.py @@ -0,0 +1,54 @@ +"""Corrupt native generations produce actionable recovery errors.""" +from __future__ import annotations + +import pytest + +from graphify.build import build_from_extraction, build_merge +from graphify.affected import load_graph +from graphify.helix.persistence import HelixEmbeddedStore +from graphify.security import validate_store_path +from tests.native_helpers import make_loaded + + +def _corrupt(tmp_path): + store_path = make_loaded(tmp_path, nodes=[{"id": "a"}]).store_path + with HelixEmbeddedStore(store_path) as store: + generation = store.active_generation + traversal = store._helix.g().n_with_label_where( + "GraphifyMeta", + store._helix.SourcePredicate.eq("graphify_generation", generation), + ).set_property("section_state_checksum", "sha256:corrupt") + store._query( + store._helix.write_batch().var_as("corrupt", traversal).returning(["corrupt"]) + ) + return store_path + + +def test_build_merge_corrupt_graph_raises_runtimeerror(tmp_path): + p = _corrupt(tmp_path) + with HelixEmbeddedStore(p, read_only=True) as store: + with pytest.raises(RuntimeError, match="checksum verification"): + store.verify() + + +def test_affected_load_graph_corrupt_raises_runtimeerror(tmp_path): + p = _corrupt(tmp_path) + with pytest.raises(RuntimeError, match=r"Cannot open Helix store|regenerate"): + load_graph(p) + + +def test_diagnostics_read_corrupt_raises_runtimeerror(tmp_path): + p = tmp_path / "legacy.json" + p.write_text('{"nodes": [', encoding="utf-8") + with pytest.raises(ValueError, match=r"obsolete|rebuild"): + validate_store_path(p) + + +def test_valid_graph_still_loads(tmp_path): + """A valid native store loads and its transient DTO can be updated.""" + p = make_loaded(tmp_path, nodes=[{"id": "a", "label": "a"}]).store_path + graph = load_graph(p) + assert graph.contains_node("a") + base = build_from_extraction({"nodes": [{"id": "a", "label": "a"}], "edges": []}) + merged = build_merge([], graph_path=p, base_graph=base, dedup=False) + assert merged.node_count == 1 diff --git a/tests/restored/test_explain_cli_restored.py b/tests/restored/test_explain_cli_restored.py new file mode 100644 index 000000000..9e3e0f9d3 --- /dev/null +++ b/tests/restored/test_explain_cli_restored.py @@ -0,0 +1,37 @@ +"""Learning-state explain regression not covered by the compact native suite.""" + +import graphify.__main__ as mainmod +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded + + +def test_explain_shows_contested_and_stale_lesson(monkeypatch, tmp_path, capsys): + state = new_state(learning={ + "version": 1, + "nodes": { + "validate": { + "status": "contested", + "score": -0.1, + "uses": 2, + "neg": 1, + "verdict": "dead end", + "source_file": "server/missing.ts", + "code_fingerprint": "deadbeef", + } + }, + }) + loaded = make_loaded( + tmp_path, + nodes=[{"id": "validate", "label": "validateSanitySession()", "source_file": "server/missing.ts"}], + state=state, + ) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "explain", "validate", "--store", str(loaded.store_path)], + ) + mainmod.main() + output = capsys.readouterr().out + assert "Lesson: contested (useful 2 / dead-end 1)" in output + assert "[code changed since — re-verify]" in output diff --git a/tests/restored/test_export_restored.py b/tests/restored/test_export_restored.py new file mode 100644 index 000000000..b9ee3779f --- /dev/null +++ b/tests/restored/test_export_restored.py @@ -0,0 +1,623 @@ +import json +import math +import re +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path +from graphify.build import build_from_extraction +from graphify.cluster import cluster +from graphify.export import to_cypher, to_graphml, to_html, to_canvas, to_obsidian +from tests.native_helpers import graph_from_build, graph_from_payload + +FIXTURES = Path(__file__).parents[1] / "fixtures" + +def make_graph(): + extraction = json.loads((FIXTURES / "extraction.json").read_text()) + return graph_from_build(build_from_extraction(extraction)) + +def test_to_cypher_creates_file(): + G = make_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "cypher.txt" + to_cypher(G, str(out)) + assert out.exists() + +def test_to_cypher_contains_merge_statements(): + G = make_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "cypher.txt" + to_cypher(G, str(out)) + content = out.read_text() + assert "MERGE" in content + +def test_to_graphml_creates_file(): + G = make_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.graphml" + to_graphml(G, communities, str(out)) + assert out.exists() + +def test_to_graphml_valid_xml(): + G = make_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.graphml" + to_graphml(G, communities, str(out)) + content = out.read_text() + assert " "" so a node/edge with a null field still exports (#1502).""" + G = make_graph() + communities = cluster(G) + G = graph_from_payload( + [{"id": "a", "label": "A", "nullable_field": None}, {"id": "b", "label": "B"}], + [{"source": "a", "target": "b", "nullable_field": None}], + ) + communities = {0: ["a", "b"]} + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.graphml" + to_graphml(G, communities, str(out)) # must not raise + content = out.read_text() + assert " list: + """Extract the RAW_NODES JSON array embedded in the generated HTML.""" + m = re.search(r"const RAW_NODES = (\[.*?\]);", content, re.DOTALL) + assert m, "RAW_NODES not found in HTML" + return json.loads(m.group(1).replace("<\\/", "= G.node_count + assert len(data["edges"]) >= 1 + assert out.stat().st_size > 32 + + +def test_to_canvas_node_grid_matches_box_columns(): + """#1452: a community's node cards are laid out in the same ceil(sqrt(n))-column + grid the group box is sized for. Previously the box width assumed sqrt(n) + columns while the placement loop hardcoded 3, so any community bigger than ~9 + rendered as a cramped 3-wide strip filling only part of an over-wide box. + Covers a perfect square (25 -> 5x5) and a non-square count (10 -> 4 cols, a + partial last row) so both the column count and the row count are pinned.""" + for n in (10, 25): + G = graph_from_payload( + [ + {"id": f"n{i}", "label": f"sym_{i:02d}", "file_type": "code", "source_file": "a.py"} + for i in range(n) + ], + ) + communities = {0: [f"n{i}" for i in range(n)]} + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.canvas" + to_canvas(G, communities, str(out)) + data = json.loads(out.read_text()) + + group = next(g for g in data["nodes"] if g.get("type") == "group") + cards = [c for c in data["nodes"] if c.get("type") == "file"] + assert len(cards) == n, f"n={n}" + + # Cards occupy the ceil(sqrt(n))-column / ceil(n/cols)-row grid the box is + # sized for — not the old fixed 3 columns, which spread cards across far + # more rows (the load-bearing checks: distinct column/row positions). + expected_cols = math.ceil(math.sqrt(n)) + expected_rows = math.ceil(n / expected_cols) + distinct_x = len({c["x"] for c in cards}) + distinct_y = len({c["y"] for c in cards}) + assert distinct_x == expected_cols, f"n={n}: expected {expected_cols} cols, got {distinct_x}" + assert distinct_y == expected_rows, f"n={n}: expected {expected_rows} rows, got {distinct_y}" + + # And every card sits fully inside its group box on both axes. + gx, gy, gw, gh = group["x"], group["y"], group["width"], group["height"] + for c in cards: + assert gx <= c["x"] and c["x"] + c["width"] <= gx + gw, (n, c) + assert gy <= c["y"] and c["y"] + c["height"] <= gy + gh, (n, c) + + +# ── Issue #1409: punctuation-only Obsidian/Canvas filenames ─────────────────── + +def _punct_graph(label: str): + """A 2-node graph where one node's label is all-punctuation (e.g. a `@/*` + tsconfig paths key) and the other is a normal symbol.""" + return graph_from_payload( + [ + {"id": "n1", "label": label, "file_type": "code", "source_file": "tsconfig.json"}, + {"id": "n2", "label": "AuthHandler", "file_type": "code", "source_file": "auth.ts"}, + ], + ) + + +def test_to_obsidian_never_emits_punctuation_only_filenames(): + """#1409: an all-punctuation label (e.g. `@/*`) must not produce a `@.md`-style + filename — valid on disk but empty once a downstream tool re-slugs on word chars + (crashes `qmd update`). It falls back to `unnamed`.""" + G = _punct_graph("@/*") + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, communities, tmp) + stems = [p.stem for p in Path(tmp).rglob("*.md")] + assert stems, "to_obsidian wrote no notes" + bad = [s for s in stems if not re.search(r"\w", s, flags=re.UNICODE)] + assert not bad, f"punctuation-only filenames emitted: {bad}" + assert any(s == "unnamed" or s.startswith("unnamed") for s in stems), stems + + +def test_to_canvas_never_emits_punctuation_only_filenames(): + """#1409: same guard on the canvas exporter's file-node names.""" + G = _punct_graph("@") + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.canvas" + to_canvas(G, communities, str(out)) + data = json.loads(out.read_text()) + file_nodes = [n for n in data["nodes"] if n.get("type") == "file"] + assert file_nodes, "canvas has no file nodes" + bad = [n["file"] for n in file_nodes if not re.search(r"\w", Path(n["file"]).stem, flags=re.UNICODE)] + assert not bad, f"punctuation-only canvas filenames: {bad}" + + +# ── Existing-vault safety: graphify must not clobber user notes / .obsidian (#1506) ── + +def _two_node_graph(): + G = graph_from_payload( + [ + {"id": "n1", "label": "Database", "community": 0, "source_file": "app/db.py", "type": "code"}, + {"id": "n2", "label": "Server", "community": 0, "source_file": "app/srv.py", "type": "code"}, + ], + [{"source": "n1", "target": "n2"}], + ) + return G, {0: ["n1", "n2"]} + + +def test_to_obsidian_preserves_existing_user_notes_and_obsidian_config(): + """#1506: exporting into an existing vault must not overwrite a user's note that + collides with a graphify node name, nor their .obsidian/ graph settings.""" + G, communities = _two_node_graph() + with tempfile.TemporaryDirectory() as tmp: + vault = Path(tmp) + (vault / "Database.md").write_text("# MY NOTES\nkeep me\n", encoding="utf-8") + (vault / ".obsidian").mkdir() + (vault / ".obsidian" / "graph.json").write_text('{"USER":"settings"}', encoding="utf-8") + to_obsidian(G, communities, str(vault), community_labels={0: "Backend"}) + # user content untouched + assert "MY NOTES" in (vault / "Database.md").read_text() + assert json.loads((vault / ".obsidian" / "graph.json").read_text()) == {"USER": "settings"} + # non-colliding graphify note still written + assert (vault / "Server.md").exists() + + +def test_to_obsidian_empty_dir_writes_full_vault(): + """No regression: a fresh/empty dir still gets every note + .obsidian/graph.json.""" + G, communities = _two_node_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + n = to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) + assert (out / "Database.md").exists() and (out / "Server.md").exists() + assert (out / ".obsidian" / "graph.json").exists() + assert n == 3 # 2 nodes + 1 community note + + +def test_to_obsidian_rerun_updates_own_notes_but_not_user_files(): + """A re-run overwrites graphify's own prior notes (via the manifest) but leaves a + user-added note in the same dir alone.""" + G, communities = _two_node_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) + (out / "UserNote.md").write_text("mine\n", encoding="utf-8") + to_obsidian(G, communities, str(out), community_labels={0: "Backend2"}) + assert (out / "Database.md").exists() # graphify re-wrote its own + assert (out / "UserNote.md").read_text().strip() == "mine" # user's untouched + + +def _four_node_two_community_graph(): + G = graph_from_payload( + [ + {"id": "n1", "label": "Database", "community": 0, "source_file": "app/db.py", "type": "code"}, + {"id": "n2", "label": "Server", "community": 0, "source_file": "app/srv.py", "type": "code"}, + {"id": "n3", "label": "Cache", "community": 1, "source_file": "infra/cache.py", "type": "code"}, + {"id": "n4", "label": "Queue", "community": 1, "source_file": "infra/queue.py", "type": "code"}, + ], + [{"source": "n1", "target": "n2"}, {"source": "n3", "target": "n4"}], + ) + return G, {0: ["n1", "n2"], 1: ["n3", "n4"]} + + +def test_to_obsidian_rerun_prunes_removed_nodes(): + """#1896: re-exporting into the same vault must delete graphify's own notes for + nodes (and communities) that dropped out of the graph, so the vault mirrors the + current graph rather than old-union-new. User files are never touched.""" + G4, comm4 = _four_node_two_community_graph() + G2, comm2 = _two_node_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + to_obsidian(G4, comm4, str(out), community_labels={0: "Backend", 1: "Infra"}) + assert (out / "Cache.md").exists() and (out / "_COMMUNITY_Infra.md").exists() + (out / "MyOwnNote.md").write_text("mine\n", encoding="utf-8") + to_obsidian(G2, comm2, str(out), community_labels={0: "Backend"}) + # notes for removed nodes and the stale community overview are pruned + assert not (out / "Cache.md").exists() + assert not (out / "Queue.md").exists() + assert not (out / "_COMMUNITY_Infra.md").exists() + # surviving graphify notes and the user's own note remain + assert (out / "Database.md").exists() and (out / "Server.md").exists() + assert (out / "_COMMUNITY_Backend.md").exists() + assert (out / "MyOwnNote.md").read_text().strip() == "mine" + + +def test_to_obsidian_removed_node_returning_is_writable_again(capsys): + """#1896 follow-on: a node that disappears and later returns must be writable + again. Before the fix, the manifest was rewritten to only this run's files, so + the orphaned note was disowned and the returning node's write was skipped as a + 'pre-existing user file' forever.""" + GA, commA = _two_node_graph() + GB = graph_from_payload([ + {"id": "n1", "label": "Database", "community": 0, "source_file": "app/db.py", "type": "code"} + ]) + commB = {0: ["n1"]} + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) + to_obsidian(GB, commB, str(out), community_labels={0: "Backend"}) + assert not (out / "Server.md").exists() # pruned while absent + capsys.readouterr() + to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) + # returned node's note exists with current content, written this run + assert (out / "Server.md").exists() + assert "# Server" in (out / "Server.md").read_text() + captured = capsys.readouterr() + assert "skipped" not in captured.err.lower() + + +# ── Case-only-distinct labels must not collide on case-insensitive filesystems ── + +def _case_collision_graph(): + """Two nodes whose labels differ only by case - on macOS/APFS and Windows/NTFS + their notes resolve to the same path unless the dedup map folds case.""" + return graph_from_payload( + [ + {"id": "n1", "label": "References", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "references", "file_type": "document", "source_file": "b.md"}, + ], + ) + + +def test_to_obsidian_case_only_distinct_labels_dont_overwrite(): + """Both notes must survive as separate files. On a case-insensitive filesystem + a missing suffix silently overwrites the first note (fewer files than nodes); + on a case-sensitive one it writes two stems equal under .lower(). Assert both: + every node note is on disk, and no two stems collide case-insensitively.""" + G = _case_collision_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, communities, tmp) + notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] + assert len(notes) == G.node_count, [p.name for p in notes] + lowered = [p.stem.lower() for p in notes] + assert len(set(lowered)) == len(lowered), [p.name for p in notes] + # the suffixed name must be the expected one, not merely distinct + assert sorted(p.stem for p in notes) == ["References", "references_1"], [p.name for p in notes] + + +def test_to_obsidian_generated_suffix_doesnt_overwrite_literal(): + """A generated `_1` suffix must not collide with a node whose literal label is + already that suffixed name. With labels [dup, dup, dup_1] the second `dup` + becomes `dup_1`, which would clobber the third node unless the candidate is + re-checked. This collides on case-sensitive filesystems too, so it guards the + dedup loop independently of case-folding.""" + G = graph_from_payload( + [ + {"id": "a", "label": "dup", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "dup", "file_type": "code", "source_file": "b.py"}, + {"id": "c", "label": "dup_1", "file_type": "code", "source_file": "c.py"}, + ], + ) + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, communities, tmp) + notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] + assert len(notes) == 3, [p.name for p in notes] + assert len({p.stem.lower() for p in notes}) == 3, [p.name for p in notes] + + +def test_to_canvas_case_only_distinct_labels_get_distinct_files(): + """Canvas file-node references for case-only-distinct labels must be distinct + case-insensitively, else both cards point at one overwritten note.""" + G = _case_collision_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.canvas" + to_canvas(G, communities, str(out)) + data = json.loads(out.read_text()) + files = [n["file"] for n in data["nodes"] if n.get("type") == "file"] + lowered = [f.lower() for f in files] + assert len(set(lowered)) == len(lowered), files + + +def test_obsidian_canvas_filenames_agree(): + """The CLI calls to_obsidian and to_canvas separately with no shared map, so + they must independently produce the same node->filename mapping - otherwise a + canvas card points at a note file that doesn't exist on disk.""" + G = _case_collision_graph() + communities = cluster(G) + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, communities, tmp) + note_stems = {p.stem for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")} + out = Path(tmp) / "graph.canvas" + to_canvas(G, communities, str(out)) + data = json.loads(out.read_text()) + canvas_stems = {Path(n["file"]).stem for n in data["nodes"] if n.get("type") == "file"} + assert canvas_stems <= note_stems, (sorted(canvas_stems), sorted(note_stems)) + + +def test_to_obsidian_community_notes_case_collision(): + """Two community labels differing only by case must each get their own + `_COMMUNITY_*.md` overview note. This path had no dedup at all, so even + same-case duplicate labels previously overwrote silently.""" + G = graph_from_payload( + [ + {"id": "n1", "label": "alpha", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "beta", "file_type": "code", "source_file": "b.py"}, + ], + ) + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "API", 1: "Api"} + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, communities, tmp, community_labels=labels) + comm = [p for p in Path(tmp).rglob("_COMMUNITY_*.md")] + assert len(comm) == 2, [p.name for p in comm] + lowered = [p.stem.lower() for p in comm] + assert len(set(lowered)) == len(lowered), [p.name for p in comm] + + +def test_to_html_handles_null_source_file_and_label(tmp_path): + """Null labels and source paths remain valid presentation-export values.""" + G = graph_from_payload([ + {"id": "n1", "label": "Foo", "source_file": None, "community": 0}, + {"id": "n2", "label": None, "source_file": "a.py", "community": 0}, + {"id": "n3", "label": None, "source_file": None, "community": 0}, + ]) + out = tmp_path / "graph.html" + to_html(G, {0: ["n1", "n2", "n3"]}, str(out)) + assert out.exists() and out.stat().st_size > 0 diff --git a/tests/restored/test_extract_cache_location_restored.py b/tests/restored/test_extract_cache_location_restored.py new file mode 100644 index 000000000..bab597e99 --- /dev/null +++ b/tests/restored/test_extract_cache_location_restored.py @@ -0,0 +1,95 @@ +"""Restored cache-location regressions, adapted to generation-owned state. + +The old tests protected source trees from cache sidecars. Helix strengthens that +contract: extraction receives a caller-owned mapping which is persisted with the +active generation, and ``cache_root`` must not create anything on disk. +""" + +from graphify.cache import file_hash, load_cached +from graphify.extract import extract + + +def _make_corpus(tmp_path): + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text("def a():\n return 1\n") + (corpus / "b.py").write_text("def b():\n return a()\n") + return corpus + + +def test_default_cache_lands_in_cwd_not_source_tree(tmp_path, monkeypatch): + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + state_cache = {} + + result = extract( + [corpus / "a.py", corpus / "b.py"], root=corpus, + cache=state_cache, parallel=False, + ) + + assert result["nodes"] + assert len(state_cache) == 2 + assert not (corpus / "graphify-out").exists() + assert not (work / "graphify-out").exists() + + +def test_default_cache_does_not_leave_stat_index_in_source_tree(tmp_path, monkeypatch): + corpus = _make_corpus(tmp_path) + work = tmp_path / "elsewhere" + work.mkdir() + monkeypatch.chdir(work) + + extract([corpus / "a.py"], root=corpus, cache={}, parallel=False) + + assert not list(corpus.rglob("stat-index.json")) + assert not list(work.rglob("stat-index.json")) + + +def test_explicit_cache_root_still_wins(tmp_path, monkeypatch): + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + deprecated_location = tmp_path / "old-sidecar-location" + monkeypatch.chdir(work) + state_cache = {} + + extract( + [corpus / "a.py"], root=corpus, cache_root=deprecated_location, + cache=state_cache, parallel=False, + ) + + assert state_cache + assert not deprecated_location.exists() + assert not (corpus / "graphify-out").exists() + + +def test_default_cache_round_trips_via_extract(tmp_path, monkeypatch): + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + state_cache = {} + + first = extract([corpus / "a.py"], root=corpus, cache=state_cache, parallel=False) + hit = load_cached(corpus / "a.py", corpus, cache=state_cache) + second = extract([corpus / "a.py"], root=corpus, cache=state_cache, parallel=False) + + assert hit is not None + assert second == first + + +def test_cache_keys_stay_relative_for_out_of_cwd_corpus(tmp_path, monkeypatch): + corpus = _make_corpus(tmp_path) + work = tmp_path / "elsewhere" / "work" + work.mkdir(parents=True) + monkeypatch.chdir(work) + state_cache = {} + + extract([corpus / "a.py"], root=corpus, cache=state_cache, parallel=False) + + key = next(iter(state_cache)) + assert key.endswith(":a.py") + assert str(corpus) not in key + assert file_hash(corpus / "a.py", corpus) == file_hash(corpus / "a.py", work) diff --git a/tests/restored/test_extract_cli_restored.py b/tests/restored/test_extract_cli_restored.py new file mode 100644 index 000000000..72ade9c9a --- /dev/null +++ b/tests/restored/test_extract_cli_restored.py @@ -0,0 +1,412 @@ +"""Restored v8 extraction regressions against native Helix state.""" + +from __future__ import annotations + +import copy +import os +from pathlib import Path + +import pytest + +import graphify.__main__ as mainmod +from graphify.cache import check_semantic_cache, save_semantic_cache +from graphify.helix.model import node_attributes +from graphify.helix.persistence import HelixEmbeddedStore, load_graph + + +def _run(monkeypatch, argv: list[str]) -> None: + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + mainmod.main() + + +def _make_corpus(tmp_path: Path, *, second_doc: bool = False) -> Path: + root = tmp_path / "project" + root.mkdir() + (root / "main.py").write_text("def main():\n return 1\n") + (root / "README.md").write_text("# Readme\nNative storage.\n") + if second_doc: + (root / "OTHER.md").write_text("# Other\nSecond document.\n") + return root + + +def _semantic(paths, **kwargs): + return { + "nodes": [ + { + "id": f"semantic-{Path(path).stem}", + "label": Path(path).stem, + "file_type": "document", + "source_file": Path(path).name, + } + for path in paths + ], + "edges": [], "hyperedges": [], "input_tokens": 10, + "output_tokens": 5, "failed_chunks": 0, + } + + +def _extract_argv(project: Path, output: Path, *extra: str) -> list[str]: + return [ + "graphify", "extract", str(project), "--backend", "openai", + "--out", str(output), *extra, + ] + + +def _store(output: Path) -> Path: + return output / "graphify-out" / "graph.helix" + + +def _sources(store: Path) -> set[str]: + graph = load_graph(store).graph + return { + str(node_attributes(graph, node.id).get("source_file", "")) + for node in graph.nodes() + } + + +def _semantic_cache(store: Path) -> dict: + return load_graph(store).state["incremental"]["extraction_cache"] + + +def test_extract_exits_nonzero_when_all_semantic_chunks_fail(monkeypatch, tmp_path, capsys): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr( + "graphify.llm.extract_corpus_parallel", + lambda *a, **k: { + "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, + "output_tokens": 0, "failed_chunks": 2, + }, + ) + + with pytest.raises(SystemExit) as exc: + _run(monkeypatch, _extract_argv(project, output)) + + assert exc.value.code == 1 + assert "active generation was left unchanged" in capsys.readouterr().err + assert not _store(output).exists() + + +def test_extract_succeeds_when_at_least_one_chunk_completes(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic) + + _run(monkeypatch, _extract_argv(project, output)) + + loaded = load_graph(_store(output)) + assert loaded.graph.node_count > 0 + assert loaded.state["semantic"]["input_tokens"] == 10 + + +def test_incremental_partial_run_preserves_untouched_semantic_hash(monkeypatch, tmp_path): + project = _make_corpus(tmp_path, second_doc=True) + output = tmp_path / "out" + dispatched: list[list[str]] = [] + + def semantic(paths, **kwargs): + dispatched.append(sorted(Path(path).name for path in paths)) + return _semantic(paths, **kwargs) + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", semantic) + _run(monkeypatch, _extract_argv(project, output)) + before = copy.deepcopy(_semantic_cache(_store(output))) + other_key = next(key for key in before if key.endswith(":OTHER.md")) + (project / "README.md").write_text("# Readme\nChanged.\n") + + _run(monkeypatch, _extract_argv(project, output)) + + after = _semantic_cache(_store(output)) + assert dispatched[-1] == ["README.md"] + assert after[other_key] == before[other_key] + + +def test_truncated_doc_semantic_hash_is_cleared_for_requeue(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + calls = 0 + + def semantic(paths, **kwargs): + nonlocal calls + calls += 1 + result = _semantic(paths, **kwargs) + if calls == 2: + for node in result["nodes"]: + node["_partial"] = True + return result + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", semantic) + _run(monkeypatch, _extract_argv(project, output)) + (project / "README.md").write_text("# Readme\nChanged.\n") + _run(monkeypatch, _extract_argv(project, output)) + partial = _semantic_cache(_store(output)) + readme_key = next( + key for key in partial + if key.startswith("semantic:") and key.endswith(":README.md") + ) + assert partial[readme_key]["partial"] is True + + _run(monkeypatch, _extract_argv(project, output)) + assert calls == 3 + assert _semantic_cache(_store(output))[readme_key]["partial"] is False + + +def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic) + _run(monkeypatch, _extract_argv(project, output)) + + loaded = load_graph(_store(output)) + assert loaded.state["incremental"]["files"]["README.md"]["content_hash"] + assert any(key.endswith(":README.md") for key in _semantic_cache(_store(output))) + assert not (_store(output).parent / "manifest.json").exists() + + +def test_stamped_manifest_files_normalizes_both_sides(tmp_path): + doc = tmp_path / "docs" / "Guide.md" + doc.parent.mkdir() + doc.write_text("# Guide") + state = {} + saved = save_semantic_cache( + [{"id": "guide", "source_file": str(doc)}], [], root=tmp_path, + allowed_source_files=["docs/Guide.md"], cache=state, + ) + assert saved == 1 + assert next(iter(state)).endswith(":docs/Guide.md") + + +def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): + doc = tmp_path / "relationships.md" + doc.write_text("# Relationships") + state = {} + saved = save_semantic_cache( + [], [], [{"id": "h", "nodes": ["a", "b"], "source_file": "relationships.md"}], + root=tmp_path, allowed_source_files=[doc], cache=state, + ) + assert saved == 1 + result = next(iter(state.values()))["result"] + assert result["nodes"] == [] and len(result["hyperedges"]) == 1 + + +def test_manifest_stamps_hyperedge_only_docs(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + doc = project / "README.md" + state = {} + save_semantic_cache( + [], [], [{"id": "h", "nodes": ["a", "b"], "source_file": doc.name}], + root=project, cache=state, + ) + nodes, edges, hyperedges, uncached = check_semantic_cache( + [str(doc)], state, root=project + ) + assert not nodes and not edges and not uncached + assert [edge["id"] for edge in hyperedges] == ["h"] + + +def test_extract_mode_deep_dispatches_over_warm_cache(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + calls: list[bool] = [] + + def semantic(paths, **kwargs): + calls.append(bool(kwargs.get("deep_mode"))) + return _semantic(paths, **kwargs) + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", semantic) + _run(monkeypatch, _extract_argv(project, output)) + _run(monkeypatch, _extract_argv(project, output, "--mode", "deep")) + assert calls == [False, True] + + +def test_extract_force_flag_redispatches_and_stamps_manifest(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + calls = 0 + + def semantic(paths, **kwargs): + nonlocal calls + calls += 1 + return _semantic(paths, **kwargs) + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", semantic) + _run(monkeypatch, _extract_argv(project, output)) + _run(monkeypatch, _extract_argv(project, output, "--force")) + assert calls == 2 + assert any(key.endswith(":README.md") for key in _semantic_cache(_store(output))) + + +def test_extract_graphify_force_env_redispatches(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + calls = 0 + + def semantic(paths, **kwargs): + nonlocal calls + calls += 1 + return _semantic(paths, **kwargs) + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", semantic) + _run(monkeypatch, _extract_argv(project, output)) + monkeypatch.setenv("GRAPHIFY_FORCE", "1") + _run(monkeypatch, _extract_argv(project, output)) + assert calls == 2 + + +def test_cache_check_mode_deep_reads_deep_namespace(monkeypatch, tmp_path, capsys): + project = _make_corpus(tmp_path) + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic) + _run(monkeypatch, _extract_argv(project, project, "--mode", "deep")) + files = tmp_path / "files.txt" + files.write_text(str(project / "README.md") + "\n") + + _run(monkeypatch, [ + "graphify", "cache-check", str(files), "--root", str(project), "--deep", + ]) + assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out + + +def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr("graphify.llm.detect_backend", lambda: None) + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--out", str(output), + ]) + assert load_graph(_store(output)).graph.node_count > 0 + + +def test_missing_manifest_code_only_preserves_semantic_layer(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic) + _run(monkeypatch, _extract_argv(project, output)) + assert any("README.md" in source for source in _sources(_store(output))) + + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--out", str(output), + ]) + assert any("README.md" in source for source in _sources(_store(output))) + + +def test_extract_out_keeps_project_root_clean(monkeypatch, tmp_path): + project = _make_corpus(tmp_path) + output = tmp_path / "elsewhere" + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic) + _run(monkeypatch, _extract_argv(project, output)) + assert _store(output).is_dir() + assert not (project / "graphify-out").exists() + + +def test_extract_without_key_still_errors_when_docs_present(monkeypatch, tmp_path, capsys): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + monkeypatch.setattr("graphify.llm.detect_backend", lambda: None) + with pytest.raises(SystemExit) as exc: + _run(monkeypatch, ["graphify", "extract", str(project), "--out", str(output)]) + assert exc.value.code == 1 + assert "no LLM backend is configured" in capsys.readouterr().err + assert not _store(output).exists() + + +def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): + project = _make_corpus(tmp_path) + output = tmp_path / "out" + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--no-cluster", + "--out", str(output), "--timing", + ]) + err = capsys.readouterr().err + assert "[graphify timing] detect:" in err + assert "[graphify timing] total:" in err + + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--no-cluster", + "--out", str(tmp_path / "without-timing"), + ]) + assert "graphify timing" not in capsys.readouterr().err + + +def _two_file_corpus(tmp_path): + project = tmp_path / "project" + project.mkdir() + (project / "x.py").write_text("def secret():\n return 42\n") + (project / "keep.py").write_text("def kept():\n return 1\n") + return project + + +def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest(monkeypatch, tmp_path): + project = _two_file_corpus(tmp_path) + output = tmp_path / "out" + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--out", str(output), + ]) + (project / ".graphifyignore").write_text("x.py\n") + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--out", str(output), + ]) + sources = _sources(_store(output)) + assert not any(source.endswith("x.py") for source in sources) + assert any(source.endswith("keep.py") for source in sources) + + +def test_incremental_extract_prunes_excluded_file_listed_in_manifest(monkeypatch, tmp_path): + project = _two_file_corpus(tmp_path) + output = tmp_path / "out" + argv = ["graphify", "extract", str(project), "--code-only", "--out", str(output)] + _run(monkeypatch, argv) + assert "x.py" in load_graph(_store(output)).state["incremental"]["files"] + (project / ".graphifyignore").write_text("x.py\n") + _run(monkeypatch, argv) + _run(monkeypatch, argv) + loaded = load_graph(_store(output)) + assert "x.py" not in loaded.state["incremental"]["files"] + assert not any(source.endswith("x.py") for source in _sources(_store(output))) + + +def test_no_cluster_incremental_prunes_newly_excluded_file(monkeypatch, tmp_path, capsys): + project = _two_file_corpus(tmp_path) + output = tmp_path / "out" + argv = [ + "graphify", "extract", str(project), "--code-only", "--no-cluster", + "--out", str(output), + ] + _run(monkeypatch, argv) + capsys.readouterr() + (project / ".graphifyignore").write_text("x.py\n") + _run(monkeypatch, argv) + assert "deleted" not in capsys.readouterr().out + assert not any(source.endswith("x.py") for source in _sources(_store(output))) + + +def test_cache_check_prompt_file_scopes_hits_to_that_prompt(monkeypatch, tmp_path, capsys): + project = _make_corpus(tmp_path) + _run(monkeypatch, [ + "graphify", "extract", str(project), "--code-only", "--out", str(project), + ]) + store_path = project / "graphify-out" / "graph.helix" + loaded = load_graph(store_path) + state = copy.deepcopy(dict(loaded.state)) + cache = state["incremental"]["extraction_cache"] + spec = tmp_path / "extraction-spec.md" + spec.write_text("PROMPT V1") + save_semantic_cache( + [{"id": "d", "source_file": "README.md"}], [], root=project, + prompt_file=spec, cache=cache, + ) + with HelixEmbeddedStore(store_path) as store: + store.replace_state(state, previous_state=loaded.state) + files = tmp_path / "files.txt" + files.write_text(str(project / "README.md") + "\n") + base = [ + "graphify", "cache-check", str(files), "--root", str(project), + "--prompt-file", str(spec), + ] + _run(monkeypatch, base) + assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out + + spec.write_text("PROMPT V2") + os.utime(spec, ns=(0, 0)) + _run(monkeypatch, base) + assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out diff --git a/tests/restored/test_global_graph_restored.py b/tests/restored/test_global_graph_restored.py new file mode 100644 index 000000000..97a035c4e --- /dev/null +++ b/tests/restored/test_global_graph_restored.py @@ -0,0 +1,217 @@ +"""Retained global-graph regressions, adapted to native Helix stores.""" +from __future__ import annotations + +import pytest + +from graphify.build import prefix_graph_for_global, prune_repo_from_graph +from graphify.dedup import deduplicate_entities +from graphify.global_graph import aggregate, global_add, global_list, global_remove +from graphify.helix.model import GraphBuildData +from graphify.helix.persistence import load_graph +from tests.native_helpers import make_loaded + + +def _make_graph(nodes, edges=None): + return GraphBuildData.from_node_link({ + "directed": False, + "multigraph": False, + "graph": {}, + "nodes": nodes, + "links": edges or [], + }) + + +def _source(root, dirname, nodes, edges=None): + project = root / dirname / "graphify-out" + project.mkdir(parents=True, exist_ok=True) + return make_loaded(project, nodes=nodes, edges=edges or []).store_path + + +def _destination(tmp_path, monkeypatch): + destination = tmp_path / "global.helix" + monkeypatch.setattr("graphify.global_graph.DEFAULT_GLOBAL_STORE", destination) + return destination + + +def test_prefix_graph_preserves_label(): + graph = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) + prefixed = prefix_graph_for_global(graph, "repoA") + assert [node.id for node in prefixed.nodes] == ["repoA::userservice"] + assert prefixed.nodes[0].attributes["label"] == "UserService" + + +def test_prefix_graph_sets_repo_and_local_id(): + prefixed = prefix_graph_for_global( + _make_graph([{"id": "userservice", "label": "UserService"}]), "repoA" + ) + attrs = prefixed.nodes[0].attributes + assert attrs["repo"] == "repoA" + assert attrs["local_id"] == "userservice" + + +def test_prefix_graph_rewrites_edges(): + prefixed = prefix_graph_for_global( + _make_graph( + [{"id": "a", "label": "A"}, {"id": "b", "label": "B"}], + [{"source": "a", "target": "b"}], + ), + "repo1", + ) + assert [(edge.source, edge.target) for edge in prefixed.edges] == [ + ("repo1::a", "repo1::b") + ] + + +def test_prune_repo_removes_correct_nodes(): + graph = _make_graph( + [ + {"id": "repoA::userservice", "repo": "repoA", "label": "UserService"}, + {"id": "repoB::userservice", "repo": "repoB", "label": "UserService"}, + {"id": "repoA::auth", "repo": "repoA", "label": "Auth"}, + ] + ) + assert prune_repo_from_graph(graph, "repoA") == 2 + assert [node.id for node in graph.nodes] == ["repoB::userservice"] + + +def test_prune_repo_returns_zero_if_not_present(): + graph = _make_graph([{"id": "repoA::x", "repo": "repoA"}]) + assert prune_repo_from_graph(graph, "repoB") == 0 + assert graph.node_count == 1 + + +def test_global_add_creates_global_graph(tmp_path, monkeypatch): + destination = _destination(tmp_path, monkeypatch) + source = _source( + tmp_path, + "source", + [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], + ) + result = global_add(source, "repoA") + assert result["skipped"] is False + assert result["nodes_added"] == 1 + assert destination.is_dir() + assert "repoA" in global_list() + + +def test_global_add_skip_on_unchanged_hash(tmp_path, monkeypatch): + _destination(tmp_path, monkeypatch) + source = _source( + tmp_path, + "source", + [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], + ) + global_add(source, "repoA") + assert global_add(source, "repoA")["skipped"] is True + + +def test_global_add_two_repos_no_collision(tmp_path, monkeypatch): + destination = _destination(tmp_path, monkeypatch) + node = [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}] + global_add(_source(tmp_path, "one", node), "repoA") + global_add(_source(tmp_path, "two", node), "repoB") + graph = load_graph(destination).graph + assert {record.id for record in graph.nodes()} == { + "repoA::userservice", + "repoB::userservice", + } + + +def test_global_remove(tmp_path, monkeypatch): + _destination(tmp_path, monkeypatch) + source = _source( + tmp_path, + "source", + [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], + ) + global_add(source, "repoA") + assert global_remove("repoA") == 1 + assert "repoA" not in global_list() + + +def test_global_remove_unknown_tag_raises(tmp_path, monkeypatch): + _destination(tmp_path, monkeypatch) + with pytest.raises(KeyError): + global_remove("nonexistent") + + +def test_global_add_replaces_same_tag_from_new_source(tmp_path, monkeypatch): + destination = _destination(tmp_path, monkeypatch) + first = _source(tmp_path, "one", [{"id": "x", "label": "X", "source_file": "x.py"}]) + second = _source(tmp_path, "two", [{"id": "y", "label": "Y", "source_file": "y.py"}]) + global_add(first, "myrepo") + result = global_add(second, "myrepo") + assert result["nodes_removed"] == 1 and result["nodes_added"] == 1 + assert {node.id for node in load_graph(destination).graph.nodes()} == {"myrepo::y"} + + +def test_dedup_raises_on_cross_repo_nodes(): + nodes = [ + {"id": "repoA::userservice", "label": "UserService", "repo": "repoA"}, + {"id": "repoB::userservice", "label": "UserService", "repo": "repoB"}, + ] + with pytest.raises(ValueError, match="multiple repos"): + deduplicate_entities(nodes, [], communities={}) + + +def test_dedup_ok_with_single_repo(): + nodes = [ + {"id": "repoA::userservice", "label": "UserService", "repo": "repoA"}, + {"id": "repoA::auth", "label": "Auth", "repo": "repoA"}, + ] + result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 2 + assert result_edges == [] + + +def test_dedup_ok_with_no_repo_attr(): + nodes = [ + {"id": "userservice", "label": "UserService"}, + {"id": "auth", "label": "Auth"}, + ] + result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 2 + assert result_edges == [] + + +def test_aggregate_prefixes_duplicate_ids(tmp_path): + node = [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}] + first = _source(tmp_path, "repo1", node) + second = _source(tmp_path, "repo2", node) + destination = tmp_path / "aggregate.helix" + aggregate([first, second], destination) + assert {record.id for record in load_graph(destination).graph.nodes()} == { + "repo1::userservice", + "repo2::userservice", + } + + +def test_global_add_rewires_edges_to_deduplicated_externals(tmp_path, monkeypatch): + destination = _destination(tmp_path, monkeypatch) + source_a = _source( + tmp_path, + "one", + [ + {"id": "moda", "label": "ModA", "source_file": "src/a.py"}, + {"id": "requests", "label": "requests"}, + ], + [{"source": "moda", "target": "requests", "relation": "imports"}], + ) + source_b = _source( + tmp_path, + "two", + [ + {"id": "modb", "label": "ModB", "source_file": "src/b.py"}, + {"id": "requests", "label": "requests"}, + ], + [{"source": "modb", "target": "requests", "relation": "imports"}], + ) + global_add(source_a, "repoA") + global_add(source_b, "repoB") + graph = load_graph(destination).graph + ids = {node.id for node in graph.nodes()} + externals = {node_id for node_id in ids if str(node_id).startswith("external::")} + assert len(externals) == 1 + external = next(iter(externals)) + assert graph.edges_between("repoA::moda", external) + assert graph.edges_between("repoB::modb", external) diff --git a/tests/restored/test_hooks_restored.py b/tests/restored/test_hooks_restored.py new file mode 100644 index 000000000..35a3cbf80 --- /dev/null +++ b/tests/restored/test_hooks_restored.py @@ -0,0 +1,530 @@ +"""Tests for hooks.py - git hook install/uninstall.""" +import os +import shutil +import subprocess +from types import SimpleNamespace +from pathlib import Path +import pytest +from graphify.hooks import install, uninstall, status, _hooks_dir, _HOOK_MARKER, _CHECKOUT_MARKER +from tests.native_helpers import make_loaded + + +def _make_git_repo(tmp_path: Path) -> Path: + subprocess.run(["git", "init", str(tmp_path)], check=True, capture_output=True) + return tmp_path + + +def test_install_creates_hook(tmp_path): + repo = _make_git_repo(tmp_path) + result = install(repo) + hook = repo / ".git" / "hooks" / "post-commit" + assert hook.exists() + assert _HOOK_MARKER in hook.read_text() + assert "installed" in result + + +def test_install_is_executable(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + hook = repo / ".git" / "hooks" / "post-commit" + if os.name == "nt": + assert hook.read_text(encoding="utf-8").startswith("#!/bin/sh\n") + else: + assert hook.stat().st_mode & 0o111 # executable bit set + + +def test_install_idempotent(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + result = install(repo) + assert "already installed" in result + # marker appears only once + hook = repo / ".git" / "hooks" / "post-commit" + assert hook.read_text().count(_HOOK_MARKER) == 1 + + +def test_install_appends_to_existing_hook(tmp_path): + repo = _make_git_repo(tmp_path) + hook = repo / ".git" / "hooks" / "post-commit" + hook.write_text("#!/bin/bash\necho existing\n") + hook.chmod(0o755) + install(repo) + content = hook.read_text() + assert "existing" in content + assert _HOOK_MARKER in content + + +def test_uninstall_removes_hook(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + result = uninstall(repo) + hook = repo / ".git" / "hooks" / "post-commit" + assert not hook.exists() + assert "removed" in result.lower() + + +def test_uninstall_no_hook(tmp_path): + repo = _make_git_repo(tmp_path) + result = uninstall(repo) + assert "nothing to remove" in result + + +def test_status_installed(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + result = status(repo) + assert "installed" in result + + +def test_status_not_installed(tmp_path): + repo = _make_git_repo(tmp_path) + result = status(repo) + assert "not installed" in result + + +def test_no_git_repo_raises(tmp_path): + with pytest.raises(RuntimeError, match="No git repository"): + install(tmp_path / "not_a_repo") + + +def test_install_creates_post_checkout_hook(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + hook = repo / ".git" / "hooks" / "post-checkout" + assert hook.exists() + assert _CHECKOUT_MARKER in hook.read_text() + + +def test_install_post_checkout_is_executable(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + hook = repo / ".git" / "hooks" / "post-checkout" + if os.name == "nt": + assert hook.read_text(encoding="utf-8").startswith("#!/bin/sh\n") + else: + assert hook.stat().st_mode & 0o111 + + +def test_uninstall_removes_post_checkout_hook(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + uninstall(repo) + hook = repo / ".git" / "hooks" / "post-checkout" + assert not hook.exists() + + +def test_status_shows_both_hooks(tmp_path): + repo = _make_git_repo(tmp_path) + install(repo) + result = status(repo) + assert "post-commit" in result + assert "post-checkout" in result + assert result.count("installed") >= 2 + + + +def test_hooks_dir_resolves_relative_git_hooks_path(tmp_path, monkeypatch): + repo = _make_git_repo(tmp_path) + + def fake_run(*args, **kwargs): + return SimpleNamespace(returncode=0, stdout=".git/hooks\n") + + monkeypatch.setattr("subprocess.run", fake_run) + + assert _hooks_dir(repo) == (repo / ".git" / "hooks").resolve() + + +def test_hooks_dir_rejects_multiline_git_output(tmp_path, monkeypatch): + repo = _make_git_repo(tmp_path) + + def fake_run(*args, **kwargs): + return SimpleNamespace(returncode=0, stdout="--path-format=absolute\n.git/hooks\n") + + monkeypatch.setattr("subprocess.run", fake_run) + + assert _hooks_dir(repo) == repo / ".git" / "hooks" + assert not (repo / "--path-format=absolute\n.git").exists() + + +def test_hooks_dir_accepts_absolute_git_hooks_path(tmp_path, monkeypatch): + repo = _make_git_repo(tmp_path) + hooks = tmp_path / "actual-hooks" + + def fake_run(*args, **kwargs): + return SimpleNamespace(returncode=0, stdout=f"{hooks}\n") + + monkeypatch.setattr("subprocess.run", fake_run) + + assert _hooks_dir(repo) == hooks.resolve() + +def test_hook_skips_head_on_exe(): + """Hook script must skip shebang extraction for .exe binaries (Windows).""" + from graphify.hooks import _PYTHON_DETECT + assert "*.exe) _SHEBANG=" in _PYTHON_DETECT or '*.exe)' in _PYTHON_DETECT + + +def test_install_embeds_pinned_interpreter(tmp_path): + """Hook scripts must embed sys.executable so the hook works without the + graphify launcher on PATH (uv tool / pipx isolation, #1127). + + When graphify is installed via `uv tool install graphifyy` or `pipx install + graphifyy`, the interpreter lives in an isolated venv and the launcher is in + ~/.local/bin. GUI git clients and CI runners often run with a minimal PATH + that omits that directory, so `command -v graphify` fails, the python3/python + fallbacks cannot import graphify (wrong venv), and the hook silently exits 0. + Pinning sys.executable at install time makes the hook work regardless of PATH. + """ + import re, sys + repo = _make_git_repo(tmp_path) + install(repo) + commit_hook = (repo / ".git" / "hooks" / "post-commit").read_text() + checkout_hook = (repo / ".git" / "hooks" / "post-checkout").read_text() + # Compute the sanitized value the same way install() does. + expected = sys.executable if not re.search(r"[^a-zA-Z0-9/_.@:\\-]", sys.executable) else "" + if expected: + assert expected in commit_hook, "sanitized sys.executable missing from post-commit" + assert expected in checkout_hook, "sanitized sys.executable missing from post-checkout" + # The placeholder must be fully substituted -- no __PINNED_PYTHON__ left. + assert "__PINNED_PYTHON__" not in commit_hook, "placeholder not substituted in post-commit" + assert "__PINNED_PYTHON__" not in checkout_hook, "placeholder not substituted in post-checkout" + + +def test_install_fallback_is_loud_not_silent(tmp_path): + """The detection fallback must emit a message to stderr rather than bare exit 0. + + A silent no-op (the pre-fix behaviour) leaves the user with no indication + that the hook ran but found nothing, making the bug extremely hard to diagnose. + """ + from graphify.hooks import _PYTHON_DETECT + assert "could not locate" in _PYTHON_DETECT, ( + "fallback branch must print a diagnostic message; bare 'exit 0' is silent and unhelpful" + ) + + +def test_hook_check_no_additionalContext(tmp_path): + """graphify hook-check must not emit additionalContext — Codex Desktop rejects it.""" + import sys + make_loaded(tmp_path / "graphify-out", nodes=[]) + + result = subprocess.run( + [sys.executable, "-m", "graphify", "hook-check"], + cwd=tmp_path, + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert result.stdout == "" + assert result.stderr == "" + + +# ── #1161: background rebuild must not rely on nohup (missing on Git for Windows) ── + +import ast # noqa: E402 +import re # noqa: E402 + +from graphify.hooks import ( # noqa: E402 + _HOOK_SCRIPT, + _CHECKOUT_SCRIPT, + _REBUILD_BODY_COMMIT, + _REBUILD_BODY_CHECKOUT, + _detached_launch, +) + +_HOOK_SCRIPTS = [("post-commit", _HOOK_SCRIPT), ("post-checkout", _CHECKOUT_SCRIPT)] + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_do_not_use_nohup(name, script): + """Git for Windows' bundled shell ships no `nohup`/`setsid`, so the old + `nohup ... &` launch died with 'nohup: command not found' and the rebuild + silently never ran (#1161). The generated hooks must not reference either.""" + assert "nohup" not in script, f"{name} still references nohup (#1161)" + assert "setsid" not in script, f"{name} still references setsid (#1161)" + assert "disown" not in script, f"{name} still uses disown (#1161)" + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_use_cross_platform_detach(name, script): + """The replacement detaches via Python: start_new_session on POSIX and + DETACHED_PROCESS|CREATE_NEW_PROCESS_GROUP on Windows (#1161).""" + assert "subprocess.Popen" in script + assert "start_new_session=True" in script, f"{name} missing POSIX detach" + assert "0x00000008" in script, f"{name} missing Windows DETACHED_PROCESS flag" + assert "0x00000200" in script, f"{name} missing CREATE_NEW_PROCESS_GROUP flag" + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_limit_windows_workers_by_default(name, script): + """Git for Windows/MSYS hooks can expose fragile pipe handles to spawned + ProcessPoolExecutor children. Hook-triggered rebuilds should default to one + worker there, while still allowing explicit user overrides.""" + assert '[ -n "${WINDIR:-}" ] || [ -n "${MSYSTEM:-}" ]' in script + assert 'export GRAPHIFY_MAX_WORKERS="${GRAPHIFY_MAX_WORKERS:-1}"' in script + + +def _launcher_payload(script: str) -> str: + """Extract the `python -c ""` the hook hands to GRAPHIFY_PYTHON. + + The launcher is the only `-c` invocation whose body begins with + `import os, subprocess, sys` (the interpreter-detection probes in + _PYTHON_DETECT use `-c "$_GFY_PROBE"`).""" + m = re.search(r'-c "(import os, subprocess, sys.*?)"\n', script, re.DOTALL) + assert m, "launcher payload not found" + return m.group(1) + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_launcher_payload_is_shell_quote_safe(name, script): + """The launcher is carried inside a shell double-quoted `-c "..."` argument, + so it must contain no characters the shell would interpret there: an + unescaped double-quote, $, backtick or backslash would corrupt the hook.""" + payload = _launcher_payload(script) + for bad in ('"', "$", "`", "\\"): + assert bad not in payload, f"{name} launcher payload contains unsafe {bad!r}" + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_launcher_and_rebuild_body_are_valid_python(name, script): + """Both the launcher and the rebuild body it re-executes must parse, so a + quoting slip can't ship a hook that crashes the moment git fires it.""" + payload = _launcher_payload(script) + ast.parse(payload) # launcher itself + inner = re.search(r"_src = '''(.*?)'''", payload, re.DOTALL) + assert inner, f"{name}: embedded rebuild body not found" + ast.parse(inner.group(1)) # the detached child's source + + +def test_rebuild_bodies_are_shell_quote_safe(): + """The shared rebuild bodies are embedded verbatim into the launcher, so they + too must avoid characters unsafe inside a shell double-quoted argument.""" + for body in (_REBUILD_BODY_COMMIT, _REBUILD_BODY_CHECKOUT): + for bad in ('"', "$", "`", "\\"): + assert bad not in body + assert "'''" not in body # would terminate the launcher's _src literal + + +@pytest.mark.parametrize( + "name,body", + [("post-commit", _REBUILD_BODY_COMMIT), ("post-checkout", _REBUILD_BODY_CHECKOUT)], +) +def test_rebuild_bodies_read_graphify_root(name, body): + """The rebuild must honour the persisted scan root rather than hardcoding the + repo top (#1173). Both bodies read /.graphify_root and pass the + recovered root to _rebuild_code instead of the bare Path('.').""" + assert ".graphify_root" in body, f"{name} ignores .graphify_root (#1173)" + # The output dir is resolved from GRAPHIFY_OUT at hook-run time, not hardcoded + # to graphify-out/, so a renamed output dir is still found (#1423). + assert "GRAPHIFY_OUT" in body, f"{name} ignores the GRAPHIFY_OUT override (#1423)" + # The recovered root is what gets rebuilt, not a hardcoded cwd. + assert "_rebuild_code(_root" in body, f"{name} does not pass the recovered root" + # Quote-safe inside the shell-double-quoted launcher: single quotes only. + assert "read_text(encoding='utf-8')" in body, f"{name} root read is not single-quoted" + + +def test_rebuild_bodies_with_graphify_root_are_valid_python(): + """The .graphify_root snippet must parse so a quoting slip can't ship a hook + that crashes the moment git fires it (#1173).""" + for body in (_REBUILD_BODY_COMMIT, _REBUILD_BODY_CHECKOUT): + ast.parse(body) + + +def test_detached_launch_targets_graphify_python(): + """The launcher must run via the resolved $GRAPHIFY_PYTHON, not a bare + `python`, so it uses the same interpreter the detection block selected.""" + snippet = _detached_launch(_REBUILD_BODY_COMMIT) + assert snippet.startswith('"$GRAPHIFY_PYTHON" -c "') + assert "nohup" not in snippet + + +def test_installed_hooks_contain_no_nohup(tmp_path): + """End-to-end: the files written to .git/hooks must be nohup-free (#1161).""" + repo = _make_git_repo(tmp_path) + install(repo) + for name in ("post-commit", "post-checkout"): + text = (repo / ".git" / "hooks" / name).read_text(encoding="utf-8") + assert "nohup" not in text, f"installed {name} still references nohup" + assert "start_new_session=True" in text + + +# ── #1385: reject Windows-style hooks paths instead of creating a junk dir ─── + +def _set_hookspath(repo: Path, value: str) -> None: + subprocess.run(["git", "-C", str(repo), "config", "--local", "core.hooksPath", value], + check=True, capture_output=True) + + +@pytest.mark.parametrize("winpath", [ + r"C:\Users\u\repo\.git\hooks", + r"c:/Users/u/.git/hooks", + r"D:\hooks", + r"some\back\slashed\path", +]) +def test_windows_hookspath_rejected_no_junk_dir_on_posix(tmp_path, monkeypatch, winpath): + """A Windows-style core.hooksPath must raise (loud failure), not silently + create a backslash-named junk directory and report success on POSIX/WSL (#1385).""" + monkeypatch.setattr("graphify.hooks.os.name", "posix") + repo = _make_git_repo(tmp_path) + _set_hookspath(repo, winpath) + with pytest.raises(RuntimeError, match="Windows path"): + install(repo) + # no junk directory got created anywhere under the repo + junk = [p for p in repo.rglob("*") if "\\" in p.name or p.name.startswith(("C:", "c:", "D:"))] + assert junk == [], f"junk dir created: {junk}" + + +def test_posix_custom_hookspath_still_works(tmp_path): + """A legitimate POSIX core.hooksPath (Husky-style) must still install.""" + repo = _make_git_repo(tmp_path) + _set_hookspath(repo, ".husky") + msg = install(repo) + assert "post-commit" in msg + assert (repo / ".husky" / "post-commit").exists() + + +def test_default_hooks_dir_unaffected(tmp_path): + """No core.hooksPath -> normal .git/hooks install, no rejection.""" + repo = _make_git_repo(tmp_path) + install(repo) + assert (repo / ".git" / "hooks" / "post-commit").exists() + + +# ── foreground hook cost: probes must be cheap and quiet ───────────────────── + +def test_probes_use_find_spec_not_full_import(): + """`python -c "import graphify"` executes the FULL package import — 10s+ on a + cold cache or AV-scanned site-packages — and could run up to four times + synchronously before the detached launch even started, so every commit + stalled for tens of seconds. Probes must locate the package with + importlib.util.find_spec (no execution); the detached rebuild still reports + a broken install loudly in its log.""" + from graphify.hooks import _PYTHON_DETECT + assert '-c "import graphify"' not in _PYTHON_DETECT, ( + "interpreter probe still imports the full package in the hook foreground" + ) + assert "find_spec" in _PYTHON_DETECT + + +def test_shebang_read_is_null_byte_safe(): + """On Windows, `command -v graphify` can return the launcher path WITHOUT its + .exe suffix, so the `*.exe)` guard misses and the shebang probe reads a + BINARY: the shell then warns 'ignored null byte in input' on every commit and + the extracted garbage always falls through to the slow fallbacks. The read + must strip NULs before the command substitution sees them.""" + from graphify.hooks import _PYTHON_DETECT + assert "tr -d '\\000'" in _PYTHON_DETECT, "shebang read is not NUL-safe" + + +def test_probe_prefers_sibling_python_exe_on_windows_layouts(): + """pip on Windows puts Scripts/graphify(.exe) beside ..\\python.exe (or + .\\python.exe in a venv). Resolving that directly beats shebang-parsing a + binary launcher — and works whether or not command -v kept the suffix.""" + from graphify.hooks import _PYTHON_DETECT + assert "/../python.exe" in _PYTHON_DETECT + assert "/python.exe" in _PYTHON_DETECT + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_reuse_git_dir_from_env(name, script): + """git exports GIT_DIR to hooks, so the rev-parse fallback should only run + when the script is invoked by hand — each extra git exec costs 1s+ on + AV-scanned Windows machines and lands in the commit's foreground.""" + assert "GIT_DIR=${GIT_DIR:-" in script, f"{name} always re-runs git rev-parse" + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_honor_skip_env(name, script): + """GRAPHIFY_SKIP_HOOK=1 must suppress BOTH hooks. post-checkout previously + lacked the check, so the var stopped commit rebuilds but not branch-switch + ones (#1809).""" + assert '[ "${GRAPHIFY_SKIP_HOOK:-0}" = "1" ] && exit 0' in script, ( + f"{name} does not honor GRAPHIFY_SKIP_HOOK" + ) + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_skip_linked_worktrees(name, script): + """Both hooks must short-circuit in a linked worktree (git-dir != common-dir), + and must compare ABSOLUTE paths so the primary checkout (where --git-common-dir + is the relative ".git") is not false-positived and wrongly skipped (#1809, #1806).""" + assert script.count("_GFY_GITDIR=") == 1, f"{name} guard not present exactly once" + assert "git rev-parse --git-common-dir" in script + # absolute-normalized compare, not a raw string compare of git output + assert 'cd "$(git rev-parse --git-dir 2>/dev/null)" 2>/dev/null && pwd' in script + assert '[ "$_GFY_GITDIR" != "$_GFY_COMMONDIR" ]' in script + + +def _worktree_guard_snippet() -> str: + from graphify.hooks import _WORKTREE_GUARD + return _WORKTREE_GUARD + "echo RAN\n" + + +def test_worktree_guard_runs_on_primary_skips_linked(tmp_path): + """End-to-end against a real `git worktree`: the guard falls through on the + primary checkout and exits early inside a linked worktree (#1809, #1806).""" + if shutil.which("git") is None: # pragma: no cover + pytest.skip("git not available") + primary = tmp_path / "primary" + primary.mkdir() + + def _git(*args, cwd): + subprocess.run(["git", *args], cwd=cwd, check=True, + capture_output=True, text=True) + + _git("init", "-q", ".", cwd=primary) + _git("config", "user.email", "t@t.co", cwd=primary) + _git("config", "user.name", "t", cwd=primary) + (primary / "a.txt").write_text("x") + _git("add", "-A", cwd=primary) + _git("commit", "-qm", "init", cwd=primary) + linked = tmp_path / "linked" + _git("worktree", "add", "-q", str(linked), "-b", "feature", cwd=primary) + + snippet = _worktree_guard_snippet() + r_primary = subprocess.run(["sh", "-c", snippet], cwd=primary, + capture_output=True, text=True) + r_linked = subprocess.run(["sh", "-c", snippet], cwd=linked, + capture_output=True, text=True) + assert "RAN" in r_primary.stdout, "guard wrongly skipped the primary checkout" + assert "RAN" not in r_linked.stdout, "guard failed to skip the linked worktree" + + +# ── #1907: duplicate keys in .git/config must not trigger spurious warnings ── + +def _append_duplicate_config_entries(repo: Path) -> None: + """Append git-legal duplicate keys/sections (as VS Code writes them).""" + cfg = repo / ".git" / "config" + cfg.write_text( + cfg.read_text(encoding="utf-8") + + '[remote "origin"]\n' + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "[core]\n" + + "\tignorecase = true\n", + encoding="utf-8", + ) + + +def test_hooks_dir_no_warning_on_duplicate_config_keys(tmp_path, capsys): + """git legally allows duplicate keys and repeated sections in .git/config; + a strict configparser raised DuplicateOptionError/DuplicateSectionError and + printed a spurious 'could not read core.hooksPath' warning on every hook + command (#1907). _hooks_dir must resolve cleanly with no stderr noise.""" + repo = _make_git_repo(tmp_path) + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".git" / "hooks").resolve() + + +def test_hooks_dir_duplicate_config_keys_honor_custom_hookspath(tmp_path, capsys): + """With duplicate keys present, a custom core.hooksPath must still be + honored (no fall-through to .git/hooks) and no warning printed (#1907).""" + repo = _make_git_repo(tmp_path) + _set_hookspath(repo, ".husky") + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".husky").resolve() diff --git a/tests/restored/test_hypergraph_restored.py b/tests/restored/test_hypergraph_restored.py new file mode 100644 index 000000000..b7ce5cb77 --- /dev/null +++ b/tests/restored/test_hypergraph_restored.py @@ -0,0 +1,277 @@ +"""Tests for hyperedge support in graphify.""" +from __future__ import annotations +from graphify.build import build_from_extraction +from graphify.export import attach_hyperedges +from graphify.helix.model import GraphBuildData +from graphify.report import generate +from tests.native_helpers import graph_from_build + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +SAMPLE_EXTRACTION = { + "nodes": [ + {"id": "BasicAuth", "label": "BasicAuth", "file_type": "code", "source_file": "auth.py"}, + {"id": "DigestAuth", "label": "DigestAuth", "file_type": "code", "source_file": "auth.py"}, + {"id": "Request", "label": "Request", "file_type": "code", "source_file": "http.py"}, + {"id": "Response", "label": "Response", "file_type": "code", "source_file": "http.py"}, + {"id": "BaseClient", "label": "BaseClient", "file_type": "code", "source_file": "client.py"}, + ], + "edges": [ + {"source": "BasicAuth", "target": "Request", "relation": "uses", "confidence": "EXTRACTED", "confidence_score": 1.0, "source_file": "auth.py"}, + ], + "hyperedges": [ + { + "id": "auth_flow", + "label": "Auth Flow", + "nodes": ["BasicAuth", "DigestAuth", "Request", "Response", "BaseClient"], + "relation": "participate_in", + "confidence": "INFERRED", + "confidence_score": 0.75, + "source_file": "auth.py", + } + ], + "input_tokens": 10, + "output_tokens": 5, +} + +SAMPLE_DETECTION = { + "total_files": 3, + "total_words": 500, + "files": {"code": ["auth.py", "http.py", "client.py"]}, + "skipped_sensitive": [], + "warning": None, +} + + +def _build(extraction=SAMPLE_EXTRACTION, *, root=None): + return build_from_extraction(extraction, root=root) + + +def _graph(extraction=SAMPLE_EXTRACTION, *, root=None): + return graph_from_build(_build(extraction, root=root)) + + +# --------------------------------------------------------------------------- +# 1. Hyperedges survive build_from_json round-trip +# --------------------------------------------------------------------------- + +def test_build_from_json_stores_hyperedges(): + data = _build() + assert len(data.attributes["hyperedges"]) == 1 + assert data.attributes["hyperedges"][0]["id"] == "auth_flow" + + +def test_build_from_json_relativizes_hyperedge_source_file(tmp_path): + """build_from_json(root=...) must relativize hyperedge source_file like it + already does for nodes and edges. to_json writes G.graph['hyperedges'] + verbatim and has no root parameter, so an absolute path emitted by a semantic + subagent would otherwise leak into the native generation (#1418).""" + base = tmp_path.resolve() + abs_doc = base / "docs" / "CLAUDE.md" + extraction = { + "nodes": [ + {"id": "a", "label": "A", "file_type": "document", "source_file": str(abs_doc)}, + ], + "edges": [], + "hyperedges": [ + { + "id": "arch", + "label": "Architecture", + "nodes": ["a"], + "relation": "participate_in", + "confidence": "INFERRED", + "confidence_score": 0.75, + "source_file": str(abs_doc), + } + ], + } + data = _build(extraction, root=str(base)) + assert data.attributes["hyperedges"][0]["source_file"] == "docs/CLAUDE.md" + # Anchor: the node path is relativized the same way (the contract this mirrors). + assert data.nodes[0].attributes["source_file"] == "docs/CLAUDE.md" + + +def test_build_from_json_no_hyperedges(): + extraction = {**SAMPLE_EXTRACTION, "hyperedges": []} + assert _build(extraction).attributes.get("hyperedges", []) == [] + + +def test_build_from_json_missing_hyperedges_key(): + extraction = {k: v for k, v in SAMPLE_EXTRACTION.items() if k != "hyperedges"} + assert _build(extraction).attributes.get("hyperedges", []) == [] + + +# --------------------------------------------------------------------------- +# 2. attach_hyperedges deduplicates by id +# --------------------------------------------------------------------------- + +def test_attach_hyperedges_adds_new(): + G = GraphBuildData() + attach_hyperedges(G, [{"id": "auth_flow", "label": "Auth Flow", "nodes": ["A", "B", "C"]}]) + assert len(G.attributes["hyperedges"]) == 1 + + +def test_attach_hyperedges_deduplicates(): + G = GraphBuildData() + h = {"id": "auth_flow", "label": "Auth Flow", "nodes": ["A", "B", "C"]} + attach_hyperedges(G, [h]) + attach_hyperedges(G, [h]) # second call with same id should not duplicate + assert len(G.attributes["hyperedges"]) == 1 + + +def test_attach_hyperedges_multiple_different_ids(): + G = GraphBuildData() + attach_hyperedges(G, [ + {"id": "flow_a", "label": "Flow A", "nodes": ["A", "B", "C"]}, + {"id": "flow_b", "label": "Flow B", "nodes": ["D", "E", "F"]}, + ]) + assert len(G.attributes["hyperedges"]) == 2 + + +def test_attach_hyperedges_skips_entry_without_id(): + G = GraphBuildData() + attach_hyperedges(G, [{"label": "No ID", "nodes": ["A", "B", "C"]}]) + assert G.attributes.get("hyperedges", []) == [] + + +# --------------------------------------------------------------------------- +# 5. Report includes hyperedges section when hyperedges present +# --------------------------------------------------------------------------- + +def _make_report(G): + communities = {0: [node.id for node in G.nodes()]} + cohesion = {0: 1.0} + labels = {0: "All"} + gods = [{"label": "BasicAuth", "degree": 2}] + surprises = [] + return generate(G, communities, cohesion, labels, gods, surprises, SAMPLE_DETECTION, {"input": 10, "output": 5}, ".") + + +def test_report_includes_hyperedges_section(): + G = _graph() + report = _make_report(G) + assert "## Hyperedges (group relationships)" in report + assert "Auth Flow" in report + assert "INFERRED 0.75" in report + + +def test_report_includes_hyperedge_node_list(): + G = _graph() + report = _make_report(G) + # Node IDs should appear in the report line + assert "BasicAuth" in report + assert "DigestAuth" in report + + +# --------------------------------------------------------------------------- +# 6. Report skips hyperedges section when none present +# --------------------------------------------------------------------------- + +def test_report_skips_hyperedges_section_when_empty(): + extraction = {**SAMPLE_EXTRACTION, "hyperedges": []} + G = _graph(extraction) + report = _make_report(G) + assert "## Hyperedges" not in report + + +def test_report_skips_hyperedges_section_when_key_missing(): + extraction = {k: v for k, v in SAMPLE_EXTRACTION.items() if k != "hyperedges"} + G = _graph(extraction) + report = _make_report(G) + assert "## Hyperedges" not in report + + +# --------------------------------------------------------------------------- +# 7. Hyperedge member-key alias normalization (#1561) +# --------------------------------------------------------------------------- + +def _alias_extraction(): + """Three hyperedges, one per member-key spelling: nodes / members / node_ids.""" + return { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "m.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "m.py"}, + {"id": "c", "label": "C", "file_type": "code", "source_file": "m.py"}, + ], + "edges": [], + "hyperedges": [ + {"id": "he_nodes", "label": "canon", "nodes": ["a", "b", "c"]}, + {"id": "he_members", "label": "alias1", "members": ["a", "b", "c"]}, + {"id": "he_node_ids", "label": "alias2", "node_ids": ["a", "b", "c"]}, + ], + } + + +def test_build_normalizes_member_aliases_to_nodes(): + data = _build(_alias_extraction()) + hes = {he["id"]: he for he in data.attributes["hyperedges"]} + for hid in ("he_nodes", "he_members", "he_node_ids"): + assert hes[hid]["nodes"] == ["a", "b", "c"], hid + # alias keys are dropped post-normalization + assert "members" not in hes[hid] + assert "node_ids" not in hes[hid] + + +def test_build_dedups_alias_members_preserving_order(): + extraction = { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "m.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "m.py"}, + ], + "edges": [], + "hyperedges": [{"id": "h", "label": "x", "members": ["a", "a", "b"]}], + } + data = _build(extraction) + assert data.attributes["hyperedges"][0]["nodes"] == ["a", "b"] + assert "members" not in data.attributes["hyperedges"][0] + + +def test_build_canonical_nodes_wins_over_alias(): + extraction = { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "m.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "m.py"}, + {"id": "x", "label": "X", "file_type": "code", "source_file": "m.py"}, + ], + "edges": [], + "hyperedges": [ + {"id": "h", "label": "x", "nodes": ["a", "b"], "members": ["x"]}, + ], + } + data = _build(extraction) + he = data.attributes["hyperedges"][0] + assert he["nodes"] == ["a", "b"] # canonical untouched + assert "members" not in he # stray alias dropped + + +def test_build_rekeys_alias_keyed_hyperedge_members(): + """Alias normalization must run BEFORE the semantic id-remap loop so a + `members`-keyed hyperedge's refs get rekeyed alongside `nodes`-keyed ones.""" + # Non-AST node whose id uses the OLD short stem (`mod_foo`) for source_file + # pkg/mod.py -> new canonical stem pkg_mod -> remap mod_foo => pkg_mod_foo. + extraction = { + "nodes": [ + {"id": "mod_foo", "label": "foo", "file_type": "code", "source_file": "pkg/mod.py"}, + {"id": "mod_bar", "label": "bar", "file_type": "code", "source_file": "pkg/mod.py"}, + ], + "edges": [], + "hyperedges": [ + {"id": "h", "label": "x", "members": ["mod_foo", "mod_bar"]}, + ], + } + data = _build(extraction) + he = data.attributes["hyperedges"][0] + assert he["nodes"] == ["pkg_mod_foo", "pkg_mod_bar"] + + +def test_build_warns_once_per_aliased_hyperedge(capsys): + _build(_alias_extraction()) + err = capsys.readouterr().err + # one warning each for the two alias hyperedges, none for the nodes-keyed one + assert err.count("normalizing") == 2 + assert "he_members" in err and "members" in err + assert "he_node_ids" in err and "node_ids" in err + assert "he_nodes" not in err diff --git a/tests/restored/test_incomplete_build_guard_restored.py b/tests/restored/test_incomplete_build_guard_restored.py new file mode 100644 index 000000000..65ddfb5e2 --- /dev/null +++ b/tests/restored/test_incomplete_build_guard_restored.py @@ -0,0 +1,187 @@ +"""Restored incomplete-build guards using atomic native generations.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import graphify.__main__ as mainmod +from graphify.helix.persistence import load_graph + + +def _docs(tmp_path: Path) -> Path: + root = tmp_path / "project" + root.mkdir() + (root / "README.md").write_text("# Notes\nThe entry point overview.\n") + (root / "GUIDE.md").write_text("# Guide\nHow to use the thing.\n") + return root + + +def _run(monkeypatch, argv: list[str]) -> None: + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + mainmod.main() + + +def _semantic(*, failed_chunks: int = 0, node_count: int = 1): + def run(paths, **kwargs): + path = Path(paths[0]) + return { + "nodes": [ + { + "id": f"semantic-{index}", "label": f"Semantic {index}", + "source_file": path.name, "file_type": "document", + } + for index in range(node_count) + ], + "edges": [], "hyperedges": [], "input_tokens": 10, + "output_tokens": 5, "failed_chunks": failed_chunks, + } + return run + + +def _seed_complete(monkeypatch, project: Path, output: Path) -> Path: + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(node_count=5)) + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--no-cluster", "--out", str(output), + ]) + return output / "graphify-out" / "graph.helix" + + +def _assert_generation_unchanged(store: Path, generation: str, counts: tuple[int, int]) -> None: + loaded = load_graph(store) + assert loaded.generation == generation + assert (loaded.graph.node_count, loaded.graph.edge_count) == counts + + +def test_partial_extraction_refuses_to_shrink_existing_graph(monkeypatch, tmp_path, capsys): + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + before = load_graph(store) + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=2)) + + with pytest.raises(SystemExit) as exc: + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--out", str(output), "--force", + ]) + + assert exc.value.code == 1 + assert "active generation was left unchanged" in capsys.readouterr().err + _assert_generation_unchanged( + store, before.generation, (before.graph.node_count, before.graph.edge_count) + ) + + +def test_partial_extraction_writes_when_not_shrinking(monkeypatch, tmp_path): + """Native activation is stricter: any failed chunk is rejected, regardless of size.""" + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + before = load_graph(store) + monkeypatch.setattr( + "graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=1, node_count=8) + ) + + with pytest.raises(SystemExit): + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--out", str(output), "--force", + ]) + + _assert_generation_unchanged( + store, before.generation, (before.graph.node_count, before.graph.edge_count) + ) + + +def test_allow_partial_forces_write_despite_incomplete(monkeypatch, tmp_path): + """The retired escape hatch cannot bypass atomic-generation verification.""" + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + before = load_graph(store) + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=1)) + + with pytest.raises(SystemExit): + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--out", str(output), "--force", "--allow-partial", + ]) + + _assert_generation_unchanged( + store, before.generation, (before.graph.node_count, before.graph.edge_count) + ) + + +def test_complete_extraction_keeps_force_write(monkeypatch, tmp_path): + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + generation = load_graph(store).generation + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(node_count=2)) + + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--out", str(output), "--force", + ]) + + assert load_graph(store).generation != generation + + +def test_no_cluster_incomplete_build_refuses_to_shrink(tmp_path, monkeypatch, capsys): + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + before = load_graph(store) + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=1)) + + with pytest.raises(SystemExit): + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--no-cluster", "--out", str(output), "--force", + ]) + + assert "active generation was left unchanged" in capsys.readouterr().err + _assert_generation_unchanged( + store, before.generation, (before.graph.node_count, before.graph.edge_count) + ) + + +def test_no_cluster_allow_partial_overwrites(tmp_path, monkeypatch): + project = _docs(tmp_path) + output = tmp_path / "out" + store = _seed_complete(monkeypatch, project, output) + generation = load_graph(store).generation + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=1)) + + with pytest.raises(SystemExit): + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--no-cluster", "--out", str(output), "--force", "--allow-partial", + ]) + + assert load_graph(store).generation == generation + + +def test_no_cluster_incomplete_build_fails_closed_on_malformed_existing_graph( + tmp_path, monkeypatch, capsys +): + project = _docs(tmp_path) + output = tmp_path / "out" + store = output / "graphify-out" / "graph.helix" + store.mkdir(parents=True) + sentinel = store / "corrupt-sentinel" + sentinel.write_text("do not replace") + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _semantic(failed_chunks=1)) + + with pytest.raises(SystemExit): + _run(monkeypatch, [ + "graphify", "extract", str(project), "--backend", "openai", + "--no-cluster", "--out", str(output), + ]) + + assert sentinel.read_text() == "do not replace" + assert capsys.readouterr().err diff --git a/tests/restored/test_incremental_restored.py b/tests/restored/test_incremental_restored.py new file mode 100644 index 000000000..a5c60ba0b --- /dev/null +++ b/tests/restored/test_incremental_restored.py @@ -0,0 +1,113 @@ +"""Restored incremental-build tests against durable native generation state.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from graphify.helix.model import edge_attributes +from graphify.helix.persistence import load_graph + + +_KEYS = { + "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", + "MOONSHOT_API_KEY", "DEEPSEEK_API_KEY", "OLLAMA_BASE_URL", "AWS_PROFILE", +} + + +def _run(cwd, *args): + env = {key: value for key, value in os.environ.items() if key not in _KEYS} + return subprocess.run( + [sys.executable, "-m", "graphify", *args], cwd=cwd, + capture_output=True, text=True, env=env, + ) + + +def _project(tmp_path): + root = tmp_path / "project" + root.mkdir() + (root / "a.py").write_text("def a():\n return 1\n") + return root + + +def test_manifest_written_after_extract(tmp_path): + project = _project(tmp_path) + result = _run(tmp_path, "extract", str(project), "--code-only", "--no-cluster") + assert result.returncode == 0, result.stderr + + loaded = load_graph(project / "graphify-out" / "graph.helix") + assert "a.py" in loaded.state["incremental"]["files"] + assert loaded.state["incremental"]["files"]["a.py"]["content_hash"] + assert not (project / "graphify-out" / "manifest.json").exists() + + +def test_incremental_mode_detected_via_manifest(tmp_path): + project = _project(tmp_path) + first = _run(tmp_path, "extract", str(project), "--code-only", "--no-cluster") + assert first.returncode == 0, first.stderr + store = project / "graphify-out" / "graph.helix" + before = load_graph(store) + + second = _run(tmp_path, "update", str(project), "--no-cluster") + assert second.returncode == 0, second.stderr + after = load_graph(store) + + assert after.state["incremental"]["extractor_state"]["mode"] == "ast" + assert after.state["incremental"]["extraction_cache"] + assert (after.graph.node_count, after.graph.edge_count) == ( + before.graph.node_count, before.graph.edge_count, + ) + + +def test_no_incremental_without_manifest(tmp_path): + project = _project(tmp_path) + result = _run(tmp_path, "update", str(project), "--no-cluster") + assert result.returncode == 0, result.stderr + loaded = load_graph(project / "graphify-out" / "graph.helix") + assert loaded.graph.node_count > 0 + assert loaded.state["incremental"]["files"] + + +def test_extract_no_cluster_incremental_noop_preserves_existing_graph(tmp_path): + project = _project(tmp_path) + first = _run(tmp_path, "extract", str(project), "--code-only", "--no-cluster") + assert first.returncode == 0, first.stderr + store = project / "graphify-out" / "graph.helix" + before = load_graph(store) + + second = _run(tmp_path, "extract", str(project), "--code-only", "--no-cluster") + assert second.returncode == 0, second.stderr + after = load_graph(store) + + assert (after.graph.node_count, after.graph.edge_count) == ( + before.graph.node_count, before.graph.edge_count, + ) + + +def test_update_prunes_a_removed_imports_edge(tmp_path): + project = tmp_path / "project" + package = project / "pkg" + package.mkdir(parents=True) + (package / "b.py").write_text("def helper():\n return 1\n") + source = package / "a.py" + source.write_text("from pkg.b import helper\ndef use():\n return helper()\n") + first = _run(tmp_path, "extract", str(project), "--code-only", "--no-cluster") + assert first.returncode == 0, first.stderr + store = project / "graphify-out" / "graph.helix" + before = load_graph(store) + assert any( + edge_attributes(edge).get("relation") in {"imports", "imports_from"} + and str(edge_attributes(edge).get("source_file", "")).endswith("a.py") + for edge in before.graph.edges() + ) + + source.write_text("def use():\n return 1\n") + updated = _run(tmp_path, "update", str(project), "--no-cluster") + assert updated.returncode == 0, updated.stderr + after = load_graph(store) + assert not any( + edge_attributes(edge).get("relation") in {"imports", "imports_from"} + and str(edge_attributes(edge).get("source_file", "")).endswith("a.py") + for edge in after.graph.edges() + ) diff --git a/tests/test_merge_chunks_validation.py b/tests/restored/test_merge_chunks_validation_restored.py similarity index 100% rename from tests/test_merge_chunks_validation.py rename to tests/restored/test_merge_chunks_validation_restored.py diff --git a/tests/restored/test_multigraph_diagnostics_restored.py b/tests/restored/test_multigraph_diagnostics_restored.py new file mode 100644 index 000000000..c335e5058 --- /dev/null +++ b/tests/restored/test_multigraph_diagnostics_restored.py @@ -0,0 +1,161 @@ +"""Retained diagnostics for transient extraction DTOs.""" + +from __future__ import annotations + +from copy import deepcopy +import json + +from graphify.diagnostics import ( + diagnose_extraction, + format_diagnostic_json, + format_diagnostic_report, + scan_producer_suppression_sites, +) + + +def _diagnostic_fixture() -> dict: + nodes = [ + {"id": value, "label": value.upper(), "file_type": "code", "source_file": f"{value}.py"} + for value in ("a", "b", "c") + ] + edges = [ + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED", "source_file": "a.py", "source_location": "L1", "context": "call"}, + {"source": "a", "target": "b", "relation": "imports", "confidence": "EXTRACTED", "source_file": "a.py", "source_location": "L2", "context": "import"}, + {"source": "a", "target": "b", "relation": "calls", "confidence": "INFERRED", "source_file": "a.py", "source_location": "L3", "context": "call"}, + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED", "source_file": "a.py", "source_location": "L1", "context": "call"}, + {"source": "a", "target": "missing", "relation": "calls", "source_file": "a.py"}, + {"source": "a", "relation": "calls", "source_file": "a.py"}, + {"source": "c", "target": "c", "relation": "references", "source_file": "c.py"}, + ] + return {"nodes": nodes, "edges": edges} + + +def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: + summary = diagnose_extraction(_diagnostic_fixture(), directed=True) + assert summary["node_count"] == 3 + assert summary["raw_edge_count"] == 7 + assert summary["valid_candidate_edges"] == 5 + assert summary["missing_endpoint_edges"] == 1 + assert summary["dangling_endpoint_edges"] == 1 + assert summary["self_loop_edges"] == 1 + assert summary["exact_duplicate_edges"] == 1 + assert summary["directed_unique_endpoint_pairs"] == 2 + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + assert summary["relation_variant_groups"] == 1 + assert summary["post_build_graph_type"] == "digraph" + assert summary["post_build_edge_count"] == 2 + + +def test_diagnose_extraction_accepts_node_link_links_key() -> None: + extraction = _diagnostic_fixture() + extraction["links"] = extraction.pop("edges") + summary = diagnose_extraction(extraction, directed=True) + assert summary["raw_edge_count"] == 7 + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + + +def test_diagnose_extraction_does_not_mutate_input() -> None: + extraction = _diagnostic_fixture() + original = deepcopy(extraction) + diagnose_extraction(extraction, directed=True) + assert extraction == original + + +def test_diagnose_extraction_handles_malformed_shapes_without_crashing() -> None: + extraction = { + "nodes": [{"id": "a"}, ["bad"], {"id": "b"}], + "edges": [ + None, + ["bad"], + {"from": "a", "to": "b", "relation": "legacy"}, + {"source": "a", "target": {"bad": "target"}}, + {"source": "a", "target": "missing"}, + {"source": "", "target": "b"}, + ], + } + summary = diagnose_extraction(extraction, directed=True) + assert summary["node_count"] == 2 + assert summary["raw_edge_count"] == 6 + assert summary["non_object_edges"] == 2 + assert summary["missing_endpoint_edges"] == 1 + assert summary["dangling_endpoint_edges"] == 2 + assert summary["valid_candidate_edges"] == 1 + assert summary["post_build_error"].startswith(("TypeError:", "AttributeError:")) + + +def test_diagnose_extraction_handles_non_list_nodes_and_edges() -> None: + summary = diagnose_extraction( + {"nodes": {"id": "a"}, "edges": {"source": "a", "target": "b"}}, + directed=True, + ) + assert summary["node_count"] == 0 + assert summary["raw_edge_count"] == 0 + assert summary["valid_candidate_edges"] == 0 + + +def test_diagnose_extraction_bounds_examples() -> None: + summary = diagnose_extraction(_diagnostic_fixture(), directed=True, max_examples=0) + assert summary["directed_same_endpoint_collapsed_edges"] == 3 + assert summary["examples"] == [] + + +def test_diagnose_extraction_stops_examples_at_requested_limit() -> None: + extraction = _diagnostic_fixture() + extraction["nodes"].append({"id": "d", "source_file": "d.py"}) + extraction["edges"].extend([ + {"source": "b", "target": "d", "relation": "imports"}, + {"source": "b", "target": "d", "relation": "calls"}, + ]) + summary = diagnose_extraction(extraction, directed=True, max_examples=1) + assert summary["same_endpoint_group_count"] == 2 + assert len(summary["examples"]) == 1 + + +def test_format_diagnostic_report_includes_build_and_suppression_errors(tmp_path) -> None: + summary = diagnose_extraction( + {"nodes": [{"id": "a"}, ["bad"]], "edges": []}, + extract_path=tmp_path / "missing-extract.py", + ) + report = format_diagnostic_report(summary) + assert "post_build_error:" in report + assert "producer_suppression_error: file not found" in report + + +def test_diagnostic_json_report_is_serializable() -> None: + payload = format_diagnostic_json( + diagnose_extraction(_diagnostic_fixture(), directed=True) + ) + assert payload["schema_version"] == 1 + assert payload["summary"]["raw_edge_count"] == 7 + assert "producer_suppression" in payload + json.dumps(payload) + + +def test_scan_producer_suppression_sites_finds_seen_sets(tmp_path) -> None: + source = tmp_path / "extract.py" + source.write_text( + "seen_call_pairs: set[tuple[str, str]] = set()\n" + "seen_static_ref_pairs: set[tuple[str, str, str]] = set()\n" + "other = set()\n" + ) + result = scan_producer_suppression_sites(source) + assert result["total_sites"] == 2 + assert [site["tuple_arity"] for site in result["sites"]] == [2, 3] + + +def test_scan_producer_suppression_sites_handles_unknown_tuple_arity(tmp_path) -> None: + source = tmp_path / "extract.py" + source.write_text("seen_blank: set[tuple[ ]] = set()\n") + result = scan_producer_suppression_sites(source) + assert result["total_sites"] == 1 + assert result["sites"][0]["tuple_arity"] == 0 + + +def test_scan_producer_suppression_sites_reports_missing_file(tmp_path) -> None: + result = scan_producer_suppression_sites(tmp_path / "missing-extract.py") + assert result == { + "path": str(tmp_path / "missing-extract.py"), + "total_sites": 0, + "sites": [], + "error": "file not found", + } diff --git a/tests/restored/test_partial_cache_restored.py b/tests/restored/test_partial_cache_restored.py new file mode 100644 index 000000000..4aef682d8 --- /dev/null +++ b/tests/restored/test_partial_cache_restored.py @@ -0,0 +1,197 @@ +"""Restored partial-extraction cache promotion tests for Helix-owned state.""" + +from graphify import llm +from graphify.cache import ( + _group_has_partial_marker, + check_semantic_cache, + load_cached, + save_semantic_cache, +) + + +def _doc(tmp_path): + doc = tmp_path / "doc.md" + doc.write_text("# Heading\nsome prose\n", encoding="utf-8") + return doc + + +def test_intrinsic_partial_marker_makes_entry_a_cache_miss(tmp_path): + doc = _doc(tmp_path) + state = {} + saved = save_semantic_cache( + [{"id": "n1", "source_file": "doc.md", "_partial": True}], [], + root=tmp_path, prompt="P", cache=state, + ) + assert saved == 1 + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + + +def test_partial_source_files_arg_stamps_entry(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [{"id": "n1", "source_file": "doc.md"}], [], root=tmp_path, + prompt="P", partial_source_files=["doc.md"], cache=state, + ) + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + + +def test_non_partial_entry_loads_normally(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [{"id": "n1", "source_file": "doc.md"}], [], + root=tmp_path, prompt="P", cache=state, + ) + loaded = load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) + assert loaded is not None and len(loaded["nodes"]) == 1 + + +def test_partial_entry_self_heals_on_complete_reextraction(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [{"id": "n1", "source_file": "doc.md", "_partial": True}], [], + root=tmp_path, prompt="P", cache=state, + ) + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + save_semantic_cache( + [ + {"id": "n1", "source_file": "doc.md"}, + {"id": "n2", "source_file": "doc.md"}, + ], + [], root=tmp_path, prompt="P", cache=state, + ) + loaded = load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) + assert loaded is not None and len(loaded["nodes"]) == 2 + + +def test_merge_existing_accumulates_slices_and_stays_partial(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [{"id": "n1", "source_file": "doc.md", "_partial": True}], [], + root=tmp_path, prompt="P", cache=state, + ) + save_semantic_cache( + [{"id": "n2", "source_file": "doc.md"}], [], root=tmp_path, + prompt="P", merge_existing=True, cache=state, + ) + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + peek = load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", + allow_partial=True, cache=state, + ) + assert peek is not None + assert {node["id"] for node in peek["nodes"]} == {"n1", "n2"} + + +def test_save_stamps_partial_file_with_no_items(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [{"id": "n1", "source_file": "doc.md"}], [], + root=tmp_path, prompt="P", cache=state, + ) + save_semantic_cache( + [], [], root=tmp_path, prompt="P", merge_existing=True, + partial_source_files=["doc.md"], cache=state, + ) + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + peek = load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", + allow_partial=True, cache=state, + ) + assert peek is not None and {node["id"] for node in peek["nodes"]} == {"n1"} + + +def test_clean_slice_does_not_repromote_empty_parse_partial(tmp_path): + doc = _doc(tmp_path) + state = {} + save_semantic_cache( + [], [], root=tmp_path, prompt="P", + partial_source_files=["doc.md"], cache=state, + ) + save_semantic_cache( + [{"id": "n2", "source_file": "doc.md"}], [], root=tmp_path, + prompt="P", merge_existing=True, cache=state, + ) + assert load_cached( + doc, root=tmp_path, kind="semantic", prompt="P", cache=state + ) is None + + +def test_partial_files_carries_empty_parse_truncation(): + result = {"nodes": [], "edges": [], "hyperedges": [], "_partial_files": ["big.md"]} + assert llm._partial_source_files(result) == ["big.md"] + result["nodes"] = [{"id": "a", "source_file": "x.md", "_partial": True}] + assert llm._partial_source_files(result) == ["big.md", "x.md"] + + +def test_stamped_manifest_excludes_partial_files(tmp_path): + """A partial durable entry is a miss, so the file is re-queued.""" + a = tmp_path / "a.md" + b = tmp_path / "b.md" + a.write_text("A") + b.write_text("B") + state = {} + save_semantic_cache( + [{"id": "a", "source_file": "a.md"}], [], + root=tmp_path, prompt="P", cache=state, + ) + save_semantic_cache( + [{"id": "b", "source_file": "b.md"}], [], root=tmp_path, + prompt="P", partial_source_files=["b.md"], cache=state, + ) + nodes, _edges, _hyperedges, uncached = check_semantic_cache( + [str(a), str(b)], state, root=tmp_path, prompt="P" + ) + assert {node["id"] for node in nodes} == {"a"} + assert uncached == [str(b)] + + +def test_group_has_partial_marker(): + assert _group_has_partial_marker({"nodes": [{"_partial": True}]}) is True + assert _group_has_partial_marker({"edges": [{"_partial": True}]}) is True + assert _group_has_partial_marker({"nodes": [{"id": "a"}]}) is False + assert _group_has_partial_marker({}) is False + + +def test_mark_partial_and_partial_source_files(): + result = { + "nodes": [{"id": "a", "source_file": "x.md"}], + "edges": [{"source": "a", "target": "b", "source_file": "x.md"}], + "hyperedges": [{"id": "h", "source_file": "y.md"}], + } + llm._mark_partial(result) + assert all(item["_partial"] is True for bucket in result.values() for item in bucket) + assert llm._partial_source_files(result) == ["x.md", "y.md"] + + +def test_partial_source_files_empty_when_unmarked(): + result = {"nodes": [{"id": "a", "source_file": "x.md"}], "edges": [], "hyperedges": []} + assert llm._partial_source_files(result) == [] + + +def test_strip_partial_markers_removes_internal_key(): + result = { + "nodes": [{"id": "a", "_partial": True}], + "edges": [{"source": "a", "target": "b", "_partial": True}], + "hyperedges": [{"id": "h", "_partial": True}], + } + llm._strip_partial_markers(result) + assert all("_partial" not in item for bucket in result.values() for item in bucket) diff --git a/tests/restored/test_pipeline_restored.py b/tests/restored/test_pipeline_restored.py new file mode 100644 index 000000000..3fa0fa1c0 --- /dev/null +++ b/tests/restored/test_pipeline_restored.py @@ -0,0 +1,147 @@ +""" +End-to-end pipeline test: detect → extract → build → cluster → analyze → report → export. +Uses the existing test fixtures (code + markdown). No LLM calls - AST extraction only. +Catches regressions in how modules connect, not just individual module behaviour. +""" +from pathlib import Path + +from graphify.detect import detect +from graphify.extract import extract +from graphify.build import build_from_extraction +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_html, to_obsidian +from tests.native_helpers import graph_from_build + +FIXTURES = Path(__file__).parents[1] / "fixtures" + + +def run_pipeline(tmp_path: Path) -> dict: + """Run the full pipeline on the fixtures directory. Returns a dict of outputs.""" + # Step 1: detect + detection = detect(FIXTURES) + assert detection["total_files"] > 0 + # fixtures corpus is intentionally small (< 5k words), so needs_graph may be False + assert "files" in detection + + # Step 2: extract (AST only - no LLM) + code_files = [Path(f) for f in detection["files"].get("code", [])] + assert len(code_files) > 0 + extraction = extract(code_files) + assert len(extraction["nodes"]) > 0 + assert len(extraction["edges"]) > 0 + + # Step 3: build + G = graph_from_build(build_from_extraction(extraction)) + assert G.node_count > 0 + assert G.edge_count > 0 + + # Step 4: cluster + communities = cluster(G) + assert len(communities) > 0 + cohesion = score_all(G, communities) + assert len(cohesion) == len(communities) + for score in cohesion.values(): + assert 0.0 <= score <= 1.0 + + # Step 5: analyze + gods = god_nodes(G) + assert len(gods) > 0 + assert all("id" in g and "degree" in g for g in gods) + + surprises = surprising_connections(G, communities) + assert isinstance(surprises, list) + + labels = {cid: f"Group {cid}" for cid in communities} + questions = suggest_questions(G, communities, labels) + assert isinstance(questions, list) + + # Step 6: report + tokens = {"input": 0, "output": 0} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, str(FIXTURES), suggested_questions=questions) + assert "God Nodes" in report + assert "Communities" in report + assert len(report) > 100 + + # Step 7: export - HTML directly from the native snapshot + html_path = tmp_path / "graph.html" + to_html(G, communities, str(html_path), community_labels=labels) + assert html_path.exists() + html = html_path.read_text() + assert "vis-network" in html + assert "RAW_NODES" in html + + # Step 8: export - Obsidian vault + vault_path = tmp_path / "obsidian" + n_notes = to_obsidian(G, communities, str(vault_path), community_labels=labels, cohesion=cohesion) + assert n_notes > 0 + assert (vault_path / ".obsidian" / "graph.json").exists() + md_files = list(vault_path.glob("*.md")) + assert len(md_files) > 0 + + return { + "detection": detection, + "extraction": extraction, + "graph": G, + "communities": communities, + "cohesion": cohesion, + "gods": gods, + "surprises": surprises, + "questions": questions, + "report": report, + } + + +def test_pipeline_runs_end_to_end(tmp_path): + result = run_pipeline(tmp_path) + assert result["graph"].node_count > 0 + + +def test_pipeline_graph_has_edges(tmp_path): + result = run_pipeline(tmp_path) + assert result["graph"].edge_count > 0 + + +def test_pipeline_all_nodes_have_community(tmp_path): + result = run_pipeline(tmp_path) + G = result["graph"] + communities = result["communities"] + all_community_nodes = {n for nodes in communities.values() for n in nodes} + for node in G.nodes(): + assert node.id in all_community_nodes, f"Node {node.id!r} has no community" + + +def test_pipeline_report_mentions_top_god_node(tmp_path): + result = run_pipeline(tmp_path) + top_god = result["gods"][0]["label"] + assert top_god in result["report"] + + +def test_pipeline_detection_finds_code_and_docs(tmp_path): + result = run_pipeline(tmp_path) + assert len(result["detection"]["files"].get("code", [])) > 0 + assert len(result["detection"]["files"].get("document", [])) > 0 + + +def test_pipeline_incremental_update(tmp_path): + """Second run on unchanged corpus should produce identical node/edge counts.""" + result1 = run_pipeline(tmp_path) + result2 = run_pipeline(tmp_path) + assert result1["graph"].node_count == result2["graph"].node_count + assert result1["graph"].edge_count == result2["graph"].edge_count + + +def test_pipeline_extraction_confidence_labels(tmp_path): + result = run_pipeline(tmp_path) + extraction = result["extraction"] + valid = {"EXTRACTED", "INFERRED", "AMBIGUOUS"} + for edge in extraction["edges"]: + assert edge["confidence"] in valid, f"Invalid confidence: {edge['confidence']}" + + +def test_pipeline_no_self_loops(tmp_path): + result = run_pipeline(tmp_path) + G = result["graph"] + for edge in G.edges(): + assert edge.source != edge.target, f"Self-loop found on node {edge.source!r}" diff --git a/tests/restored/test_report_restored.py b/tests/restored/test_report_restored.py new file mode 100644 index 000000000..cccae037a --- /dev/null +++ b/tests/restored/test_report_restored.py @@ -0,0 +1,157 @@ +import json +from pathlib import Path +from graphify.build import build_from_extraction +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from tests.native_helpers import graph_from_build + +FIXTURES = Path(__file__).parents[1] / "fixtures" + +def make_inputs(): + extraction = json.loads((FIXTURES / "extraction.json").read_text()) + G = graph_from_build(build_from_extraction(extraction)) + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + surprises = surprising_connections(G) + detection = {"total_files": 4, "total_words": 62400, "needs_graph": True, "warning": None} + tokens = {"input": extraction["input_tokens"], "output": extraction["output_tokens"]} + return G, communities, cohesion, labels, gods, surprises, detection, tokens + +def test_report_contains_header(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "# Graph Report" in report + +def test_report_contains_corpus_check(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Corpus Check" in report + +def test_report_contains_god_nodes(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## God Nodes" in report + +def test_report_contains_surprising_connections(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Surprising Connections" in report + +def test_report_contains_communities(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Communities" in report + +def test_report_contains_ambiguous_section(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Ambiguous Edges" in report + +def test_report_shows_token_cost(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "Token cost" in report + assert "1,200" in report + +def test_report_shows_raw_cohesion_scores(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1) + assert "Cohesion:" in report + assert "✓" not in report + assert "⚠" not in report + + +# --- work-memory lessons section ---------------------------------------------- + +def test_report_work_memory_section_present_with_overlay_and_dead_ends(): + """When a work-memory overlay (preferred sources) and query-scoped dead-ends + are supplied, the report grows a `## Work-memory lessons` section listing the + preferred sources and, separately, the dead-ends as question -> nodes.""" + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + learning = { + "overlay": { + "auth_login": {"status": "preferred", "uses": 3, "score": 2.4, + "label": "login()", "stale": False}, + "redis": {"status": "tentative", "uses": 1, "score": 0.5, + "label": "RedisClient", "stale": False}, + }, + "dead_ends": [ + {"question": "does it use websockets?", "nodes": ["WSServer"], "date": "2026-05-01"}, + ], + } + report = generate(G, communities, cohesion, labels, gods, surprises, detection, + tokens, "./project", learning=learning) + assert "## Work-memory lessons" in report + assert "**Preferred sources**" in report + assert "`login()`" in report + # Tentative is not listed in the report's preferred block. + assert "RedisClient" not in report + # Dead-ends are query-scoped: question -> nodes, NOT a node-level status. + assert "**Known dead ends**" in report + assert "does it use websockets?" in report + assert "`WSServer`" in report + + +def test_report_work_memory_section_absent_without_overlay(): + """No learning input => no section; report identical to pre-feature.""" + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + before = generate(G, communities, cohesion, labels, gods, surprises, detection, + tokens, "./project") + assert "## Work-memory lessons" not in before + # Explicit empty learning also omits the section. + empty = generate(G, communities, cohesion, labels, gods, surprises, detection, + tokens, "./project", learning={"overlay": {}, "dead_ends": []}) + assert "## Work-memory lessons" not in empty + assert before == empty + + +def test_import_cycles_section_present_for_code_corpus(): + # #1657: the fixture is a code corpus, so the Import Cycles section shows. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Import Cycles" in report + + +def test_import_cycles_section_absent_for_documents_only_corpus(): + # #1657: a documents-only corpus has no imports; the section is pure noise + # ("None detected") and must be suppressed. + extraction = { + "nodes": [ + {"id": "d1", "label": "intro.md", "file_type": "document"}, + {"id": "d2", "label": "guide.md", "file_type": "document"}, + ], + "edges": [{"source": "d1", "target": "d2", "relation": "references"}], + "input_tokens": 0, "output_tokens": 0, + } + G = graph_from_build(build_from_extraction(extraction)) + communities = cluster(G) + cohesion = score_all(G, communities) + labels = {cid: f"Community {cid}" for cid in communities} + gods = god_nodes(G) + surprises = surprising_connections(G) + detection = {"total_files": 2, "total_words": 100, "needs_graph": True, "warning": None} + tokens = {"input": 0, "output": 0} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Import Cycles" not in report + + +def test_report_hubs_are_plain_text_by_default(): + # #1712: without --obsidian the _COMMUNITY_*.md notes don't exist, so wikilinks + # would dangle (and pollute an Obsidian vault's graph view). Default to plain text. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + labels = {cid: f"Widget {cid}" for cid in communities} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1) + assert "## Community Hubs (Navigation)" in report + assert "[[_COMMUNITY_" not in report, "must not emit dangling Obsidian wikilinks by default (#1712)" + assert any(f"- Widget {cid}" in report for cid in communities) + + +def test_report_hubs_use_wikilinks_when_obsidian(): + # The opt-in path keeps the vault-navigable wikilink form. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + labels = {cid: f"Widget {cid}" for cid in communities} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1, obsidian=True) + assert "[[_COMMUNITY_" in report diff --git a/tests/restored/test_semantic_cache_out_root_restored.py b/tests/restored/test_semantic_cache_out_root_restored.py new file mode 100644 index 000000000..3a2e88af8 --- /dev/null +++ b/tests/restored/test_semantic_cache_out_root_restored.py @@ -0,0 +1,114 @@ +"""Restored #1990/#1991 cache-root regressions for Helix generation state.""" + +from __future__ import annotations + +import inspect +import warnings + +from graphify.cache import check_semantic_cache, save_semantic_cache + + +def _roots(tmp_path): + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + return corpus, out + + +def test_save_semantic_cache_writes_to_cache_root_not_corpus(tmp_path): + corpus, out = _roots(tmp_path) + doc = corpus / "report.md" + doc.write_text("# Report\nSome content here.") + state = {} + + saved = save_semantic_cache( + [{"id": "n1", "source_file": str(doc)}], [], + root=corpus, cache_root=out, cache=state, + ) + + assert saved == 1 and state + assert not (corpus / "graphify-out").exists() + assert not (out / "graphify-out").exists() + + +def test_save_semantic_cache_no_corpus_graphify_out_created(tmp_path): + corpus, out = _roots(tmp_path) + doc = corpus / "notes.md" + doc.write_text("Notes content.") + save_semantic_cache( + [{"id": "x", "source_file": str(doc)}], [], + root=corpus, cache_root=out, cache={}, + ) + assert not (corpus / "graphify-out").exists() + assert not (out / "graphify-out").exists() + + +def test_checkpoint_with_cache_root_is_found_by_check_semantic_cache(tmp_path): + corpus, out = _roots(tmp_path) + doc = corpus / "paper.md" + doc.write_text("Some academic content.") + state = {} + save_semantic_cache( + [{"id": "p1", "source_file": str(doc)}], [], root=corpus, + cache_root=out, merge_existing=True, allowed_source_files=[doc], + cache=state, + ) + + nodes, edges, hyperedges, uncached = check_semantic_cache( + [str(doc)], state, root=corpus + ) + assert not uncached and not edges and not hyperedges + assert {node["id"] for node in nodes} == {"p1"} + + +def test_final_save_with_out_root_populates_cache(tmp_path): + corpus, out = _roots(tmp_path) + doc = corpus / "report.md" + doc.write_text("# Annual Report\nKey findings.") + state = {} + + saved = save_semantic_cache( + [{"id": "r1", "source_file": "report.md"}], [], root=corpus, + cache_root=out, allowed_source_files=[doc], cache=state, + ) + + assert saved == 1 + assert len(state) == 1 + assert next(iter(state)).endswith(":report.md") + + +def test_final_save_with_wrong_root_emits_warning(tmp_path): + corpus, out = _roots(tmp_path) + (corpus / "report.md").write_text("# Report") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + saved = save_semantic_cache( + [{"id": "r1", "source_file": "report.md"}], [], + root=out, cache={}, + ) + + assert saved == 0 + assert any("corpus root" in str(item.message) for item in caught) + + +def test_save_semantic_cache_backward_compat_no_cache_root(tmp_path): + root = tmp_path / "project" + root.mkdir() + doc = root / "main.md" + doc.write_text("Main content.") + + saved = save_semantic_cache( + [{"id": "m1", "source_file": str(doc)}], [], root=root + ) + + assert saved == 0 + assert not (root / "graphify-out").exists() + + +def test_extract_corpus_parallel_accepts_cache_root_kwarg(): + from graphify.llm import extract_corpus_parallel + + signature = inspect.signature(extract_corpus_parallel) + assert "cache_root" in signature.parameters + assert "cache" in signature.parameters diff --git a/tests/restored/test_semantic_similarity_restored.py b/tests/restored/test_semantic_similarity_restored.py new file mode 100644 index 000000000..7e50832a8 --- /dev/null +++ b/tests/restored/test_semantic_similarity_restored.py @@ -0,0 +1,198 @@ +"""Tests for semantically_similar_to edge support.""" +import pytest +from graphify.build import build_from_extraction +from graphify.analyze import _surprise_score +from graphify.helix.access import first_edge_attributes +from graphify.report import generate +from tests.native_helpers import graph_from_build, graph_from_payload + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_extraction_with_semantic_edge(): + """Two nodes in separate files connected by a semantically_similar_to edge.""" + return { + "nodes": [ + {"id": "a_validate_input", "label": "validate_input", "file_type": "code", + "source_file": "auth/validators.py", "source_location": "L5"}, + {"id": "b_check_input", "label": "check_input", "file_type": "code", + "source_file": "api/checks.py", "source_location": "L12"}, + ], + "edges": [ + { + "source": "a_validate_input", + "target": "b_check_input", + "relation": "semantically_similar_to", + "confidence": "INFERRED", + "confidence_score": 0.82, + "source_file": "auth/validators.py", + "source_location": None, + "weight": 0.82, + } + ], + "input_tokens": 100, + "output_tokens": 50, + } + + +def _make_graph_with_semantic_edge(): + return graph_from_build(build_from_extraction(_make_extraction_with_semantic_edge())) + + +def _make_two_edge_graph(): + """Graph with one semantically_similar_to edge and one references edge, both cross-file.""" + return graph_from_payload( + [ + {"id": nid, "label": label, "source_file": src, "file_type": "code"} + for nid, label, src in [ + ("a", "ValidateInput", "auth/validators.py"), + ("b", "CheckInput", "api/checks.py"), + ("c", "LoadConfig", "config/loader.py"), + ("d", "ReadConfig", "utils/reader.py"), + ] + ], + [ + {"source": "a", "target": "b", "relation": "semantically_similar_to", + "confidence": "INFERRED", "confidence_score": 0.82, + "source_file": "auth/validators.py", "weight": 0.82}, + {"source": "c", "target": "d", "relation": "references", + "confidence": "INFERRED", "confidence_score": 0.7, + "source_file": "config/loader.py", "weight": 0.7}, + ], + ) + + +# --------------------------------------------------------------------------- +# Test 1: semantically_similar_to passes through build_from_json without being dropped +# --------------------------------------------------------------------------- + +def test_semantic_edge_survives_build_from_json(): + G = _make_graph_with_semantic_edge() + assert G.edge_count == 1 + data = first_edge_attributes(G, "a_validate_input", "b_check_input") + assert data["relation"] == "semantically_similar_to" + + +def test_semantic_edge_nodes_present(): + G = _make_graph_with_semantic_edge() + assert G.contains_node("a_validate_input") + assert G.contains_node("b_check_input") + + +# --------------------------------------------------------------------------- +# Test 2: confidence_score is preserved for semantically_similar_to edges +# --------------------------------------------------------------------------- + +def test_semantic_edge_confidence_score_preserved(): + G = _make_graph_with_semantic_edge() + data = first_edge_attributes(G, "a_validate_input", "b_check_input") + assert data.get("confidence_score") == pytest.approx(0.82) + assert data.get("confidence") == "INFERRED" + + +# --------------------------------------------------------------------------- +# Test 3: surprising_connections scores semantically_similar_to edges higher +# than references edges with the same community membership +# --------------------------------------------------------------------------- + +def test_semantic_edge_scores_higher_than_references(): + G = _make_two_edge_graph() + communities = {0: ["a", "b"], 1: ["c", "d"]} + node_community = {"a": 0, "b": 0, "c": 1, "d": 1} + + score_sem, reasons_sem = _surprise_score( + G, "a", "b", first_edge_attributes(G, "a", "b"), node_community, + "auth/validators.py", "api/checks.py" + ) + score_ref, _ = _surprise_score( + G, "c", "d", first_edge_attributes(G, "c", "d"), node_community, + "config/loader.py", "utils/reader.py" + ) + assert score_sem > score_ref + + +def test_semantic_edge_reason_mentions_similarity(): + G = _make_two_edge_graph() + communities = {0: ["a", "b"], 1: ["c", "d"]} + node_community = {"a": 0, "b": 0, "c": 1, "d": 1} + + _, reasons = _surprise_score( + G, "a", "b", first_edge_attributes(G, "a", "b"), node_community, + "auth/validators.py", "api/checks.py" + ) + assert any("similar" in r for r in reasons) + + +# --------------------------------------------------------------------------- +# Test 4: report renders [semantically similar] tag for these edges +# --------------------------------------------------------------------------- + +def _make_report_with_semantic_surprise(): + G = _make_graph_with_semantic_edge() + communities = {0: ["a_validate_input", "b_check_input"]} + cohesion = {0: 0.5} + labels = {0: "Validators"} + gods = [] + surprises = [ + { + "source": "validate_input", + "target": "check_input", + "relation": "semantically_similar_to", + "confidence": "INFERRED", + "confidence_score": 0.82, + "source_files": ["auth/validators.py", "api/checks.py"], + "why": "semantically similar concepts with no structural link", + } + ] + detection = {"total_files": 2, "total_words": 500, "needs_graph": True, "warning": None} + tokens = {"input": 100, "output": 50} + return generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + + +def test_report_renders_semantically_similar_tag(): + report = _make_report_with_semantic_surprise() + assert "[semantically similar]" in report + + +def test_report_semantic_tag_on_correct_line(): + report = _make_report_with_semantic_surprise() + for line in report.splitlines(): + if "semantically_similar_to" in line: + assert "[semantically similar]" in line + break + else: + pytest.fail("No line with semantically_similar_to found in report") + + +def test_report_no_semantic_tag_for_other_relations(): + """Non-semantic edges must not get the [semantically similar] tag.""" + G = graph_from_payload( + [ + {"id": "x", "label": "Alpha", "source_file": "repo1/a.py", "file_type": "code"}, + {"id": "y", "label": "Beta", "source_file": "repo2/b.py", "file_type": "code"}, + ], + [{"source": "x", "target": "y", "relation": "references", + "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": "repo1/a.py", "weight": 1.0}], + ) + + communities = {0: ["x", "y"]} + cohesion = {0: 0.5} + labels = {0: "Misc"} + gods = [] + surprises = [ + { + "source": "Alpha", + "target": "Beta", + "relation": "references", + "confidence": "EXTRACTED", + "source_files": ["repo1/a.py", "repo2/b.py"], + "why": "cross-file connection", + } + ] + detection = {"total_files": 2, "total_words": 200, "needs_graph": True, "warning": None} + tokens = {"input": 50, "output": 25} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "[semantically similar]" not in report diff --git a/tests/restored/test_serve_restored.py b/tests/restored/test_serve_restored.py new file mode 100644 index 000000000..067b87f8c --- /dev/null +++ b/tests/restored/test_serve_restored.py @@ -0,0 +1,28 @@ +"""Retained community-state tests omitted during the native serve conversion.""" + +from graphify.serve import _communities_from_graph +from graphify.helix.state import community_records, new_state +from tests.native_helpers import make_loaded + + +def _loaded(nodes, communities): + state = new_state(communities=community_records(communities)) + return make_loaded(nodes=nodes, state=state) + + +def test_communities_from_graph_basic(): + loaded = _loaded( + [{"id": "n1"}, {"id": "n2"}, {"id": "n3"}], + {0: ["n1", "n2"], 1: ["n3"]}, + ) + assert _communities_from_graph(loaded) == {0: ["n1", "n2"], 1: ["n3"]} + + +def test_communities_from_graph_no_community_attr(): + loaded = _loaded([{"id": "a", "label": "foo"}], {}) + assert _communities_from_graph(loaded) == {} + + +def test_communities_from_graph_isolated(): + loaded = _loaded([{"id": "a"}, {"id": "b"}], {0: ["a"], 2: ["b"]}) + assert _communities_from_graph(loaded) == {0: ["a"], 2: ["b"]} diff --git a/tests/restored/test_wiki_restored.py b/tests/restored/test_wiki_restored.py new file mode 100644 index 000000000..d464857f8 --- /dev/null +++ b/tests/restored/test_wiki_restored.py @@ -0,0 +1,389 @@ +"""Tests for graphify.wiki — Wikipedia-style article generation.""" +import re +import urllib.parse +import pytest +from pathlib import Path +from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article +from tests.native_helpers import graph_from_payload + +_MD_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") + + +def _inline_links(text): + """Yield (display, decoded_target) for each inline markdown link, skipping + external URLs. Targets are URL-decoded so they can be checked against the + on-disk filename. (Display text with an escaped `]` isn't matched, but the + generated labels used in link position never contain brackets.)""" + for display, target in _MD_LINK.findall(text): + if "://" in target: + continue + yield display, urllib.parse.unquote(target) + + +def _make_graph(): + return graph_from_payload( + [ + {"id": "n1", "label": "parse", "file_type": "code", "source_file": "parser.py", "community": 0}, + {"id": "n2", "label": "validate", "file_type": "code", "source_file": "parser.py", "community": 0}, + {"id": "n3", "label": "render", "file_type": "code", "source_file": "renderer.py", "community": 1}, + {"id": "n4", "label": "stream", "file_type": "code", "source_file": "renderer.py", "community": 1}, + ], + [ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}, + {"source": "n1", "target": "n3", "relation": "references", "confidence": "INFERRED", "weight": 1.0}, + {"source": "n3", "target": "n4", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}, + ], + ) + + +COMMUNITIES = {0: ["n1", "n2"], 1: ["n3", "n4"]} +LABELS = {0: "Parsing Layer", 1: "Rendering Layer"} +COHESION = {0: 0.85, 1: 0.72} +GOD_NODES = [{"id": "n1", "label": "parse", "degree": 2}] + + +def _linked_pair(*, same_community: bool = False): + return graph_from_payload( + [ + {"id": "n1", "label": "parse", "file_type": "code", "source_file": "a.py", "community": 0}, + {"id": "n2", "label": "render", "file_type": "code", "source_file": "b.py", "community": 0 if same_community else 1}, + ], + [{"source": "n1", "target": "n2", "relation": "references", "confidence": "INFERRED", "weight": 1.0}], + ) + + +def test_to_wiki_writes_index(tmp_path): + G = _make_graph() + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + assert (tmp_path / "index.md").exists() + + +def test_to_wiki_returns_article_count(tmp_path): + G = _make_graph() + # 2 communities + 1 god node = 3 + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + assert n == 3 + + +def test_to_wiki_community_articles_created(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + assert (tmp_path / "Parsing_Layer.md").exists() + assert (tmp_path / "Rendering_Layer.md").exists() + + +def test_to_wiki_god_node_article_created(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + assert (tmp_path / "parse.md").exists() + + +def test_index_links_all_communities(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + index = (tmp_path / "index.md").read_text() + assert "[Parsing Layer](Parsing_Layer.md)" in index + assert "[Rendering Layer](Rendering_Layer.md)" in index + + +def test_index_lists_god_nodes(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + index = (tmp_path / "index.md").read_text() + assert "[parse](parse.md)" in index + assert "2 connections" in index + + +def test_community_article_has_cross_links(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + # n1 (parsing) references n3 (rendering) → cross-community link + assert "[Rendering Layer](Rendering_Layer.md)" in parsing + + +def test_community_article_shows_cohesion(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + assert "cohesion 0.85" in parsing + + +def test_community_article_has_audit_trail(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + parsing = (tmp_path / "Parsing_Layer.md").read_text() + assert "EXTRACTED" in parsing + assert "INFERRED" in parsing + + +def test_god_node_article_has_connections(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + article = (tmp_path / "parse.md").read_text() + # parse's neighbours (validate, render) have no article of their own, so the + # connections list shows them as plain text rather than as links. + assert "validate" in article and "render" in article + assert "[[" not in article + assert "](validate.md)" not in article and "](render.md)" not in article + + +def test_god_node_article_links_community(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + article = (tmp_path / "parse.md").read_text() + assert "[Parsing Layer](Parsing_Layer.md)" in article + + +def test_to_wiki_skips_missing_god_node_ids(tmp_path): + """God node with bad ID should not crash.""" + G = _make_graph() + bad_gods = [{"id": "nonexistent", "label": "ghost", "degree": 99}] + n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=bad_gods) + # 2 communities + 0 god nodes (nonexistent skipped) = 2 + assert n == 2 + + +def test_to_wiki_no_labels_uses_fallback(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path) # no labels + assert (tmp_path / "Community_0.md").exists() + assert (tmp_path / "Community_1.md").exists() + # fallback "Community N" labels still produce links that resolve to the file + targets = [t for _, t in _inline_links((tmp_path / "index.md").read_text())] + assert "Community_0.md" in targets and (tmp_path / "Community_0.md").exists() + + +def test_article_navigation_footer(tmp_path): + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + article = (tmp_path / "Parsing_Layer.md").read_text() + assert "[index](index.md)" in article + + +def test_community_article_truncation_notice(tmp_path): + """Communities with more than 25 nodes show a truncation notice.""" + nodes = [f"n{i}" for i in range(30)] + G = graph_from_payload( + [{"id": nid, "label": f"concept_{nid}", "file_type": "code", "source_file": "a.py", "community": 0} for nid in nodes], + [{"source": nodes[i], "target": nodes[i + 1], "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0} for i in range(len(nodes) - 1)], + ) + communities = {0: nodes} + to_wiki(G, communities, tmp_path, community_labels={0: "Big Community"}) + article = (tmp_path / "Big_Community.md").read_text() + assert "and 5 more nodes" in article + + +# Regression tests for #925 - cross-community links always empty when node attrs lack community +def test_cross_community_links_without_node_community_attrs(tmp_path): + """Cross-community links must work even when nodes have no 'community' attribute (#925).""" + G = graph_from_payload( + [{"id": "n1", "label": "parse", "file_type": "code", "source_file": "parser.py"}, {"id": "n2", "label": "render", "file_type": "code", "source_file": "renderer.py"}], + [{"source": "n1", "target": "n2", "relation": "references", "confidence": "INFERRED", "weight": 1.0}], + ) + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "Parsing", 1: "Rendering"} + to_wiki(G, communities, tmp_path, community_labels=labels) + article = (tmp_path / "Parsing.md").read_text() + assert "[Rendering](Rendering.md)" in article + + +def test_god_node_article_community_without_node_attr(tmp_path): + """God node article must show community name even when node has no 'community' attr (#925).""" + G = graph_from_payload( + [{"id": "n1", "label": "parse", "file_type": "code", "source_file": "parser.py"}, {"id": "n2", "label": "validate", "file_type": "code", "source_file": "parser.py"}], + [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}], + ) + communities = {0: ["n1", "n2"]} + labels = {0: "Core Logic"} + god_nodes = [{"id": "n1", "label": "parse", "degree": 1}] + to_wiki(G, communities, tmp_path, community_labels=labels, god_nodes_data=god_nodes) + article = (tmp_path / "parse.md").read_text() + assert "[Core Logic](Core_Logic.md)" in article + + +# Regression tests for #936 - stale community node IDs crash to_wiki after dedup/re-extract + +def test_to_wiki_drops_stale_community_nodes(tmp_path): + """Stale node IDs in communities dict are silently dropped without crash (#936).""" + G = _make_graph() + # Add a stale ID that exists in communities but not in G + communities = {0: ["n1", "n2", "stale_ghost"], 1: ["n3", "n4"]} + n = to_wiki(G, communities, tmp_path, community_labels=LABELS) + assert n == 2 # both community articles still written + article = (tmp_path / "Parsing_Layer.md").read_text() + assert "parse" in article + assert "stale_ghost" not in article + + +def test_to_wiki_all_stale_raises(tmp_path): + """If every community node is stale, raise ValueError with a helpful message (#936).""" + G = _make_graph() + all_stale = {0: ["ghost1", "ghost2"], 1: ["ghost3"]} + with pytest.raises(ValueError, match="stale"): + to_wiki(G, all_stale, tmp_path, community_labels=LABELS) + + +def test_to_wiki_stale_nodes_prints_warning(tmp_path, capsys): + """Stale node IDs trigger a stderr warning showing the drop count (#936).""" + G = _make_graph() + communities = {0: ["n1", "stale1", "stale2"], 1: ["n3", "n4"]} + to_wiki(G, communities, tmp_path, community_labels=LABELS) + err = capsys.readouterr().err + assert "2" in err # dropped count + assert "stale" in err.lower() + + +def test_community_article_handles_null_source_file(tmp_path): + """source_file=None on a node must not crash sorted() with TypeError (#1016).""" + G = graph_from_payload( + [{"id": "n1", "label": "parse", "file_type": "code", "source_file": None, "community": 0}, {"id": "n2", "label": "validate", "file_type": "code", "source_file": "parser.py", "community": 0}], + [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}], + ) + communities = {0: ["n1", "n2"]} + labels = {0: "Parsing Layer"} + # Must not raise TypeError + to_wiki(G, communities, tmp_path, community_labels=labels) + assert (tmp_path / "index.md").exists() + + +def test_to_wiki_case_only_distinct_labels_dont_overwrite(tmp_path): + """Two community labels differing only by case must each get their own + article. The slug-dedup set folds case, so on case-insensitive filesystems + (macOS/APFS, Windows/NTFS) the second article gets a numeric suffix instead + of silently overwriting the first.""" + G = _linked_pair() + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "Parser", 1: "parser"} + n = to_wiki(G, communities, tmp_path, community_labels=labels) + articles = [p for p in tmp_path.glob("*.md") if p.name != "index.md"] + # both communities survive as separate files on disk (no silent overwrite) + assert len(articles) == n == 2, [p.name for p in articles] + # filenames are distinct even when compared case-insensitively + lowered = [p.stem.lower() for p in articles] + assert len(set(lowered)) == len(lowered), [p.name for p in articles] + + +def test_to_wiki_god_node_label_case_collides_with_community(tmp_path): + """Community and god-node articles share one slug-dedup set, so a god-node + label differing only by case from a community label must still get its own + file rather than overwriting the community article.""" + G = _linked_pair(same_community=True) + communities = {0: ["n1", "n2"]} + labels = {0: "Parser"} + god_nodes = [{"id": "n1", "label": "parser", "degree": 1}] + n = to_wiki(G, communities, tmp_path, community_labels=labels, god_nodes_data=god_nodes) + articles = [p for p in tmp_path.glob("*.md") if p.name != "index.md"] + assert len(articles) == n == 2, [p.name for p in articles] + lowered = [p.stem.lower() for p in articles] + assert len(set(lowered)) == len(lowered), [p.name for p in articles] + + +# Regression tests for portable wiki links - Obsidian [[wikilinks]] break in +# every non-Obsidian renderer (VS Code preview, GitHub, GitLab, plain browsers). + + +def test_wiki_emits_no_obsidian_wikilinks(tmp_path): + """No generated file may contain Obsidian [[...]] syntax. Those links resolve + only inside Obsidian (by note title); everywhere else [[Domain Data Models]] + points at a literal `Domain Data Models.md` that doesn't exist.""" + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + for md in tmp_path.glob("*.md"): + assert "[[" not in md.read_text(), md.name + + +def test_wiki_links_resolve_to_real_files(tmp_path): + """Every inline markdown link target across the whole wiki must point at a + file that actually exists on disk. The display text may keep spaces/special + characters, but the target is the URL-encoded slug, so it has to round-trip + back to a real filename in any renderer.""" + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) + seen_link = False + for md in tmp_path.glob("*.md"): + for display, target in _inline_links(md.read_text()): + seen_link = True + assert (tmp_path / target).exists(), f"{md.name}: [{display}] -> {target} is dead" + # guard against the test passing vacuously if links ever stop being emitted + assert seen_link, "expected the wiki to contain inline markdown links" + + +def test_wiki_link_display_keeps_label_but_target_is_filename(tmp_path): + """The fix's whole point: a link's display text is the human label (with + spaces) while its target is the on-disk slug (underscores). This is what + [[Domain Data Models]] could never express portably.""" + G = _make_graph() + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) + index = (tmp_path / "index.md").read_text() + assert "[Parsing Layer](Parsing_Layer.md)" in index + assert "Parsing Layer.md" not in index # the broken Obsidian-only target + + +def test_wiki_special_characters_in_label_resolve(tmp_path): + """Labels with spaces, &, #, and parentheses must still produce a link whose + URL-encoded target decodes back to the real (underscored) filename, so it + works in CommonMark renderers and Obsidian alike. # is the dangerous one — + left raw in a relative link it would be misread as a fragment.""" + G = _linked_pair() + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "C# & Auth (v2)", 1: "Other"} + to_wiki(G, communities, tmp_path, community_labels=labels) + article = (tmp_path / "Other.md").read_text() + # the cross-link to the special-char community resolves to its real file + targets = [t for _, t in _inline_links(article)] + assert "C#_&_Auth_(v2).md" in targets + assert (tmp_path / "C#_&_Auth_(v2).md").exists() + # the raw target is fully percent-encoded — no bare ( ) that would terminate + # the link early, no bare # that would be misread as a fragment + assert "C%23_%26_Auth_%28v2%29.md" in article + + +def test_wiki_link_with_bracketed_label_resolves(tmp_path): + """A label containing `[` / `]` (e.g. a generic like `Array[T]`) still + produces a resolvable link: the brackets are escaped in the display text so + they don't break the markdown, and percent-encoded in the target so it + decodes back to the real file. (`_safe_filename` keeps brackets in the slug, + so they reach the link target.)""" + G = _linked_pair() + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "Array[T] Models", 1: "Other"} + to_wiki(G, communities, tmp_path, community_labels=labels) + article = (tmp_path / "Other.md").read_text() + assert r"[Array\[T\] Models](Array%5BT%5D_Models.md)" in article + assert (tmp_path / "Array[T]_Models.md").exists() + + +def test_wiki_links_to_nodes_without_articles_are_plain_text(tmp_path): + """A god node links its neighbours, but only communities and god nodes get + article files — neighbours without one must render as plain text, not as a + link (dead even inside Obsidian).""" + G = _make_graph() + # only `parse` (n1) is a god node; its neighbours validate/render are not, + # and have no article of their own + to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) + article = (tmp_path / "parse.md").read_text() + assert "validate" in article and "render" in article + # they appear as plain list items, not links + assert "- validate" in article and "- render" in article + # not wrapped in an Obsidian wikilink (the old form — dead even in Obsidian + # since validate/render have no article)... + assert "[[validate]]" not in article and "[[render]]" not in article + # ...nor in a standard link to a non-existent article file + for _, target in _inline_links(article): + assert target not in ("validate.md", "render.md"), target + + +def test_wiki_links_use_collision_suffixed_slug(tmp_path): + """When two labels collide on disk and the second article gets a numeric + suffix (`parser_2.md`), links to it must target the suffixed slug, not the + bare label. The resolver records the exact filename each article was written + under, so the link target tracks the collision suffix.""" + G = _linked_pair() + communities = {0: ["n1"], 1: ["n2"]} + labels = {0: "Parser", 1: "parser"} # collide case-insensitively + to_wiki(G, communities, tmp_path, community_labels=labels) + index_targets = [t for _, t in _inline_links((tmp_path / "index.md").read_text())] + assert "parser_2.md" in index_targets # link points at the suffixed file... + for t in index_targets: + assert (tmp_path / t).exists(), t # ...and every target is a real file diff --git a/tests/test_affected_cli.py b/tests/test_affected_cli.py index ca608b6b3..ee61142c1 100644 --- a/tests/test_affected_cli.py +++ b/tests/test_affected_cli.py @@ -1,313 +1,137 @@ -from __future__ import annotations - -import json - -import networkx as nx -from networkx.readwrite import json_graph - -import graphify.__main__ as mainmod - - -def _write_graph(tmp_path): - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_node("barrel", label="__init__.py", source_file="pkg/__init__.py", source_location=None) - graph.add_node("consumer", label="app.py", source_file="app.py", source_location=None) - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - graph.add_edge("barrel", "target", relation="re_exports", context="export", confidence="EXTRACTED") - graph.add_edge("consumer", "target", relation="imports", context="import", confidence="EXTRACTED") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") - return graph_path - - -def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, - "argv", - ["graphify", "affected", "Foo", "--graph", str(graph_path)], +from graphify import __main__ as mainmod +from graphify.affected import affected_nodes, resolve_seed +from tests.native_helpers import make_loaded + + +def _loaded(tmp_path): + return make_loaded( + tmp_path, + kind="digraph", + nodes=[ + {"id": "foo", "label": "Foo()", "source_file": "foo.py", "file_type": "code"}, + {"id": "bar", "label": "Bar", "source_file": "bar.py", "file_type": "code"}, + {"id": "baz", "label": "Baz", "source_file": "baz.py", "file_type": "code"}, + ], + edges=[ + {"source": "bar", "target": "foo", "relation": "calls"}, + {"source": "baz", "target": "bar", "relation": "imports"}, + ], ) - mainmod.main() - out = capsys.readouterr().out - assert "Affected nodes for Foo" in out - assert "X()" in out - assert "calls" in out - assert "__init__.py" in out - assert "re_exports" in out - assert "app.py" in out - assert "imports" in out +def test_native_reverse_traversal_and_relation_filter(tmp_path): + loaded = _loaded(tmp_path) + graph = loaded.graph + assert resolve_seed(graph, "Foo", node_query=loaded.query) == "foo" + all_hits = affected_nodes(graph, "foo", relations={"calls", "imports"}, depth=2) + assert {row.node_id for row in all_hits} == {"bar", "baz"} + calls = affected_nodes(graph, "foo", relations={"calls"}, depth=2) + assert {row.node_id for row in calls} == {"bar"} -def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_cli_reads_helix_store(tmp_path, monkeypatch, capsys): + loaded = _loaded(tmp_path) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", "Foo", "--relation", "calls", "--graph", str(graph_path)], + ["graphify", "affected", "Foo", "--store", str(loaded.store_path)], ) - mainmod.main() + output = capsys.readouterr().out + assert "Bar" in output and "Baz" in output - out = capsys.readouterr().out - assert "Relations: calls" in out - assert "X()" in out - assert "__init__.py" not in out - - -def test_affected_cli_forces_directed_on_undirected_graph(monkeypatch, tmp_path, capsys): - """A graph persisted with directed=false must still recover caller->callee - direction (#1174): affected on the callee returns the caller, not the callee - or nothing. Without forcing directed=True, node_link_graph builds an - undirected Graph, predecessors() collapses, and the reverse traversal breaks. - """ - graph = nx.DiGraph() - graph.add_node("A", label="caller_fn", source_file="a.py", source_location="L1") - graph.add_node("B", label="callee_fn", source_file="b.py", source_location="L2") - graph.add_edge("A", "B", relation="calls", context="call", confidence="EXTRACTED") - - data = json_graph.node_link_data(graph, edges="links") - # Persist as undirected on disk to reproduce the bug condition. - data["directed"] = False - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(data), encoding="utf-8") +def test_cli_rejects_legacy_json_path(tmp_path, monkeypatch, capsys): + legacy = tmp_path / "legacy.json" + legacy.write_text("{}") monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", "B", "--relation", "calls", "--graph", str(graph_path)], + ["graphify", "affected", "Foo", "--store", str(legacy)], + ) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code == 1 + assert "obsolete" in capsys.readouterr().err + + +def _callsite_graph(tmp_path, *, with_location: bool): + edge = { + "source": "loader", + "target": "transition", + "relation": "calls", + "confidence": "EXTRACTED", + } + if with_location: + edge.update( + source_file="apollo_pipeline_status.py", + source_location="L158", + ) + return make_loaded( + tmp_path, + kind="digraph", + nodes=[ + { + "id": "loader", + "label": "_load_apollo_app_state()", + "source_file": "apollo_pipeline_status.py", + "source_location": "L90", + }, + { + "id": "transition", + "label": "transition_state()", + "source_file": "state.py", + "source_location": "L56", + }, + ], + edges=[edge], ) - mainmod.main() - - out = capsys.readouterr().out - # A (the caller) is affected by a change to B (the callee). - assert "caller_fn" in out - assert "calls" in out - # B is the query node, not an affected node, and the result is not empty. - assert "No affected nodes found." not in out - - -def test_affected_cli_loads_edges_keyed_graph(monkeypatch, tmp_path, capsys): - """graphify's `extract` writes graph.json with an "edges" key (not networkx's - default "links"). affected.load_graph must handle it; before the edges/links - normalization it raised an uncaught KeyError: 'links' (same class as #1198).""" - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - - # Emulate graphify extract output: top-level "edges" key instead of "links". - data = json_graph.node_link_data(graph, edges="links") - data["edges"] = data.pop("links") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(data), encoding="utf-8") +def test_affected_reports_call_site_line_not_def_line( + monkeypatch, tmp_path, capsys +): + loaded = _callsite_graph(tmp_path, with_location=True) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", "Foo", "--graph", str(graph_path)], + [ + "graphify", + "affected", + "transition_state", + "--store", + str(loaded.store_path), + ], ) mainmod.main() - out = capsys.readouterr().out - assert "Affected nodes for Foo" in out - assert "X()" in out - assert "calls" in out - - -def test_resolve_seed_bare_name_matches_callable_label(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="classifyProperty()", source_file="pkg/entity.py") - graph.add_node("b", label="classifyPropertySafe()", source_file="app/context.py") - - assert resolve_seed(graph, "classifyProperty") == "a" - assert resolve_seed(graph, "classifyPropertySafe") == "b" - - -def test_resolve_seed_decorated_query_matches_bare_label(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="Foo", source_file="pkg/foo.py") - graph.add_node("b", label="FooBar", source_file="pkg/foobar.py") - - assert resolve_seed(graph, "Foo()") == "a" - - -def test_resolve_seed_matches_unicode_normalized_label(): - import unicodedata - - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="Auditoría", source_file="pkg/auditoria.py") - - assert resolve_seed(graph, unicodedata.normalize("NFD", "Auditoría")) == "a" - - -def test_resolve_seed_preserves_distinct_accents(): - from graphify.affected import resolve_seed + output = capsys.readouterr().out + assert "apollo_pipeline_status.py:L158" in output + assert "apollo_pipeline_status.py:L90" not in output - graph = nx.DiGraph() - graph.add_node("a", label="resume", source_file="pkg/resume.py") - graph.add_node("b", label="résumé", source_file="pkg/resume_accented.py") - - assert resolve_seed(graph, "resume") == "a" - - -def test_resolve_seed_bare_name_tie_still_returns_none(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="dup()", source_file="pkg/one.py") - graph.add_node("b", label="dup()", source_file="pkg/two.py") - - assert resolve_seed(graph, "dup") is None - - -def test_resolve_seed_source_file_path_prefers_file_level_node(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node( - "example_route_get", - label="GET()", - source_file=source_file, - source_location="L42", - ) - graph.add_node( - "example_route", - label="route.ts", - source_file=source_file, - source_location="L1", - ) - - assert resolve_seed(graph, source_file) == "example_route" - - -def test_resolve_seed_source_file_trailing_slash_parity(): - """A trailing path separator must not change the match (parity with explain's - _find_node, which tokenizes the path and drops the slash).""" - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node("get", label="GET()", source_file=source_file, source_location="L42") - graph.add_node("file", label="route.ts", source_file=source_file, source_location="L1") - - assert resolve_seed(graph, source_file + "/") == "file" - - -def test_resolve_seed_source_file_ambiguous_no_file_node_returns_none(): - """Several nodes share a source_file but none is the L1 file node and none's - basename matches the path — must not guess; return None.""" - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "pkg/handlers.py" - graph.add_node("a", label="handle_a()", source_file=source_file, source_location="L10") - graph.add_node("b", label="handle_b()", source_file=source_file, source_location="L20") - - assert resolve_seed(graph, source_file) is None - - -def test_affected_cli_source_file_path_uses_file_level_node(monkeypatch, tmp_path, capsys): - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node( - "example_route_get", - label="GET()", - source_file=source_file, - source_location="L42", - ) - graph.add_node( - "example_route", - label="route.ts", - source_file=source_file, - source_location="L1", - ) - graph.add_node( - "consumer", - label="consumer.ts", - source_file="app/consumer.ts", - source_location="L1", - ) - graph.add_edge( - "consumer", - "example_route", - relation="imports_from", - context="import", - confidence="EXTRACTED", - ) - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") +def test_affected_falls_back_to_def_line_when_edge_has_no_location( + monkeypatch, tmp_path, capsys +): + loaded = _callsite_graph(tmp_path, with_location=False) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", source_file, "--graph", str(graph_path)], + [ + "graphify", + "affected", + "transition_state", + "--store", + str(loaded.store_path), + ], ) mainmod.main() - out = capsys.readouterr().out - assert "Affected nodes for route.ts" in out - assert "consumer.ts" in out - assert "imports_from" in out - assert "No unique node matched" not in out - - -# ── BUG1: caller lists must show the call-SITE line, not the caller def line ── - -def _write_callsite_graph(tmp_path): - """A caller whose call site (L158) differs from its own def line (L90).""" - g = nx.DiGraph() - g.add_node("loader", label="_load_apollo_app_state()", - source_file="apollo_pipeline_status.py", source_location="L90") - g.add_node("transition", label="transition_state()", - source_file="state.py", source_location="L56") - # The call happens at line 158 inside the caller's file. - g.add_edge("loader", "transition", relation="calls", context="call", - confidence="EXTRACTED", source_file="apollo_pipeline_status.py", - source_location="L158") - gp = tmp_path / "graph.json" - gp.write_text(json.dumps(json_graph.node_link_data(g, edges="links")), encoding="utf-8") - return gp - - -def test_affected_reports_call_site_line_not_def_line(monkeypatch, tmp_path, capsys): - gp = _write_callsite_graph(tmp_path) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr(mainmod.sys, "argv", - ["graphify", "affected", "transition_state", "--graph", str(gp)]) - mainmod.main() - out = capsys.readouterr().out - assert "apollo_pipeline_status.py:L158" in out, "must report the call SITE line (BUG1)" - assert "apollo_pipeline_status.py:L90" not in out, "must NOT report the caller's def line" - - -def test_affected_falls_back_to_def_line_when_edge_has_no_location(monkeypatch, tmp_path, capsys): - """An edge with no stored location honestly falls back to the node's def line.""" - g = nx.DiGraph() - g.add_node("loader", label="load()", source_file="a.py", source_location="L90") - g.add_node("t", label="target()", source_file="b.py", source_location="L5") - g.add_edge("loader", "t", relation="calls", confidence="INFERRED") # no source_location - gp = tmp_path / "graph.json" - gp.write_text(json.dumps(json_graph.node_link_data(g, edges="links")), encoding="utf-8") - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "affected", "target", "--graph", str(gp)]) - mainmod.main() - assert "a.py:L90" in capsys.readouterr().out + assert "apollo_pipeline_status.py:L90" in capsys.readouterr().out diff --git a/tests/test_affected_member_seed.py b/tests/test_affected_member_seed.py index d1f6b3ae7..2b5f07d23 100644 --- a/tests/test_affected_member_seed.py +++ b/tests/test_affected_member_seed.py @@ -1,61 +1,20 @@ -"""#1669 — affected must reach callers that bind to the class's method -nodes (post-#1634 method-granularity resolution), by seeding the reverse walk -with the root's member nodes (one method/contains hop). method/contains stay out -of the general relation-filtered walk, so no forward noise is added elsewhere. -""" -from __future__ import annotations - -import networkx as nx - from graphify.affected import affected_nodes - - -def _g(): - g = nx.DiGraph() - for nid, label in [ - ("proc", "Processor"), ("proc_call", ".call()"), - ("runner", "Runner"), ("runner_run", ".run()"), - ]: - g.add_node(nid, label=label) - g.add_edge("proc", "proc_call", relation="method") # class owns method - g.add_edge("runner", "runner_run", relation="method") - g.add_edge("runner_run", "proc_call", relation="calls") # caller binds to method node (#1634) - return g - - -def test_class_affected_reaches_method_bound_caller(): - g = _g() - hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} - assert "runner_run" in hits, "caller of Processor.call must be reachable from Processor" - - -def test_member_method_node_not_reported_as_hit(): - g = _g() - hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} - # the class's own method node is a seed, not an affected node - assert "proc_call" not in hits - - -def test_method_contains_still_excluded_from_general_walk(): - # A node two method-hops away (method of a DIFFERENT class discovered during - # the walk) must NOT be pulled in: only the root's own members are seeded. - g = nx.DiGraph() - for nid, label in [("a", "A"), ("a_m", ".m()"), ("b", "B"), ("b_m", ".n()")]: - g.add_node(nid, label=label) - g.add_edge("a", "a_m", relation="method") - g.add_edge("a_m", "b", relation="calls") # A.m calls class B - g.add_edge("b", "b_m", relation="method") # B's own method - hits = {h.node_id for h in affected_nodes(g, "a", depth=3)} - # We seeded A's members and walk reverse; B and B's method are downstream of A - # (A.m -> B), not reverse-callers of A, so they must not appear. - assert hits == set() or "b_m" not in hits - - -def test_class_level_caller_still_works(): - # A caller bound to the class node itself (not a method) is unaffected. - g = nx.DiGraph() - g.add_node("svc", label="Svc") - g.add_node("caller", label=".use()") - g.add_edge("caller", "svc", relation="references") - hits = {h.node_id for h in affected_nodes(g, "svc", depth=2)} - assert "caller" in hits +from tests.native_helpers import graph_from_payload + + +def test_class_seed_reaches_method_bound_caller_without_reporting_member(): + graph = graph_from_payload( + [ + {"id": "class", "label": "Service", "source_file": "service.py"}, + {"id": "method", "label": "Service.run", "source_file": "service.py"}, + {"id": "caller", "label": "Controller", "source_file": "controller.py"}, + ], + [ + {"source": "class", "target": "method", "relation": "contains"}, + {"source": "caller", "target": "method", "relation": "calls"}, + ], + kind="digraph", + ) + hits = affected_nodes(graph, "class", relations={"calls", "contains"}, depth=2) + assert "caller" in {row.node_id for row in hits} + assert "method" not in {row.node_id for row in hits} diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7bff432cf..ed20bd05b 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,736 +1,135 @@ -"""Tests for analyze.py.""" -import json -import networkx as nx -import pytest -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff, _surprise_score, _file_category, _is_json_key_node, find_import_cycles, suggest_questions -from graphify.extract import _make_id - -FIXTURES = Path(__file__).parent / "fixtures" - - -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) - - -def test_god_nodes_returns_list(): - G = make_graph() - result = god_nodes(G, top_n=3) - assert isinstance(result, list) - assert len(result) <= 3 - - -def test_god_nodes_sorted_by_degree(): - G = make_graph() - result = god_nodes(G, top_n=10) - degrees = [r["degree"] for r in result] - assert degrees == sorted(degrees, reverse=True) - - -def test_god_nodes_have_required_keys(): - G = make_graph() - result = god_nodes(G, top_n=1) - assert "id" in result[0] - assert "label" in result[0] - assert "degree" in result[0] - - -def test_surprising_connections_cross_source_multi_file(): - """Multi-file graph: should find cross-file edges between real entities.""" - G = make_graph() - communities = cluster(G) - surprises = surprising_connections(G, communities) - assert len(surprises) > 0 - for s in surprises: - assert s["source_files"][0] != s["source_files"][1] - - -def test_surprising_connections_excludes_concept_nodes(): - """Concept nodes (empty source_file) must not appear in surprises.""" - G = make_graph() - # Add a concept node with empty source_file - G.add_node("concept_x", label="Abstract Concept", file_type="document", source_file="") - G.add_edge("n_transformer", "concept_x", relation="relates_to", - confidence="INFERRED", source_file="", weight=0.5) - communities = cluster(G) - surprises = surprising_connections(G, communities) - labels = [s["source"] for s in surprises] + [s["target"] for s in surprises] - assert "Abstract Concept" not in labels - - -def test_surprising_connections_single_file_uses_community_bridges(): - """Single-file graph: should return cross-community edges, not empty list.""" - G = nx.Graph() - # Build a graph with 2 clear communities + 1 bridge edge - for i in range(5): - G.add_node(f"a{i}", label=f"A{i}", file_type="code", source_file="single.py", - source_location=f"L{i}") - for i in range(5): - G.add_node(f"b{i}", label=f"B{i}", file_type="code", source_file="single.py", - source_location=f"L{i+10}") - # Dense intra-community edges - for i in range(4): - G.add_edge(f"a{i}", f"a{i+1}", relation="calls", confidence="EXTRACTED", - source_file="single.py", weight=1.0) - for i in range(4): - G.add_edge(f"b{i}", f"b{i+1}", relation="calls", confidence="EXTRACTED", - source_file="single.py", weight=1.0) - # One cross-community bridge - G.add_edge("a4", "b0", relation="references", confidence="INFERRED", - source_file="single.py", weight=0.5) - - communities = cluster(G) - surprises = surprising_connections(G, communities) - # Should find at least the bridge edge - assert len(surprises) > 0 - - -def test_surprising_connections_ambiguous_scores_higher_than_extracted(): - """AMBIGUOUS edge should score higher than an otherwise identical EXTRACTED edge.""" - G = nx.Graph() - for nid, label, src in [ - ("a", "Alpha", "repo1/model.py"), - ("b", "Beta", "repo2/train.py"), - ("c", "Gamma", "repo1/data.py"), - ("d", "Delta", "repo2/eval.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - G.add_edge("a", "b", relation="calls", confidence="AMBIGUOUS", weight=1.0, source_file="repo1/model.py") - G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="repo1/data.py") - communities = {0: ["a", "c"], 1: ["b", "d"]} - nc = {"a": 0, "c": 0, "b": 1, "d": 1} - score_amb, _ = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "repo1/model.py", "repo2/train.py") - score_ext, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "repo1/data.py", "repo2/eval.py") - assert score_amb > score_ext - - -def test_surprise_score_accepts_precomputed_degrees(): - G = nx.Graph() - for nid, label, src in [ - ("hub", "Hub", "repo1/hub.py"), - ("leaf", "Leaf", "repo2/leaf.py"), - ("n1", "N1", "repo1/n1.py"), - ("n2", "N2", "repo1/n2.py"), - ("n3", "N3", "repo1/n3.py"), - ("n4", "N4", "repo1/n4.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - for node in ("leaf", "n1", "n2", "n3", "n4"): - G.add_edge("hub", node, relation="calls", confidence="EXTRACTED", weight=1.0) - - nc = {"hub": 0, "leaf": 1} - edge = G.edges["hub", "leaf"] - args = (G, "hub", "leaf", edge, nc, "repo1/hub.py", "repo2/leaf.py") - - assert _surprise_score(*args) == _surprise_score(*args, dict(G.degree())) - - -def test_surprising_connections_cross_type_scores_higher(): - """Code↔paper edge should score higher than code↔code edge.""" - G = nx.Graph() - for nid, label, src in [ - ("a", "Transformer", "code/model.py"), - ("b", "FlashAttn", "papers/flash.pdf"), - ("c", "Trainer", "code/train.py"), - ("d", "Dataset", "code/data.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - G.add_edge("a", "b", relation="references", confidence="EXTRACTED", weight=1.0, source_file="code/model.py") - G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="code/train.py") - nc = {"a": 0, "b": 1, "c": 0, "d": 0} - score_cross, reasons_cross = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "code/model.py", "papers/flash.pdf") - score_same, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "code/train.py", "code/data.py") - assert score_cross > score_same - assert any("code" in r and "paper" in r for r in reasons_cross) - - -def _make_cross_lang_graph(): - """Helper: Python node in backend/, TypeScript node in frontend/, different communities.""" - G = nx.Graph() - G.add_node("py_auth", label="AuthError", source_file="backend/auth.py", file_type="code") - G.add_node("ts_member", label="Member", source_file="frontend/types.ts", file_type="code") - G.add_node("py_a", label="ServiceA", source_file="backend/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="backend/utils.py", file_type="code") - return G - - -def test_cross_language_inferred_calls_suppressed(): - """Cross-language INFERRED calls edge should score lower than same-language EXTRACTED.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="calls", confidence="INFERRED", - weight=0.8, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_cross, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_cross <= score_same - - -def test_cross_language_inferred_uses_suppressed(): - """Cross-language INFERRED uses edge (the exact rsl-siege-manager false positive) should be suppressed.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="uses", confidence="INFERRED", - weight=0.8, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_cross, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_cross <= score_same - - -def test_cross_language_semantically_similar_not_suppressed(): - """`semantically_similar_to` across languages is a genuine insight — must not be suppressed.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="semantically_similar_to", - confidence="INFERRED", weight=0.85, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_sem, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_sem > score_same - - -def test_same_language_inferred_calls_not_suppressed(): - """INFERRED calls within the same language family must not be affected.""" - G = nx.Graph() - G.add_node("py_a", label="ModuleA", source_file="src/a.py", file_type="code") - G.add_node("py_b", label="ModuleB", source_file="src/b.py", file_type="code") - G.add_node("py_c", label="ModuleC", source_file="src/c.py", file_type="code") - G.add_node("py_d", label="ModuleD", source_file="src/d.py", file_type="code") - G.add_edge("py_a", "py_b", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/a.py") - G.add_edge("py_c", "py_d", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/c.py") - nc = {"py_a": 0, "py_b": 1, "py_c": 0, "py_d": 1} - score_inf, _ = _surprise_score(G, "py_a", "py_b", G.edges["py_a", "py_b"], nc, - "src/a.py", "src/b.py") - score_ext, _ = _surprise_score(G, "py_c", "py_d", G.edges["py_c", "py_d"], nc, - "src/c.py", "src/d.py") - assert score_inf > score_ext - - -def test_cross_language_extracted_calls_not_suppressed(): - """EXTRACTED cross-language edges are real structural facts — must not be penalised.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/auth.py") - nc = {"py_auth": 0, "ts_member": 1} - score, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - assert score >= 1 - - -def test_surprising_connections_have_why_field(): - G = make_graph() - communities = cluster(G) - for s in surprising_connections(G, communities): - assert "why" in s - assert isinstance(s["why"], str) - assert len(s["why"]) > 0 - - -def test_file_category(): - assert _file_category("model.py") == "code" - assert _file_category("flash.pdf") == "paper" - assert _file_category("diagram.png") == "image" - assert _file_category("notes.md") == "doc" - # Languages added in later releases — would misclassify as "doc" without detect.py import - assert _file_category("app.swift") == "code" - assert _file_category("plugin.lua") == "code" - assert _file_category("build.zig") == "code" - assert _file_category("deploy.ps1") == "code" - assert _file_category("server.ex") == "code" - assert _file_category("component.jsx") == "code" - assert _file_category("analysis.jl") == "code" - assert _file_category("view.m") == "code" - - -def test_is_concept_node_empty_source(): - G = nx.Graph() - G.add_node("c1", source_file="") - assert _is_concept_node(G, "c1") is True - - -def test_is_concept_node_real_file(): - G = nx.Graph() - G.add_node("n1", source_file="model.py") - assert _is_concept_node(G, "n1") is False - - -def test_surprising_connections_have_required_keys(): - G = make_graph() - communities = cluster(G) - for s in surprising_connections(G, communities): - assert "source" in s - assert "target" in s - assert "source_files" in s - assert "confidence" in s - - -# --- graph_diff tests --- - -def _make_simple_graph(nodes, edges): - """Helper: build a small nx.Graph from node/edge specs.""" - G = nx.Graph() - for node_id, label in nodes: - G.add_node(node_id, label=label, source_file="test.py") - for src, tgt, rel, conf in edges: - G.add_edge(src, tgt, relation=rel, confidence=conf) - return G - - -def test_graph_diff_new_nodes(): - G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) - G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) - diff = graph_diff(G_old, G_new) - assert len(diff["new_nodes"]) == 1 - assert diff["new_nodes"][0]["id"] == "n3" - assert diff["new_nodes"][0]["label"] == "Gamma" - assert diff["removed_nodes"] == [] - assert "1 new node" in diff["summary"] - - -def test_graph_diff_removed_nodes(): - G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) - G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) - diff = graph_diff(G_old, G_new) - assert diff["new_nodes"] == [] - assert len(diff["removed_nodes"]) == 1 - assert diff["removed_nodes"][0]["id"] == "n3" - assert "removed" in diff["summary"] - - -def test_graph_diff_new_edges(): - nodes = [("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")] - G_old = _make_simple_graph(nodes, [("n1", "n2", "calls", "EXTRACTED")]) - G_new = _make_simple_graph( - nodes, - [("n1", "n2", "calls", "EXTRACTED"), ("n2", "n3", "uses", "INFERRED")], +"""Analysis behavior on real embedded-Helix snapshots.""" + +from graphify.analyze import ( + _file_category, + _is_concept_node, + _is_json_key_node, + _surprise_score, + find_import_cycles, + god_nodes, + graph_diff, + suggest_questions, + surprising_connections, +) +from graphify.helix.access import first_edge_attributes +from tests.native_helpers import graph_from_payload, triangle + + +def _graph(): + return graph_from_payload( + [ + {"id": "hub", "label": "Hub", "file_type": "code", "source_file": "src/hub.py"}, + {"id": "left", "label": "Left", "file_type": "code", "source_file": "src/left.py"}, + {"id": "right", "label": "Right", "file_type": "code", "source_file": "web/right.ts"}, + {"id": "doc", "label": "Design", "file_type": "document", "source_file": "docs/design.md"}, + ], + [ + {"source": "hub", "target": "left", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "hub", "target": "right", "relation": "references", "confidence": "INFERRED"}, + {"source": "hub", "target": "doc", "relation": "documents", "confidence": "INFERRED"}, + ], + kind="digraph", ) - diff = graph_diff(G_old, G_new) - assert len(diff["new_edges"]) == 1 - new_edge = diff["new_edges"][0] - assert new_edge["relation"] == "uses" - assert new_edge["confidence"] == "INFERRED" - assert diff["removed_edges"] == [] - assert "new edge" in diff["summary"] - - -def test_graph_diff_empty_diff(): - nodes = [("n1", "Alpha"), ("n2", "Beta")] - edges = [("n1", "n2", "calls", "EXTRACTED")] - G_old = _make_simple_graph(nodes, edges) - G_new = _make_simple_graph(nodes, edges) - diff = graph_diff(G_old, G_new) - assert diff["new_nodes"] == [] - assert diff["removed_nodes"] == [] - assert diff["new_edges"] == [] - assert diff["removed_edges"] == [] - assert diff["summary"] == "no changes" - - -# --- code↔doc INFERRED suppression tests --- - -def _make_code_doc_graph(): - G = nx.Graph() - G.add_node("py_fn", label="ProcessData", source_file="src/processor.py", file_type="code") - G.add_node("md_doc", label="README Section", source_file="docs/readme.md", file_type="document") - G.add_node("py_a", label="ServiceA", source_file="src/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="src/utils.py", file_type="code") - return G - - -def test_code_doc_inferred_calls_suppressed(): - """Code→doc INFERRED calls edge should score lower than same-language EXTRACTED.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_noise, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_real, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_noise <= score_real -def test_code_doc_inferred_uses_suppressed(): - """Code→doc INFERRED uses edge should score lower than same-language EXTRACTED.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="uses", confidence="INFERRED", - weight=0.8, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_noise, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_real, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_noise <= score_real - - -def test_code_doc_extracted_calls_not_suppressed(): - """EXTRACTED code↔doc edges are real facts — must not be penalised.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/processor.py") - nc = {"py_fn": 0, "md_doc": 1} - score, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - assert score >= 1 - - -def test_code_doc_inferred_semantically_similar_not_suppressed(): - """`semantically_similar_to` across code↔doc is explicit LLM insight — must not be suppressed.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="semantically_similar_to", - confidence="INFERRED", weight=0.85, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_sem, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_sem > score_same - - -def test_code_unknown_extension_inferred_calls_suppressed(): - """_file_category falls back to 'doc' for unknown extensions, so INFERRED - calls/uses to unknown-extension files are suppressed the same as code↔doc.""" - assert _file_category("vendor/random.xyz") == "doc" - G = nx.Graph() - G.add_node("py_fn", label="Handler", source_file="src/handler.py", file_type="code") - G.add_node("unk", label="Handler", source_file="vendor/unknown.xyz", file_type="document") - G.add_node("py_a", label="A", source_file="src/a.py", file_type="code") - G.add_node("py_b", label="B", source_file="src/b.py", file_type="code") - G.add_edge("py_fn", "unk", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/handler.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/a.py") - nc = {"py_fn": 0, "unk": 1, "py_a": 0, "py_b": 0} - score_unk, _ = _surprise_score(G, "py_fn", "unk", - G.edges["py_fn", "unk"], nc, - "src/handler.py", "vendor/unknown.xyz") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/a.py", "src/b.py") - assert score_unk <= score_same - - -def test_code_paper_inferred_calls_not_suppressed(): - """Code↔paper INFERRED calls should still surface — it is a meaningful link.""" - G = nx.Graph() - G.add_node("py_model", label="Transformer", source_file="src/model.py", file_type="code") - G.add_node("pdf_paper", label="Attention Is All You Need", source_file="papers/vaswani.pdf", - file_type="paper") - G.add_node("py_a", label="ServiceA", source_file="src/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="src/utils.py", file_type="code") - G.add_edge("py_model", "pdf_paper", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/model.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_model": 0, "pdf_paper": 1, "py_a": 0, "py_b": 1} - score_cross, _ = _surprise_score(G, "py_model", "pdf_paper", - G.edges["py_model", "pdf_paper"], nc, - "src/model.py", "papers/vaswani.pdf") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_cross > score_same - - -# --- JSON key node filtering tests --- - -def test_is_json_key_node_noise_label(): - G = nx.Graph() - G.add_node("j1", label="name", source_file="schema.json") - assert _is_json_key_node(G, "j1") is True - - -def test_is_json_key_node_non_json_file(): - G = nx.Graph() - G.add_node("n1", label="name", source_file="model.py") - assert _is_json_key_node(G, "n1") is False - - -# --- npm dep-block key god-node filtering tests --- - -@pytest.mark.parametrize("dep_key", [ - "dependencies", - "devDependencies", - "peerDependencies", - "optionalDependencies", - "bundledDependencies", -]) -def test_god_nodes_excludes_npm_dep_block_keys(dep_key: str) -> None: - """npm package.json dep-block keys must be filtered from god_nodes output. - - Constructs a small graph with one node labelled with an npm dep-block key - (sourced from a .json file) and one real-domain node that has high degree. - Asserts that god_nodes() excludes the dep-block node even when it has the - highest degree, while the real-domain node is included. - - Args: - dep_key: The npm dependency-block key label to test (parametrized). - """ - G = nx.Graph() - # Real-domain node with a realistic source file. - G.add_node( - "real_node", - label="AuthService", - source_file="src/auth.py", - file_type="code", - source_location="L1", +def test_analysis_runs_on_native_snapshot(tmp_path): + graph = triangle(tmp_path).graph + communities = {0: ["a", "b"], 1: ["c"]} + assert god_nodes(graph) + assert surprising_connections(graph, communities) + assert suggest_questions(graph, communities, {0: "Core", 1: "Leaf"}) + + +def test_god_nodes_are_ranked_and_structured(): + result = god_nodes(_graph(), top_n=3) + assert result[0]["id"] == "hub" + assert result[0]["degree"] == 3 + assert {"id", "label", "degree"} <= set(result[0]) + + +def test_surprises_exclude_concepts_and_keep_cross_file_edges(): + graph = graph_from_payload( + [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "concept", "label": "Concept", "file_type": "document", "source_file": ""}, + ], + [ + {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, + {"source": "a", "target": "concept", "relation": "references", "confidence": "INFERRED"}, + ], ) - # npm dep-block key node — sourced from a JSON file so _is_json_key_node fires. - G.add_node( - "dep_node", - label=dep_key, - source_file="frontend/package.json", - file_type="code", - source_location="L1", + result = surprising_connections(graph, {0: ["a", "concept"], 1: ["b"]}) + labels = {item["source"] for item in result} | {item["target"] for item in result} + assert result and "Concept" not in labels + assert result[0]["source_files"][0] != result[0]["source_files"][1] + assert result[0]["why"] + + +def test_surprise_scoring_retains_confidence_and_language_rules(): + graph = graph_from_payload( + [ + {"id": "py", "label": "Auth", "source_file": "api/auth.py", "file_type": "code"}, + {"id": "ts", "label": "Member", "source_file": "web/types.ts", "file_type": "code"}, + {"id": "doc", "label": "Auth", "source_file": "docs/auth.md", "file_type": "document"}, + ], + [ + {"source": "py", "target": "ts", "relation": "calls", "confidence": "INFERRED"}, + {"source": "py", "target": "doc", "relation": "semantically_similar_to", "confidence": "INFERRED"}, + ], ) - # Wire up enough edges so dep_node has high degree — it would be a god-node - # without the filter. - for i in range(20): - peer = f"pkg_{i}" - G.add_node( - peer, - label=f"package-{i}", - source_file="frontend/package.json", - file_type="code", - source_location=f"L{i + 2}", - ) - G.add_edge( - "dep_node", - peer, - relation="contains", - confidence="EXTRACTED", - source_file="frontend/package.json", - weight=1.0, - ) - # Give real_node a couple of edges too. - G.add_edge( - "real_node", - "dep_node", - relation="imports", - confidence="EXTRACTED", - source_file="src/auth.py", - weight=1.0, + communities = {"py": 0, "ts": 1, "doc": 1} + cross, _ = _surprise_score( + graph, "py", "ts", first_edge_attributes(graph, "py", "ts"), communities, + "api/auth.py", "web/types.ts", ) - - result = god_nodes(G, top_n=10) - result_ids = [r["id"] for r in result] - - assert "dep_node" not in result_ids, ( - f"god_nodes() should filter npm dep-block key '{dep_key}' " - f"but it appeared in the result: {result}" + semantic, _ = _surprise_score( + graph, "py", "doc", first_edge_attributes(graph, "py", "doc"), communities, + "api/auth.py", "docs/auth.md", ) - assert "real_node" in result_ids, ( - f"god_nodes() should include real-domain node 'AuthService' " - f"but it was absent: {result}" + assert semantic > cross + + +def test_node_noise_helpers_use_native_attributes(): + graph = graph_from_payload([ + {"id": "concept", "source_file": ""}, + {"id": "json", "label": "dependencies", "source_file": "package.json"}, + {"id": "real", "label": "Auth", "source_file": "auth.py"}, + ]) + assert _is_concept_node(graph, "concept") + assert _is_json_key_node(graph, "json") + assert not _is_json_key_node(graph, "real") + assert _file_category("paper.pdf") == "paper" + + +def test_graph_diff_native_nodes_and_edges(): + old = graph_from_payload([ + {"id": "a", "label": "A"}, {"id": "b", "label": "B"}, + ], [{"source": "a", "target": "b", "relation": "calls"}]) + new = graph_from_payload([ + {"id": "a", "label": "A"}, {"id": "c", "label": "C"}, + ], [{"source": "a", "target": "c", "relation": "uses"}]) + result = graph_diff(old, new) + assert [row["id"] for row in result["new_nodes"]] == ["c"] + assert [row["id"] for row in result["removed_nodes"]] == ["b"] + assert result["new_edges"][0]["relation"] == "uses" + assert result["removed_edges"][0]["relation"] == "calls" + + +def test_import_cycles_support_self_loops_and_length_limit(): + graph = graph_from_payload( + [ + {"id": "a", "label": "A", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}, + {"id": "c", "label": "C", "source_file": "c.py"}, + ], + [ + {"source": "a", "target": "b", "relation": "imports_from", "source_file": "a.py"}, + {"source": "b", "target": "a", "relation": "imports_from", "source_file": "b.py"}, + {"source": "c", "target": "c", "relation": "imports_from", "source_file": "c.py"}, + ], + kind="digraph", ) - - -def test_is_json_key_node_real_label(): - G = nx.Graph() - G.add_node("j2", label="UserProfile", source_file="schema.json") - assert _is_json_key_node(G, "j2") is False - - -def test_god_nodes_excludes_json_noise(): - """god_nodes must not return generic JSON key nodes like 'name' or 'id'.""" - G = nx.Graph() - # Add many edges to a real node - G.add_node("real", label="AuthService", source_file="src/auth.py") - # Add a noisy JSON key node with high degree - G.add_node("json_name", label="name", source_file="schema.json") - for i in range(8): - n = f"peer{i}" - G.add_node(n, label=f"Peer{i}", source_file=f"src/peer{i}.py") - G.add_edge("json_name", n) - G.add_edge("real", n) - result = god_nodes(G, top_n=10) - labels = [r["label"] for r in result] - assert "name" not in labels - assert "AuthService" in labels - - -def test_god_nodes_filter_is_case_insensitive(): - """JSON-key filter must match regardless of label casing.""" - G = nx.Graph() - G.add_node("real", label="RealAbstraction", source_file="libs/real.py") - for i in range(3): - G.add_node(f"peer{i}", label=f"P{i}", source_file=f"src/p{i}.py") - G.add_edge("real", f"peer{i}") - for variant in ("Start", "START", "Name", "ID"): - nid = f"json_{variant.lower()}" - G.add_node(nid, label=variant, source_file="testhelpers/data.json") - for i in range(15): - t = f"{nid}_t{i}" - G.add_node(t, label=f"X{i}", source_file="testhelpers/data.json") - G.add_edge(t, nid) - result = god_nodes(G, top_n=10) - labels = [r["label"] for r in result] - for variant in ("Start", "START", "Name", "ID"): - assert variant not in labels, f"`{variant}` should be filtered as JSON-key noise" - - -def test_suggest_questions_excludes_rationale_nodes_from_isolated_count(): - G = nx.Graph() - G.add_node("service", label="Service", file_type="code", source_file="service.py") - G.add_node("reason", label="Explains service", file_type="rationale", source_file="service.py") - - questions = suggest_questions(G, communities={}, community_labels={}, top_n=10) - isolated = next(question for question in questions if question["type"] == "isolated_nodes") - - assert isolated["why"].startswith("1 weakly-connected node") - assert "`Service`" in isolated["question"] - assert "Explains service" not in isolated["question"] - - -# ── find_import_cycles tests ────────────────────────────────────────────────── - - -def _make_file_node(path: str) -> tuple[str, dict]: - """Create a graph node resembling real graphify schema.""" - nid = _make_id(path) - return nid, {"label": Path(path).name, "source_file": path, "file_type": "code"} - - -def _make_cycle_graph_directed() -> nx.DiGraph: - G = nx.DiGraph() - - a_id, a = _make_file_node("src/a.ts") - b_id, b = _make_file_node("src/b.ts") - c_id, c = _make_file_node("src/c.ts") - d_id, d = _make_file_node("src/d.ts") - ext_id = _make_id("react") - - G.add_node(a_id, **a) - G.add_node(b_id, **b) - G.add_node(c_id, **c) - G.add_node(d_id, **d) - # External-like node (no source_file): must be skipped safely. - G.add_node(ext_id, label="react", file_type="code") - - # 2-cycle: a <-> b - G.add_edge(a_id, b_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - G.add_edge(b_id, a_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - - # 3-cycle: b -> c -> d -> b - G.add_edge(b_id, c_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - G.add_edge(c_id, d_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - G.add_edge(d_id, b_id, relation="imports_from", source_file="src/d.ts", confidence="EXTRACTED") - - # Self-loop: c imports itself - G.add_edge(c_id, c_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - - # Mixed edge types: must not bleed into cycle graph - G.add_edge(a_id, ext_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(a_id, ext_id, relation="contains", source_file="src/a.ts", confidence="EXTRACTED") - - # Edge whose target has no source_file: must be skipped, no garbage label fallback - G.add_edge(a_id, ext_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - - return G - - -def test_find_import_cycles_returns_structured_records(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - assert isinstance(cycles, list) - assert cycles - assert isinstance(cycles[0], dict) - assert "cycle" in cycles[0] - assert "length" in cycles[0] - assert "why" in cycles[0] - - -def test_find_import_cycles_detects_2_and_3_cycles(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - cycle_sets = [set(c["cycle"]) for c in cycles] - assert any({"src/a.ts", "src/b.ts"}.issubset(s) for s in cycle_sets) - assert any({"src/b.ts", "src/c.ts", "src/d.ts"}.issubset(s) for s in cycle_sets) - - -def test_find_import_cycles_includes_self_loop_cycle(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - assert any(c["cycle"] == ["src/c.ts"] and c["length"] == 1 for c in cycles) - - -def test_find_import_cycles_respects_max_cycle_length(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G, max_cycle_length=2) - assert all(c["length"] <= 2 for c in cycles) - - -def test_find_import_cycles_skips_nodes_without_source_file(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - flat = " ".join(" ".join(c["cycle"]) for c in cycles) - assert "react" not in flat - - -def test_find_import_cycles_handles_undirected_graph_input(): - Gd = _make_cycle_graph_directed() - Gu = nx.Graph() - Gu.add_nodes_from(Gd.nodes(data=True)) - Gu.add_edges_from(Gd.edges(data=True)) - cycles = find_import_cycles(Gu) - assert cycles # should still resolve orientation via edge.source_file - - -def test_find_import_cycles_ignores_non_import_relations(): - G = nx.DiGraph() - a_id, a = _make_file_node("src/a.ts") - b_id, b = _make_file_node("src/b.ts") - G.add_node(a_id, **a) - G.add_node(b_id, **b) - # Bidirectional non-import edges should not be considered a dependency cycle. - G.add_edge(a_id, b_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(b_id, a_id, relation="contains", source_file="src/b.ts", confidence="EXTRACTED") - assert find_import_cycles(G) == [] - - -def test_find_import_cycles_empty_graph(): - assert find_import_cycles(nx.DiGraph()) == [] - - -def test_find_import_cycles_no_cycles(): - G = nx.DiGraph() - x_id, x = _make_file_node("x.ts") - y_id, y = _make_file_node("y.ts") - G.add_node(x_id, **x) - G.add_node(y_id, **y) - G.add_edge(x_id, y_id, relation="imports_from", source_file="x.ts", confidence="EXTRACTED") - assert find_import_cycles(G) == [] + cycles = find_import_cycles(graph, max_cycle_length=2) + assert {tuple(item["cycle"]) for item in cycles} + assert any(len(item["cycle"]) == 1 for item in cycles) diff --git a/tests/test_atomic_writes.py b/tests/test_atomic_writes.py index 968cb0b96..f8971eb53 100644 --- a/tests/test_atomic_writes.py +++ b/tests/test_atomic_writes.py @@ -1,4 +1,4 @@ -"""Tests for atomic JSON writes (graph.json / manifest.json). +"""Tests for atomic writes used by presentation artifacts and manifests. A crash, kill, or disk-full mid-write must not leave a truncated/corrupt file that a later load chokes on. `write_text_atomic` writes a temp file in the same @@ -13,15 +13,15 @@ def test_write_text_atomic_writes_and_leaves_no_tmp(tmp_path): - p = tmp_path / "out" / "graph.json" # parent doesn't exist yet + p = tmp_path / "out" / "artifact.json" # parent doesn't exist yet write_text_atomic(p, '{"a": 1}') assert json.loads(p.read_text()) == {"a": 1} # No leftover temp file in the target directory. - assert [x.name for x in p.parent.iterdir()] == ["graph.json"] + assert [x.name for x in p.parent.iterdir()] == ["artifact.json"] def test_write_text_atomic_preserves_existing_on_failure(tmp_path, monkeypatch): - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("original", encoding="utf-8") def boom(src, dst): @@ -33,12 +33,12 @@ def boom(src, dst): # The original file is intact and the temp file was cleaned up. assert p.read_text() == "original" - assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + assert sorted(x.name for x in tmp_path.iterdir()) == ["artifact.json"] def test_write_text_atomic_preserves_existing_mode(tmp_path): # An atomic replace must not tighten a 0644 file to mkstemp's 0600 default. - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("{}", encoding="utf-8") os.chmod(p, 0o644) write_text_atomic(p, '{"x": 1}') @@ -47,7 +47,7 @@ def test_write_text_atomic_preserves_existing_mode(tmp_path): def test_write_text_atomic_new_file_respects_umask(tmp_path): # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's - # 0600 — otherwise every fresh graph.json would be owner-only. + # 0600 — otherwise every fresh presentation artifact would be owner-only. p = tmp_path / "new.json" write_text_atomic(p, "{}") umask = os.umask(0) @@ -56,7 +56,7 @@ def test_write_text_atomic_new_file_respects_umask(tmp_path): def test_write_text_atomic_writes_through_symlink(tmp_path): - # Shared-output setups symlink graph.json to shared storage; the atomic write + # Shared-output setups may symlink an artifact to shared storage; the atomic write # must update the target and keep the link, not replace it with a real file. target = tmp_path / "real.json" target.write_text("old", encoding="utf-8") @@ -76,20 +76,6 @@ def test_write_json_atomic_roundtrip(tmp_path): assert not any(name.name.endswith(".tmp") for name in tmp_path.iterdir()) -def test_to_json_writes_atomically_no_tmp_leftover(tmp_path): - import networkx as nx - from graphify.export import to_json - - G = nx.Graph() - G.add_node("a", label="a", file_type="code") - G.add_node("b", label="b", file_type="code") - G.add_edge("a", "b") - out = tmp_path / "graph.json" - assert to_json(G, {}, str(out), force=True) is True - json.loads(out.read_text()) # valid JSON - assert not any(x.name.endswith(".tmp") for x in tmp_path.iterdir()) - - def test_save_manifest_writes_atomically(tmp_path): from graphify.detect import save_manifest @@ -105,7 +91,7 @@ def test_write_text_atomic_windows_permission_fallback(tmp_path, monkeypatch): """On Windows os.replace raises PermissionError when the destination is briefly locked (antivirus, an open reader); the copy-then-delete fallback must still land the new content and leave no temp file.""" - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("original", encoding="utf-8") real_replace = os.replace @@ -120,7 +106,7 @@ def flaky_replace(src, dst): assert calls["n"] == 1 # the fallback path was actually exercised assert p.read_text() == "new-content" - assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + assert sorted(x.name for x in tmp_path.iterdir()) == ["artifact.json"] def test_write_json_atomic_ensure_ascii_false_preserves_utf8(tmp_path): diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index b5751adcc..4d90cf858 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,183 +1,45 @@ -"""Tests for graphify/benchmark.py.""" -from __future__ import annotations -import json -import pytest -import networkx as nx -from networkx.readwrite import json_graph - -from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS, _safe, _hr - - -def _make_graph() -> nx.Graph: - G = nx.Graph() - G.add_node("n1", label="authentication", source_file="auth.py", source_location="L1", community=0) - G.add_node("n2", label="api_handler", source_file="api.py", source_location="L5", community=0) - G.add_node("n3", label="main_entry", source_file="main.py", source_location="L1", community=1) - G.add_node("n4", label="error_handler", source_file="errors.py", source_location="L1", community=1) - G.add_node("n5", label="database_layer", source_file="db.py", source_location="L1", community=2) - G.add_edge("n1", "n2", relation="calls", confidence="INFERRED") - G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED") - G.add_edge("n3", "n4", relation="uses", confidence="EXTRACTED") - G.add_edge("n5", "n2", relation="provides", confidence="EXTRACTED") - return G - - -def _write_graph(G: nx.Graph, path) -> None: - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data)) - - -# --- _query_subgraph_tokens --- - -def test_query_returns_positive_for_matching_question(): - G = _make_graph() - tokens = _query_subgraph_tokens(G, "how does authentication work") - assert tokens > 0 - -def test_query_returns_zero_for_no_match(): - G = _make_graph() - tokens = _query_subgraph_tokens(G, "xyzzy plugh zorkmid") - assert tokens == 0 - -def test_query_bfs_expands_neighbors(): - G = _make_graph() - # "authentication" matches n1, BFS depth=3 should reach n2, n3, n4 - tokens_deep = _query_subgraph_tokens(G, "authentication", depth=3) - tokens_shallow = _query_subgraph_tokens(G, "authentication", depth=1) - assert tokens_deep >= tokens_shallow - - -def test_query_keeps_short_non_english_terms(): - G = nx.Graph() - G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0) - tokens = _query_subgraph_tokens(G, "前端", depth=1) - assert tokens > 0 - - -# --- run_benchmark --- - -def test_run_benchmark_returns_reduction(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=10_000) - assert "reduction_ratio" in result - assert result["reduction_ratio"] > 1.0 - -def test_run_benchmark_corpus_tokens_proportional(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - r1 = run_benchmark(str(graph_file), corpus_words=1_000) - r2 = run_benchmark(str(graph_file), corpus_words=10_000) - # corpus_tokens scales linearly with corpus_words (within integer-division rounding) - assert abs(r2["corpus_tokens"] - r1["corpus_tokens"] * 10) <= r1["corpus_tokens"] - -def test_run_benchmark_per_question_list(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000, - questions=["how does authentication work", "what is the main entry"]) - assert len(result["per_question"]) >= 1 - for p in result["per_question"]: - assert "question" in p - assert "query_tokens" in p - assert "reduction" in p - -def test_run_benchmark_estimates_corpus_if_no_words(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=None) - assert result["corpus_words"] > 0 - -def test_run_benchmark_error_on_empty_graph(tmp_path): - G = nx.Graph() - graph_file = tmp_path / "empty.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=1_000) - assert "error" in result - -def test_run_benchmark_includes_node_edge_counts(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) - assert result["nodes"] == G.number_of_nodes() - assert result["edges"] == G.number_of_edges() - - -# --- print_benchmark --- - -def test_print_benchmark_no_crash(tmp_path, capsys): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) +from graphify.benchmark import _estimate_tokens, _query_subgraph_tokens, print_benchmark, run_benchmark +from tests.native_helpers import make_loaded + + +def _loaded(tmp_path): + return make_loaded( + tmp_path, + nodes=[ + {"id": "auth", "label": "Authentication", "source_file": "auth.py"}, + {"id": "token", "label": "Token Validator", "source_file": "token.py"}, + {"id": "main", "label": "Main Entry", "source_file": "main.py"}, + ], + edges=[ + {"source": "auth", "target": "token", "relation": "calls"}, + {"source": "main", "target": "auth", "relation": "uses"}, + ], + ) + + +def test_token_estimate_and_native_query(tmp_path): + loaded = _loaded(tmp_path) + assert _estimate_tokens("abcdefgh") == 2 + assert _query_subgraph_tokens(loaded.graph, "authentication", depth=2) > 0 + assert _query_subgraph_tokens(loaded.graph, "xyzzy plugh") == 0 + + +def test_run_benchmark_uses_native_store(tmp_path): + loaded = _loaded(tmp_path) + result = run_benchmark( + str(loaded.store_path), + corpus_words=10_000, + questions=["authentication", "main entry"], + ) + assert result["nodes"] == 3 and result["edges"] == 2 + assert result["reduction_ratio"] > 1 + assert len(result["per_question"]) == 2 + + +def test_print_benchmark_handles_success_and_error(tmp_path, capsys): + result = run_benchmark(str(_loaded(tmp_path).store_path), corpus_words=1000, + questions=["authentication"]) print_benchmark(result) - out = capsys.readouterr().out - assert "reduction" in out.lower() - assert "x" in out - -def test_print_benchmark_error_message(capsys): - print_benchmark({"error": "test error message"}) - out = capsys.readouterr().out - assert "test error message" in out - - -# --- cp1252 / Windows-console encoding compatibility (regression for #?) --- -# print_benchmark previously crashed on Windows consoles (cp1252) because it -# unconditionally printed U+2500 and U+2192. _safe() falls back to ASCII when -# stdout cannot encode the glyph. - -def test_safe_returns_unicode_when_encodable(): - import io, sys - real_stdout = sys.stdout - try: - sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") - assert _safe("→", "->") == "→" - assert _hr(5) == "─" * 5 - finally: - sys.stdout = real_stdout - -def test_safe_falls_back_when_unencodable(): - import io, sys - real_stdout = sys.stdout - try: - sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252") - assert _safe("→", "->") == "->" - assert _hr(5) == "-" * 5 - finally: - sys.stdout = real_stdout - -def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): - """Regression: U+2500 / U+2192 used to crash with UnicodeEncodeError on cp1252.""" - import io, sys - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) - - # Replace stdout with a strict cp1252 stream — same behaviour as the - # legacy Windows console that surfaced this bug. - cp1252_stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors="strict") - monkeypatch.setattr(sys, "stdout", cp1252_stdout) - print_benchmark(result) # must not raise UnicodeEncodeError - cp1252_stdout.flush() - written = cp1252_stdout.buffer.getvalue().decode("cp1252") - assert "reduction" in written.lower() - # ASCII fallbacks must be present, fancy glyphs must not. - assert "─" not in written - assert "→" not in written - - -def test_run_benchmark_rejects_oversized_graph(monkeypatch, tmp_path): - """#F4: run_benchmark must refuse to read a graph.json that exceeds - the size cap before parsing it into memory.""" - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - run_benchmark(str(graph_file)) + assert "Reduction" in capsys.readouterr().out + print_benchmark({"error": "empty"}) + assert "Benchmark error" in capsys.readouterr().out diff --git a/tests/test_build.py b/tests/test_build.py index 3d8582c35..2c2a56a72 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,1063 +1,253 @@ -import json -from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges, dedupe_nodes +"""Transient DTO construction and native incremental merge tests.""" -FIXTURES = Path(__file__).parent / "fixtures" +from graphify.build import ( + build, + build_from_extraction, + build_merge, + build_unclustered_extraction, + dedupe_edges, + dedupe_nodes, +) -def test_dedupe_edges_collapses_exact_parallels(): - # #1317: --no-cluster / incremental update concatenate edge lists raw. - edges = [ - {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, - {"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup - {"source": "a", "target": "b", "relation": "imports"}, # different relation: kept - {"source": "b", "target": "c", "relation": "calls"}, - ] - out = dedupe_edges(edges) - keys = [(e["source"], e["target"], e["relation"]) for e in out] - assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")] - # first occurrence wins (keeps L1, not L9) - assert out[0]["source_location"] == "L1" +def _attrs(graph, node_id): + return next(node.attributes for node in graph.nodes if node.id == node_id) -def test_dedupe_edges_is_idempotent(): - edges = [ - {"source": "a", "target": "b", "relation": "calls"}, - {"source": "a", "target": "b", "relation": "calls"}, - ] - once = dedupe_edges(edges) - twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating - assert len(once) == 1 - assert len(twice) == 1 +def _edge(graph, source, target, relation=None): + return next( + edge for edge in graph.edges + if edge.source == source + and edge.target == target + and (relation is None or edge.attributes.get("relation") == relation) + ) -def test_dedupe_nodes_collapses_by_id_last_wins(): - # #1327: a shared module anchor is emitted once per importing file; the - # --no-cluster raw writer must collapse same-id node dicts (#1317). - nodes = [ - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "A.swift"}, - {"id": "akit", "label": "AKit", "file_type": "code"}, - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "B.swift"}, +def test_dedupe_records_are_deterministic(): + nodes = [{"id": "a", "label": "old"}, {"id": "b"}, {"id": "a", "label": "new"}] + assert dedupe_nodes(nodes) == [{"id": "a", "label": "new"}, {"id": "b"}] + edges = [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "calls", "source_location": "L2"}, + {"source": "a", "target": "b", "relation": "imports"}, ] - out = dedupe_nodes(nodes) - ids = [n["id"] for n in out] - assert ids == ["foundation", "akit"] # first-appearance order - # last writer wins on attributes - assert next(n for n in out if n["id"] == "foundation")["source_file"] == "B.swift" + assert dedupe_edges(edges) == [edges[0], edges[2]] -def load_extraction(): - return json.loads((FIXTURES / "extraction.json").read_text()) -def test_build_from_json_node_count(): - G = build_from_json(load_extraction()) - assert G.number_of_nodes() == 4 - -def test_build_from_json_edge_count(): - G = build_from_json(load_extraction()) - assert G.number_of_edges() == 4 - -def test_null_weight_edge_builds_and_clusters(tmp_path): - """#1960: an explicit ``"weight": null`` (JSON null -> None) used to survive - ``.get("weight", 1.0)`` and crash Louvain/Leiden modularity with a TypeError. - It must now coerce to the 1.0 default, build, and cluster without raising.""" - from graphify.cluster import cluster - extraction = { +def test_build_normalizes_attributes_weights_and_direction(): + graph = build_from_extraction({ "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, - {"id": "c", "label": "C", "file_type": "code", "source_file": "c.py"}, + {"id": "a", "label": "A", "source": "src\\a.py", "file_type": None, "_origin": "ast"}, + {"id": "b", "label": "B", "source_file": "src/b.py", "file_type": "code", "_origin": "ast"}, ], "edges": [ - {"source": "a", "target": "b", "relation": "references", "weight": None, + {"from": "a", "to": "b", "relation": "calls", "weight": None, "confidence_score": None}, - {"source": "b", "target": "c", "relation": "references", "weight": 2.5}, - ], - } - G = build_from_json(extraction) - assert G["a"]["b"]["weight"] == 1.0 # null coerced to default - assert G["a"]["b"]["confidence_score"] == 1.0 # null confidence_score too - assert G["b"]["c"]["weight"] == 2.5 # a valid weight is preserved - cluster(G) # must not raise (Louvain/Leiden modularity) - - -def test_malformed_weights_normalize(): - """Non-numeric / NaN / inf / negative weights fall back to 1.0 (the backends - reject them); a missing weight key is left absent (backends default it).""" - extraction = { - "nodes": [{"id": f"n{i}", "label": str(i), "file_type": "code", - "source_file": f"{i}.py"} for i in range(4)], - "edges": [ - {"source": "n0", "target": "n1", "relation": "references", "weight": "3.5"}, - {"source": "n1", "target": "n2", "relation": "references", "weight": float("nan")}, - {"source": "n2", "target": "n3", "relation": "references", "weight": -4}, ], - } - G = build_from_json(extraction) - assert G["n0"]["n1"]["weight"] == 3.5 # numeric string coerces - assert G["n1"]["n2"]["weight"] == 1.0 # NaN -> default - assert G["n2"]["n3"]["weight"] == 1.0 # negative -> default - - -def test_nodes_have_label(): - G = build_from_json(load_extraction()) - assert G.nodes["n_transformer"]["label"] == "Transformer" - -def test_edges_have_confidence(): - G = build_from_json(load_extraction()) - data = G.edges["n_attention", "n_concept_attn"] - assert data["confidence"] == "INFERRED" - -def test_ambiguous_edge_preserved(): - G = build_from_json(load_extraction()) - data = G.edges["n_layernorm", "n_concept_attn"] - assert data["confidence"] == "AMBIGUOUS" - -def test_legacy_node_source_canonicalized(): - """Legacy 'source' key on nodes is renamed to 'source_file' before graph build.""" - ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source": "a.py"}], - "edges": [], "input_tokens": 0, "output_tokens": 0} - G = build_from_json(ext) - assert "source_file" in G.nodes["n1"] - assert G.nodes["n1"]["source_file"] == "a.py" - assert "source" not in G.nodes["n1"] - - -def test_legacy_edge_from_to_canonicalized(): - """Legacy 'from'/'to' keys on edges are accepted alongside 'source'/'target'.""" - ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], - "edges": [{"from": "n1", "to": "n2", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "a.py", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0} - G = build_from_json(ext) - assert G.number_of_edges() == 1 - - -def test_source_file_backslash_normalized(): - """Windows backslash paths and POSIX paths for the same file must produce one node.""" - extraction = { - "nodes": [ - {"id": "n1", "label": "A", "file_type": "code", "source_file": "src\\middleware\\auth.py"}, - {"id": "n2", "label": "B", "file_type": "code", "source_file": "src/middleware/auth.py"}, - ], - "edges": [], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(extraction) - sources = {G.nodes[n]["source_file"] for n in G.nodes()} - assert sources == {"src/middleware/auth.py"} - - -def test_edge_missing_source_file_backfilled_from_node(): - """#1279: a semantic/LLM edge lacking source_file must inherit it from its - source node rather than reach graph.json with no file reference.""" - extraction = { - "nodes": [ - {"id": "n1", "label": "A", "file_type": "concept", "source_file": "docs/a.md"}, - {"id": "n2", "label": "B", "file_type": "concept", "source_file": "docs/b.md"}, - ], - # No source_file on the edge (as LLM output sometimes omits it). - "edges": [{"source": "n1", "target": "n2", "relation": "relates_to", "confidence": "INFERRED"}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(extraction) - sf = edge_data(G, "n1", "n2").get("source_file") - assert sf == "docs/a.md" # backfilled from the source node - - -def test_build_merges_multiple_extractions(): - ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}], - "edges": [], "input_tokens": 0, "output_tokens": 0} - ext2 = {"nodes": [{"id": "n2", "label": "B", "file_type": "document", "source_file": "b.md"}], - "edges": [{"source": "n1", "target": "n2", "relation": "references", - "confidence": "INFERRED", "source_file": "b.md", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0} - G = build([ext1, ext2]) - assert G.number_of_nodes() == 2 - assert G.number_of_edges() == 1 - - -def test_none_file_type_defaults_to_concept(capsys): - """Legacy nodes with file_type=None (e.g. preserved from older graph.json - by `_rebuild_code`) must not trigger 'invalid file_type None' warnings (#660).""" - ext = { - "nodes": [ - {"id": "n1", "label": "Stub", "file_type": None, "source_file": "a.py"}, - {"id": "n2", "label": "Real", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - err = capsys.readouterr().err - assert "invalid file_type" not in err - # The legacy node still exists in the graph and has been canonicalized - assert G.nodes["n1"]["file_type"] == "concept" - assert G.nodes["n2"]["file_type"] == "code" - - -def test_missing_file_type_defaults_to_concept(capsys): - """Nodes missing file_type entirely should also be canonicalized to 'concept'.""" - ext = { - "nodes": [ - {"id": "n1", "label": "Bare", "source_file": "a.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - err = capsys.readouterr().err - assert "invalid file_type" not in err - assert "missing required field 'file_type'" not in err - assert G.nodes["n1"]["file_type"] == "concept" - - -def test_real_invalid_file_type_coerced_to_concept(): - """Unknown file_type values are coerced through the synonym mapper, falling - back to 'concept' for anything that isn't a known LLM synonym (#840).""" - ext = { - "nodes": [ - {"id": "n1", "label": "Bad", "file_type": "weird_type", "source_file": "a.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - assert G.nodes["n1"]["file_type"] == "concept" - - -def test_file_type_synonym_mapping(): - """Known invalid file_type values map to their canonical equivalents.""" - ext = { - "nodes": [ - {"id": "n1", "label": "MD", "file_type": "markdown", "source_file": "a.md"}, - {"id": "n2", "label": "Tool", "file_type": "tool", "source_file": "b.py"}, - {"id": "n3", "label": "Pat", "file_type": "pattern", "source_file": "c.md"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - assert G.nodes["n1"]["file_type"] == "document" - assert G.nodes["n2"]["file_type"] == "code" - assert G.nodes["n3"]["file_type"] == "concept" - - -def test_ghost_merge_unique_located_node_still_merges(): - """#1145 ghost-merge: a semantic ghost collapses into the single AST node - sharing its (basename, label), and edges re-point to the AST node.""" - ext = { - "nodes": [ - {"id": "ast_render", "label": "render", "file_type": "code", - "source_file": "src/app/index.ts", "source_location": "L10", "_origin": "ast"}, - {"id": "ghost_render", "label": "render", "file_type": "code", - "source_file": "src/app/index.ts"}, - {"id": "caller", "label": "main", "file_type": "code", - "source_file": "src/main.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "src/main.ts", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(ext) - assert "ghost_render" not in G.nodes() - assert G.has_edge("caller", "ast_render") + }, directed=True) + assert graph.kind == "digraph" + assert graph.node_count == 2 and graph.edge_count == 1 + assert _attrs(graph, "a")["source_file"] == "src/a.py" + assert _attrs(graph, "a")["file_type"] == "concept" + assert _edge(graph, "a", "b").attributes["weight"] == 1.0 + assert _edge(graph, "a", "b").attributes["confidence_score"] == 1.0 + + +def test_build_merges_last_node_attributes_and_backfills_edge_source(): + graph = build([ + {"nodes": [{"id": "a", "label": "Old", "source_file": "a.py"}], "edges": []}, + { + "nodes": [ + {"id": "a", "label": "New", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}, + ], + "edges": [{"source": "a", "target": "b", "relation": "references"}], + }, + ], dedup=False) + assert _attrs(graph, "a")["label"] == "New" + assert _edge(graph, "a", "b").attributes["source_file"] == "a.py" def test_ghost_merge_uses_source_file_not_basename(): - """#2068: the ghost-merge key is the full source_file, not the bare basename. - A ghost from src/a/index.ts merges into THAT file's AST node (a_render), never - the unrelated same-basename b_render in src/b/index.ts. (Pre-#2068 the - (basename, label) key made ('index.ts','render') ambiguous and skipped the - merge; the directory-aware key resolves it precisely.)""" - ext = { - "nodes": [ - {"id": "a_render", "label": "render", "file_type": "code", - "source_file": "src/a/index.ts", "source_location": "L10", "_origin": "ast"}, - {"id": "b_render", "label": "render", "file_type": "code", - "source_file": "src/b/index.ts", "source_location": "L20", "_origin": "ast"}, - {"id": "ghost_render", "label": "render", "file_type": "code", - "source_file": "src/a/index.ts"}, - {"id": "caller", "label": "main", "file_type": "code", - "source_file": "src/main.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "src/main.ts", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(ext) - # Ghost merges into its same-file twin; the edge re-points to a_render only. - assert "ghost_render" not in G.nodes() - assert G.has_edge("caller", "a_render") - assert not G.has_edge("caller", "b_render") - # The unrelated same-basename node in another directory is untouched. - assert "b_render" in G.nodes() - - -def test_ghost_merge_not_across_directories_same_basename(): - """#2068: two unrelated non-AST nodes with the same basename+label in - DIFFERENT directories must NOT be merged onto one survivor (the bug: bare - basename collapsed docs/product_a/index.md and docs/product_b/index.md).""" - ext = { - "nodes": [ - {"id": "docs_a_index", "label": "Quickstart", "file_type": "document", - "source_file": "docs/product_a/index.md", "source_location": "L1"}, - {"id": "docs_b_index", "label": "Quickstart", "file_type": "document", - "source_file": "docs/product_b/index.md"}, - {"id": "docs_hub", "label": "Docs", "file_type": "concept", - "source_file": "docs/hub.md", "source_location": "L1"}, - ], - "edges": [{"source": "docs_hub", "target": "docs_b_index", "relation": "links_to", - "confidence": "INFERRED", "source_file": "docs/hub.md"}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(ext, directed=False) - # Both docs survive; the edge stays on the file it was authored against. - assert "docs_a_index" in G.nodes() and "docs_b_index" in G.nodes() - assert G.has_edge("docs_hub", "docs_b_index") - assert not G.has_edge("docs_hub", "docs_a_index") - - -def test_ghost_merge_non_ast_different_files_both_survive(): - """#1753: two NON-AST (semantic) nodes sharing (basename, label) but from - DIFFERENT files are distinct concepts with no AST canonical twin. They must - not be merged into an arbitrary survivor (which flipped run-to-run with the - hash seed); both survive, mirroring the AST/AST guard (#1257).""" - ext = { - "nodes": [ - {"id": "dir_a_update_build_merge", "label": "build_merge() function", - "file_type": "concept", "source_file": "dir_a/update.md", "source_location": "L10"}, - {"id": "dir_b_update_build_merge", "label": "build_merge() function", - "file_type": "concept", "source_file": "dir_b/update.md", "source_location": "L12"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert sorted(G.nodes()) == ["dir_a_update_build_merge", "dir_b_update_build_merge"] - - -def test_ghost_merge_non_ast_same_file_still_merges(): - """A genuine duplicate — two non-AST nodes with the SAME source_file and - label — is a real ghost and still collapses to one node (deterministically), - so #1753's fix doesn't leave same-file LLM duplicates behind.""" - ext = { - "nodes": [ - {"id": "a_foo", "label": "Foo", "file_type": "concept", - "source_file": "x/doc.md", "source_location": "L1"}, - {"id": "b_foo", "label": "Foo", "file_type": "concept", - "source_file": "x/doc.md", "source_location": "L2"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert G.number_of_nodes() == 1 - - -def test_build_merge_preserves_call_edge_direction(tmp_path): - """Regression for #760. - - When the callee is defined before the caller in source, NetworkX's - undirected Graph stores edges in node-insertion order. Going through - node_link_graph() + edges() during build_merge previously flipped the - `calls` edge so that on the next save source/target were swapped. - - build_merge must read the saved JSON's source/target verbatim instead - of round-tripping through NetworkX. - """ - from graphify.extract import extract_js - from graphify.export import to_json - - # Callee `b` is defined before caller `a` so node insertion order - # is b, a. An undirected Graph then yields the edge as (b, a) on - # iteration, which is the wrong direction for `calls` (a calls b). - src = "function b() {}\nfunction a() { b(); }\n" - src_file = tmp_path / "x.js" - src_file.write_text(src) - - extraction = extract_js(src_file) - assert "error" not in extraction - - # Locate the `calls` edge in the raw extraction so we know the truth. - call_edges = [e for e in extraction["edges"] if e["relation"] == "calls"] - assert len(call_edges) == 1, "expected exactly one calls edge from the snippet" - truth_src = call_edges[0]["source"] - truth_tgt = call_edges[0]["target"] - - nodes_by_id = {n["id"]: n for n in extraction["nodes"]} - assert nodes_by_id[truth_src]["label"].startswith("a") - assert nodes_by_id[truth_tgt]["label"].startswith("b") - - # First build + save. - G1 = build([extraction], dedup=False) - graph_path = tmp_path / "graph.json" - communities: dict = {} - assert to_json(G1, communities, str(graph_path), force=True) - - # Verify direction is correct in the freshly written JSON. - saved = json.loads(graph_path.read_text()) - saved_calls = [e for e in saved.get("links", saved.get("edges", [])) - if e.get("relation") == "calls"] - assert len(saved_calls) == 1 - assert saved_calls[0]["source"] == truth_src - assert saved_calls[0]["target"] == truth_tgt - - # Now simulate `--update` with no new chunks — load + re-save. - G2 = build_merge([], graph_path, dedup=False) - assert to_json(G2, communities, str(graph_path), force=True) - - # The calls edge must still go a -> b, not b -> a. - reloaded = json.loads(graph_path.read_text()) - reloaded_calls = [e for e in reloaded.get("links", reloaded.get("edges", [])) - if e.get("relation") == "calls"] - assert len(reloaded_calls) == 1 - assert reloaded_calls[0]["source"] == truth_src, ( - f"calls edge source flipped after build_merge round-trip: " - f"expected {truth_src} (a), got {reloaded_calls[0]['source']}" + graph = build_from_extraction({ + "nodes": [ + { + "id": "a_render", + "label": "render", + "file_type": "code", + "source_file": "src/a/index.ts", + "source_location": "L10", + "_origin": "ast", + }, + { + "id": "b_render", + "label": "render", + "file_type": "code", + "source_file": "src/b/index.ts", + "source_location": "L20", + "_origin": "ast", + }, + { + "id": "ghost_render", + "label": "render", + "file_type": "code", + "source_file": "src/a/index.ts", + }, + { + "id": "caller", + "label": "main", + "file_type": "code", + "source_file": "src/main.ts", + "source_location": "L1", + "_origin": "ast", + }, + ], + "edges": [{ + "source": "caller", + "target": "ghost_render", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "src/main.ts", + }], + }) + + node_ids = {node.id for node in graph.nodes} + assert "ghost_render" not in node_ids + assert "b_render" in node_ids + assert _edge(graph, "caller", "a_render") + assert not any( + edge.source == "caller" and edge.target == "b_render" + for edge in graph.edges ) - assert reloaded_calls[0]["target"] == truth_tgt, ( - f"calls edge target flipped after build_merge round-trip: " - f"expected {truth_tgt} (b), got {reloaded_calls[0]['target']}" - ) - -def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_path): - """Regression for #1061. - When an extraction emits two `calls` edges between the same pair in - opposite directions (mutual recursion, callbacks, event handlers, etc.), - nx.Graph collapses them into a single undirected edge. The deterministic - edge sort introduced in #1010 ordered edges by (source, target, relation), - so the lexicographically-later direction always wrote second and clobbered - the first edge's _src/_tgt — the surviving edge then exported with caller - and callee systematically swapped on every collision. - - build_from_json must keep the first-seen direction for the surviving edge - instead of letting the second add_edge overwrite _src/_tgt. - """ - from graphify.export import to_json - - # Lexicographic order of (src, tgt, rel) puts `a` < `z` first, so the sort - # processes `a -> z` BEFORE `z -> a`. Without the fix, the second write - # overwrites _src/_tgt and the exported edge becomes z -> a. With the fix, - # the first-seen `a -> z` direction is preserved. - extraction = { - "nodes": [ - {"id": "a_handler", "label": "a", "file_type": "code", "source_file": "a.ts"}, - {"id": "z_emitter", "label": "z", "file_type": "code", "source_file": "z.ts"}, - ], - "edges": [ - {"source": "a_handler", "target": "z_emitter", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "a.ts"}, - {"source": "z_emitter", "target": "a_handler", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "z.ts"}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - # Only one undirected edge between the pair survives, but its stored - # direction must be the first-seen one (a_handler -> z_emitter), not the - # lexicographically-later one (z_emitter -> a_handler). - assert G.number_of_edges() == 1 - data = edge_data(G, "a_handler", "z_emitter") - assert data["_src"] == "a_handler" - assert data["_tgt"] == "z_emitter" - - graph_path = tmp_path / "graph.json" - assert to_json(G, {}, str(graph_path), force=True) - saved = json.loads(graph_path.read_text()) - saved_calls = [e for e in saved.get("links", saved.get("edges", [])) - if e.get("relation") == "calls"] - assert len(saved_calls) == 1 - assert saved_calls[0]["source"] == "a_handler", ( - f"calls edge source flipped on bidirectional collision: " - f"expected a_handler, got {saved_calls[0]['source']}" - ) - assert saved_calls[0]["target"] == "z_emitter", ( - f"calls edge target flipped on bidirectional collision: " - f"expected z_emitter, got {saved_calls[0]['target']}" +def test_ghost_merge_not_across_directories_same_basename(): + graph = build_from_extraction({ + "nodes": [ + { + "id": "docs_a_index", + "label": "Quickstart", + "file_type": "document", + "source_file": "docs/product_a/index.md", + "source_location": "L1", + }, + { + "id": "docs_b_index", + "label": "Quickstart", + "file_type": "document", + "source_file": "docs/product_b/index.md", + }, + { + "id": "docs_hub", + "label": "Docs", + "file_type": "concept", + "source_file": "docs/hub.md", + "source_location": "L1", + }, + ], + "edges": [{ + "source": "docs_hub", + "target": "docs_b_index", + "relation": "links_to", + "confidence": "INFERRED", + "source_file": "docs/hub.md", + }], + }) + + assert {node.id for node in graph.nodes} >= { + "docs_a_index", + "docs_b_index", + } + assert _edge(graph, "docs_hub", "docs_b_index") + assert not any( + edge.source == "docs_hub" and edge.target == "docs_a_index" + for edge in graph.edges ) -# Regression tests for #796 — edge_data / edge_datas helpers must tolerate -# MultiGraph and MultiDiGraph, which networkx's node_link_graph() produces -# whenever the loaded JSON has multigraph: true. Plain G.edges[u, v] crashes -# on those with `ValueError: not enough values to unpack (expected 3, got 2)`. - -def test_edge_data_simple_graph(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d["relation"] == "calls" - assert d["confidence"] == "EXTRACTED" - - -def test_edge_datas_simple_graph_returns_singleton_list(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 1 - assert ds[0]["relation"] == "calls" - - -def test_edge_data_multigraph_with_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - # First parallel edge wins; should be one of the two attribute dicts above. - assert d.get("relation") in ("calls", "references") - - -def test_edge_datas_multigraph_returns_all_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 2 - relations = {e.get("relation") for e in ds} - assert relations == {"calls", "references"} - - -def test_edge_data_multidigraph(): - G = nx.MultiDiGraph() - G.add_edge("a", "b", relation="calls") - G.add_edge("a", "b", relation="imports") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "imports") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - -def test_edge_data_node_link_multigraph_roundtrip(): - """A node_link JSON with multigraph: true must load as MultiGraph and the - helpers must operate on it without raising the 3-tuple unpack ValueError.""" - data = { - "directed": False, +def test_parallel_relations_and_self_loop_survive(): + graph = build_from_extraction({ + "directed": True, "multigraph": True, - "graph": {}, - "nodes": [ - {"id": "a", "label": "A"}, - {"id": "b", "label": "B"}, - ], + "nodes": [{"id": "a"}, {"id": "b"}], "links": [ - {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, - ], - } - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - assert isinstance(G, nx.MultiGraph) - # Plain G.edges[u, v] would raise here; the helper must not. - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "references") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - -def test_build_from_json_relativizes_absolute_source_file(tmp_path): - """Semantic subagents emit absolute source_file paths; build_from_json must - relativize them to root so MCP traversal works correctly (#932).""" - root = tmp_path / "myproject" - root.mkdir() - abs_path = str(root / "docs" / "overview.md") - extraction = { - "nodes": [ - {"id": "overview_intro", "label": "Intro", "source_file": abs_path, "file_type": "document"}, - ], - "edges": [ - {"source": "overview_intro", "target": "overview_intro", - "relation": "self", "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": abs_path}, - ], - } - G = build_from_json(extraction, root=root) - # The id-stem migration (#1504) re-keys the old short id to the full-path form. - sf = G.nodes["docs_overview_intro"]["source_file"] - assert not sf.startswith("/"), f"source_file still absolute: {sf}" - assert sf == "docs/overview.md" - - -def test_build_relativizes_absolute_source_file(tmp_path): - """build() passes root through to build_from_json (#932).""" - root = tmp_path / "proj" - root.mkdir() - abs_path = str(root / "src" / "main.py") - extraction = { - "nodes": [{"id": "main_fn", "label": "main", "source_file": abs_path, "file_type": "code"}], - "edges": [], - } - G = build([extraction], root=root) - # #1504 re-keys main_fn (old stem "main") to the full-path form "src_main". - sf = G.nodes["src_main_fn"]["source_file"] - assert sf == "src/main.py" - - -def test_build_from_json_ambiguous_old_stem_alias_stays_dangling(tmp_path): - """The #1504 old-stem alias (e.g. "ping.h" -> bare "ping") is meant to let a - stale-id edge from an un-re-extracted fragment still find its own file after - a rekey. But the old-stem form drops the extension and most of the path, so - two unrelated real files easily collapse onto the same bare alias (a C header - and a PHP script both named "ping", in different directories). A dangling - edge produced by an unrelated third file's own unscoped fallback id (e.g. the - C/C++ extractor's last-resort target for an #include it couldn't resolve to - a real path) must not silently ride that alias onto an arbitrary one of them - — it should stay dangling and get dropped, same as any other unresolvable - edge, rather than wire two unrelated files/languages together by accident.""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - # Ids given in their canonical (post-extract.py, extension-stripped) - # form, matching what a real graphify update run would already have - # produced before build_from_json assembles the final graph. - {"id": "dev_monitoring_ping", "label": "ping.h", "file_type": "code", - "source_file": "Dev/monitoring/ping.h"}, - {"id": "www_pages_api_ping", "label": "ping.php", "file_type": "code", - "source_file": "www/pages/api/ping.php"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, - ], - "edges": [ - # The unscoped, deliberately-unresolved fallback edge a C/C++ #include - # resolver leaves behind when it can't find the header on disk. - {"source": "dev_poker_server", "target": "ping", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert not G.has_edge("dev_poker_server", "dev_monitoring_ping") - assert not G.has_edge("dev_poker_server", "www_pages_api_ping") - - -def test_build_from_json_ambiguous_alias_detected_despite_header_impl_salting(tmp_path): - """A same-directory .h/.cpp pair collides on their shared pre-extension id - and gets salted apart into ids like "tools_aolserver_utility_h_..." — no - longer a clean new_stem prefix. The ambiguity check must still recognize - the salted header as a legitimate claimant for the bare old-stem alias (by - label, not id shape), so a real collision with an unrelated same-named PHP - file is still caught instead of the header silently dropping out of the - race and leaving the PHP file as the lone "unambiguous" winner (this - reproduced against the real depot: Tools/aolserver/utility.h and .cpp, - salted apart, let wwwapi.masque.com/pages/utility.php win the bare - "utility" alias uncontested).""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - {"id": "tools_aolserver_utility_h_tools_aolserver_utility", "label": "utility.h", - "file_type": "code", "source_file": "Tools/aolserver/utility.h"}, - {"id": "tools_aolserver_utility_cpp_tools_aolserver_utility", "label": "utility.cpp", - "file_type": "code", "source_file": "Tools/aolserver/utility.cpp"}, - {"id": "wwwapi_masque_com_pages_utility", "label": "utility.php", - "file_type": "code", "source_file": "wwwapi.masque.com/pages/utility.php"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, + {"source": "a", "target": "b", "key": "call", "relation": "calls"}, + {"source": "a", "target": "b", "key": "use", "relation": "uses"}, + {"source": "a", "target": "a", "key": "self", "relation": "recursive"}, ], - "edges": [ - {"source": "dev_poker_server", "target": "utility", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert not G.has_edge("dev_poker_server", "wwwapi_masque_com_pages_utility") - assert not G.has_edge("dev_poker_server", "tools_aolserver_utility_h_tools_aolserver_utility") + }) + assert graph.kind == "multidigraph" + assert graph.edge_count == 3 + assert {edge.key for edge in graph.edges} == {"call", "use", "self"} -def test_build_from_json_unambiguous_old_stem_alias_still_resolves(tmp_path): - """Companion to the ambiguous case above: when exactly one real file claims - an old-stem alias, a dangling edge to that bare alias should still resolve - to it — the #1504 migration-compat behavior this index exists for.""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - {"id": "dev_monitoring_utility", "label": "utility.h", "file_type": "code", - "source_file": "Dev/monitoring/utility.h"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, - ], +def test_unclustered_build_preserves_parallel_and_external_edges(): + graph = build_unclustered_extraction({ + "nodes": [{"id": "a"}, {"id": "b"}], "edges": [ - {"source": "dev_poker_server", "target": "utility", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert G.has_edge("dev_poker_server", "dev_monitoring_utility") - - -def test_build_from_json_relative_source_file_unchanged(tmp_path): - """Already-relative source_file paths must not be modified.""" - extraction = { - "nodes": [{"id": "foo_bar", "label": "bar", "source_file": "src/foo.py", "file_type": "code"}], - "edges": [], - } - G = build_from_json(extraction, root=tmp_path) - # source_file must be untouched; the id is re-keyed to the full-path form (#1504). - assert G.nodes["src_foo_bar"]["source_file"] == "src/foo.py" - - -def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): - """#1007: manifest stores absolute paths, graph nodes store relative paths. - prune_sources with absolute paths must still remove the right nodes and edges.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - # Simulate a graph with relative source_file paths (as built normally) - chunk = {"nodes": [ - {"id": "n1", "label": "login", "file_type": "code", "source_file": "module_a/auth.py"}, - {"id": "n2", "label": "format_date", "file_type": "code", "source_file": "module_b/utils.py"}, - ], "edges": [ - {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", - "source_file": "module_b/utils.py", "weight": 1.0}, - ]} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # prune_sources from manifest — absolute paths (what detect_incremental emits) - deleted_abs = [str(root / "module_b" / "utils.py")] - G1 = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) - - node_labels = {d["label"] for _, d in G1.nodes(data=True)} - assert "format_date" not in node_labels, "stale node from deleted file should be pruned" - assert "login" in node_labels, "unrelated node must survive" - # Edge from deleted file must also be gone - assert G1.number_of_edges() == 0, "edge from deleted source_file should be pruned" - - -def test_build_merge_prune_windows_backslash_paths(tmp_path): - """#1007: prune_sources with Windows-style backslash absolute paths must still match.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - chunk = {"nodes": [ - {"id": "n1", "label": "parse_date", "file_type": "code", "source_file": "module_b/utils.py"}, - ], "edges": []} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # Simulate Windows manifest path with backslashes - win_path = str(root / "module_b" / "utils.py").replace("/", "\\") - G1 = build_merge([], graph_path, prune_sources=[win_path], dedup=False, root=root) - - node_labels = {d["label"] for _, d in G1.nodes(data=True)} - assert "parse_date" not in node_labels, "node should be pruned even with backslash path" - - -def test_build_merge_replaces_changed_file_stale_edges(tmp_path): - """Re-extracting a CHANGED file must REPLACE its prior nodes/edges, not - accumulate them. build_merge previously only grew the graph, so an edge that - disappeared from a file's new version survived forever (only exact-duplicate - edges collapsed). The new-chunk source_file may be an absolute win32 path - while the stored graph keeps relative posix — both forms must match.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - # First build: changed.md contributed A, B and edge A->B; keep.md is unrelated. - chunk0 = {"nodes": [ - {"id": "A", "label": "A", "file_type": "document", "source_file": "changed.md"}, - {"id": "B", "label": "B", "file_type": "document", "source_file": "changed.md"}, - {"id": "K", "label": "K", "file_type": "document", "source_file": "keep.md"}, - ], "edges": [ - {"source": "A", "target": "B", "relation": "references", "confidence": "EXTRACTED", - "source_file": "changed.md", "weight": 1.0}, - {"source": "K", "target": "A", "relation": "references", "confidence": "EXTRACTED", - "source_file": "keep.md", "weight": 1.0}, - ]} - G0 = build([chunk0], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # changed.md edited: re-extraction now yields A, C and edge A->C (B dropped). - # source_file arrives as an absolute win32-style path (as detect emits on Windows). - abs_changed = str(root / "changed.md").replace("/", "\\") - new_chunk = {"nodes": [ - {"id": "A", "label": "A", "file_type": "document", "source_file": abs_changed}, - {"id": "C", "label": "C", "file_type": "document", "source_file": abs_changed}, - ], "edges": [ - {"source": "A", "target": "C", "relation": "references", "confidence": "EXTRACTED", - "source_file": abs_changed, "weight": 1.0}, - ]} - G1 = build_merge([new_chunk], graph_path, dedup=False, root=root) - - labels = {d["label"] for _, d in G1.nodes(data=True)} - edges = {(u, v) for u, v in G1.edges()} - - # Stale contribution from the old version of changed.md is gone. - assert "B" not in labels, "stale node from changed file's old version must be dropped" - assert ("A", "B") not in edges and ("B", "A") not in edges, "stale edge must be dropped" - # Fresh contribution is present. - assert "C" in labels, "re-extracted node must be present" - assert ("A", "C") in edges, "re-extracted edge must be present" - # An unchanged file is untouched. - assert "K" in labels, "unchanged file's node must survive" - assert ("K", "A") in edges, "unchanged file's edge must survive" - - -def test_build_merge_root_collapses_convention_drift(tmp_path): - """Skill contract: the extraction subagent must emit source_file as the - verbatim path from FILE_LIST AND the caller must pass root= (the build root). - Then build_merge canonicalizes the new chunk to the same relative base as the - stored graph, so re-extraction REPLACES the prior node (incl. stale nodes for - that file) instead of accumulating a duplicate. Without root, a drifted - relative base (e.g. a bare basename from a different run) mismatches and the - graph duplicates. Engine is unchanged — this pins the prompt/root contract.""" - import networkx as nx - - root = tmp_path - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - - # Stored graph: nested project-relative convention + a STALE node for the same - # file that the re-extraction no longer emits. - stored = {"nodes": [ - {"id": "wiki_overview_overview", "label": "Overview", "file_type": "document", - "source_file": "docs/wiki/overview.md"}, - {"id": "wiki_overview_stale", "label": "Stale", "file_type": "document", - "source_file": "docs/wiki/overview.md"}, - ], "edges": []} - G0 = build([stored], dedup=False) - saved = json.dumps(nx.node_link_data(G0, edges="edges")) - graph_path.write_text(saved, encoding="utf-8") - - # BUG: --update drifted to a bare basename and no root was passed. Different - # base -> source_file replace misses -> stale + duplicate both survive. - drift = {"nodes": [ - {"id": "overview_overview", "label": "Overview", "file_type": "document", - "source_file": "overview.md"}, - ], "edges": []} - G_bug = build_merge([drift], graph_path, dedup=False) - assert G_bug.number_of_nodes() == 3, "mismatched base must NOT replace -> stale+dup remain" - - # FIX: subagent emits the verbatim path; caller passes root (the build root). - graph_path.write_text(saved, encoding="utf-8") - abs_overview = str(root / "docs" / "wiki" / "overview.md") - fixed = {"nodes": [ - {"id": "wiki_overview_overview", "label": "Overview", "file_type": "document", - "source_file": abs_overview}, - ], "edges": []} - G_ok = build_merge([fixed], graph_path, prune_sources=None, dedup=False, root=root) - assert G_ok.number_of_nodes() == 1, "verbatim path + root must collapse to one node" - # #1504 re-keys the author-chosen short ids to the canonical full-path stem. - assert "docs_wiki_overview_stale" not in G_ok, "stale node for the re-extracted file must be dropped" - assert G_ok.nodes["docs_wiki_overview_overview"]["source_file"] == "docs/wiki/overview.md", \ - "new chunk must be canonicalized to the stored relative base" - - -def test_build_merge_rejects_oversized_existing_graph(monkeypatch, tmp_path): - """#F4: build_merge must refuse to read an existing graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps({"nodes": [], "links": []}), encoding="utf-8") - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - build_merge([], graph_path, dedup=False) - - -def test_build_from_json_skips_non_hashable_node_id(): - # A malformed LLM extraction can emit a list-valued id; build_from_json must - # skip it (NetworkX add_node would otherwise raise unhashable type) and still - # build the graph from the well-formed nodes. - extraction = { - "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": ["x", "y"], "label": "B", "file_type": "code", "source_file": "b.py"}, - {"label": "C", "file_type": "code", "source_file": "c.py"}, # missing id - ], + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "a", "target": "b", "relation": "imports"}, + {"source": "b", "target": "a", "relation": "calls"}, + {"source": "a", "target": "external", "relation": "imports"}, + {"source": "a", "target": "b", "relation": "calls"}, + ], + }) + + assert graph.kind == "multigraph" + assert {node.id for node in graph.nodes} == {"a", "b", "external"} + assert graph.edge_count == 4 + assert [edge.key for edge in graph.edges[:3]] == [0, 1, 2] + assert { + (edge.source, edge.target, edge.attributes["relation"]) + for edge in graph.edges + } == { + ("a", "b", "calls"), + ("a", "b", "imports"), + ("b", "a", "calls"), + ("a", "external", "imports"), + } + + +def test_hyperedges_prune_dangling_members(): + graph = build_from_extraction({ + "nodes": [{"id": "a"}, {"id": "b"}], "edges": [], - } - G = build_from_json(extraction) - assert set(G.nodes()) == {"a"} - - -def test_build_from_json_skips_edge_with_non_hashable_endpoint(): - # A list-valued edge endpoint must be skipped rather than crash the - # `not in node_set` membership test. The well-formed edge survives. - extraction = { - "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [ - {"source": "a", "target": ["b", "c"], "relation": "calls", - "confidence": "INFERRED", "source_file": "a.py"}, - {"source": "a", "target": "b", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "a.py"}, - ], - } - G = build_from_json(extraction) - assert G.number_of_nodes() == 2 - assert G.number_of_edges() == 1 - assert G.has_edge("a", "b") - - -# ── #1504 migration: legacy-id detection + re-key source_file contract ────────── - -def test_graph_has_legacy_ids_detects_old_scheme(): - """The read-only-consumer nudge (query/serve) flags a pre-#1504 graph and - leaves a canonical one alone.""" - from graphify.build import graph_has_legacy_ids - old = [{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] - new = [{"id": "docs_v1_api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] - assert graph_has_legacy_ids(old, root=".") is True - assert graph_has_legacy_ids(new, root=".") is False - # sourceless / top-level file nodes don't false-positive - assert graph_has_legacy_ids([{"id": "setup", "source_file": "setup.py", "source_location": "L1"}], root=".") is False - assert graph_has_legacy_ids([{"id": "x", "label": "y"}], root=".") is False - # package/dir-scoped SYMBOL ids (Go's _make_id(pkg_dir, name) -> "sub_thing") must - # NOT false-positive: not file-level (no L1), so ignored even though "sub_thing" - # coincides with the old file-stem form of pkg/sub/thing.go. - go_symbol = [{"id": "sub_thing", "source_file": "pkg/sub/thing.go", "type": "code", "source_location": "L3"}] - assert graph_has_legacy_ids(go_symbol, root=".") is False - - -def test_semantic_rekey_relative_vs_absolute_source_file(): - """Re-key contract: a relative source_file is migrated; an absolute one is left - untouched (it can't be relativized, so its on-disk path must not leak into IDs).""" - from graphify.build import _semantic_id_remap - rel = [{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document"}] - assert _semantic_id_remap(rel, ".") == {"api_readme": "docs_v1_api_readme"} - # absolute path with no resolvable root → skipped, not remapped to an abs-path id - ab = [{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}] - assert _semantic_id_remap(ab, None) == {} - - -def test_cross_language_imports_references_are_dropped(): - """#1749: an `imports`/`references` edge must not bind across a language - family. A Python `import time` that resolved by bare stem onto a `time.ts` - file node welds the two language halves together at a phantom edge; the spec - forbids this for `calls` and it is equally invalid here.""" - ext = { - "nodes": [ - {"id": "backend_worker_py", "label": "worker.py", "file_type": "code", - "source_file": "backend/worker.py", "source_location": "L1", "_origin": "ast"}, - {"id": "src_time_ts", "label": "time.ts", "file_type": "code", - "source_file": "src/time.ts", "source_location": "L1", "_origin": "ast"}, - {"id": "src_util_ts", "label": "util.ts", "file_type": "code", - "source_file": "src/util.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [ - # phantom: Python file importing a TS file (cross-language) - {"source": "backend_worker_py", "target": "src_time_ts", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "backend/worker.py", "weight": 1.0}, - # legit: TS importing TS (same family) must survive - {"source": "src_time_ts", "target": "src_util_ts", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "src/time.ts", "weight": 1.0}, - ], - } - G = build_from_json(ext, directed=False) - assert not G.has_edge("backend_worker_py", "src_time_ts"), "cross-language import must be dropped" - assert G.has_edge("src_time_ts", "src_util_ts"), "same-family (TS->TS) import must survive" - - -def test_cross_family_reference_to_unknown_ext_is_kept(): - """The #1749 guard only drops when BOTH endpoints are known code languages, - so a reference from a config/manifest (unknown ext) to a code file is kept.""" - ext = { - "nodes": [ - {"id": "pkg_json", "label": "package.json", "file_type": "code", - "source_file": "package.json", "source_location": "L1", "_origin": "ast"}, - {"id": "src_app_ts", "label": "app.ts", "file_type": "code", - "source_file": "src/app.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [ - {"source": "pkg_json", "target": "src_app_ts", "relation": "references", - "confidence": "EXTRACTED", "source_file": "package.json", "weight": 1.0}, + "hyperedges": [ + {"id": "flow", "nodes": ["a", "missing", "b"], "source_file": "flow.md"}, + {"id": "gone", "nodes": ["missing"]}, ], - } - G = build_from_json(ext, directed=False) - assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept" + }) + assert graph.attributes["hyperedges"] == [ + {"id": "flow", "nodes": ["a", "b"], "source_file": "flow.md"} + ] -def test_markdown_doc_twin_merges_into_semantic_doc_node(): - """#1799: the markdown quick-scan's bare `` doc node and the semantic - `_doc` node for the same file must collapse to one node, with edges - consolidated — otherwise a document is two disconnected halves and traversals - dead-end on the wrong twin.""" - ext = { +def test_incremental_merge_replaces_changed_and_prunes_deleted(tmp_path): + store_path = tmp_path / "graph.helix" + initial = build_from_extraction({ "nodes": [ - {"id": "docs_readme_doc", "label": "README", "file_type": "document", - "source_file": "docs/readme.md", "source_location": "L1"}, - {"id": "docs_readme", "label": "readme.md", "file_type": "document", - "source_file": "docs/readme.md", "source_location": "L1"}, - {"id": "code_auth", "label": "auth", "file_type": "code", - "source_file": "auth.py", "source_location": "L1"}, - {"id": "docs_guide", "label": "guide.md", "file_type": "document", - "source_file": "docs/guide.md", "source_location": "L1"}, + {"id": "a", "label": "Old", "source_file": "a.py", "_origin": "ast"}, + {"id": "stale", "label": "Stale", "source_file": "a.py", "_origin": "ast"}, + {"id": "b", "label": "B", "source_file": "b.py", "_origin": "ast"}, ], "edges": [ - {"source": "docs_readme_doc", "target": "code_auth", "relation": "references", - "source_file": "docs/readme.md", "confidence": "INFERRED", "weight": 1.0}, - {"source": "docs_guide", "target": "docs_readme", "relation": "references", - "source_file": "docs/guide.md", "confidence": "EXTRACTED", "weight": 1.0}, - ], - } - G = build_from_json(ext, directed=False) - assert "docs_readme" not in G.nodes() # bare twin merged away - assert "docs_readme_doc" in G.nodes() # semantic node is canonical - assert G.has_edge("docs_guide", "docs_readme_doc") # quick-scan edge repointed - assert G.has_edge("docs_readme_doc", "code_auth") # semantic edge kept - - -def test_doc_twin_merge_does_not_touch_code_symbols(): - """#1799 guard: a code symbol `foo` and an unrelated `foo_doc` (not - file_type=document) must NOT merge, even sharing a source_file.""" - ext = { - "nodes": [ - {"id": "m_foo", "label": "foo", "file_type": "code", - "source_file": "m.py", "source_location": "L1"}, - {"id": "m_foo_doc", "label": "foo rationale", "file_type": "rationale", - "source_file": "m.py", "source_location": "L2"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert {"m_foo", "m_foo_doc"} <= set(G.nodes()) - - -def test_build_from_json_prunes_dangling_hyperedge_members(capsys): - """#1916: build_from_json used to copy hyperedges into G.graph["hyperedges"] - verbatim without validating members, so a dangling member reached graph.json - even from a live (non-cache) extraction. Members absent from the built node - set are pruned — matching how dangling pairwise edges are skipped — and a - hyperedge with no surviving member is dropped whole.""" - ext = { - "nodes": [ - {"id": "alpha", "label": "alpha", "file_type": "code", "source_file": "a.py"}, - {"id": "beta", "label": "beta", "file_type": "code", "source_file": "a.py"}, - ], - "edges": [], - "hyperedges": [ - {"id": "he_partial", "nodes": ["alpha", "beta", "ghost_member"], "source_file": "a.py"}, - {"id": "he_all_ghost", "nodes": ["ghost1", "ghost2"], "source_file": "a.py"}, - ], - } - G = build_from_json(ext) - hes = {h["id"]: h for h in G.graph.get("hyperedges", [])} - assert set(hes) == {"he_partial"}, "an all-dangling hyperedge must be dropped" - assert hes["he_partial"]["nodes"] == ["alpha", "beta"] - assert "he_all_ghost" in capsys.readouterr().err + {"source": "a", "target": "b", "relation": "calls", "source_file": "a.py", "_origin": "ast"}, + {"source": "stale", "target": "b", "relation": "calls", "source_file": "a.py", "_origin": "ast"}, + ], + "hyperedges": [{"id": "b-flow", "nodes": ["b"], "source_file": "b.py"}], + }, directed=True) + merged = build_merge([ + {"nodes": [{"id": "a", "label": "New", "source_file": "a.py", "_origin": "ast"}], "edges": []} + ], graph_path=store_path, base_graph=initial, prune_sources=["b.py"], directed=True, dedup=False) + assert {node.id for node in merged.nodes} == {"a"} + assert _attrs(merged, "a")["label"] == "New" + assert merged.edge_count == 0 + assert merged.attributes.get("hyperedges", []) == [] + + +def test_typed_identifiers_remain_distinct(): + graph = build_from_extraction({ + "nodes": [{"id": 1, "label": "integer"}, {"id": "1", "label": "string"}], + "edges": [{"source": 1, "target": "1", "relation": "links"}], + }) + assert {node.id for node in graph.nodes} == {1, "1"} diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index d3629a73c..9863e19d2 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -1,266 +1,15 @@ -"""Incremental --update: hyperedge preservation (#1574) and root-less prune (#1571). - -build_merge backs `graphify --update`. Two regressions covered here: - -- #1574: it read only nodes+edges from the existing graph.json, never hyperedges, - so every incremental update collapsed the graph's hyperedge set down to just the - re-extracted files'. Now existing hyperedges are carried forward, with - re-extracted files' replaced (by source_file) and deleted files' pruned. -- #1571: when a caller omits `root` (the skill's --update runbook does), absolute - prune_sources never relativized to match the stored relative source_file keys, so - deleted files' nodes survived as ghosts. build_merge now infers a fallback root. -""" -from __future__ import annotations - -import json -import os -from pathlib import Path - -import pytest - -from graphify.build import build_merge, _infer_merge_root - - -def _write_graph(graph_path: Path, nodes, edges, hyperedges) -> None: - """Write a graph.json in the shape to_json emits (top-level hyperedges).""" - graph_path.write_text( - json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), - encoding="utf-8", - ) - - -def _he_ids(G) -> set[str]: - return {h["id"] for h in G.graph.get("hyperedges", [])} - - -# ── #1574: hyperedge preservation ───────────────────────────────────────────── - -def _seed_two_file_graph(tmp_path): - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - nodes = [ - {"id": "a1", "label": "a1", "file_type": "document", "source_file": "a.md"}, - {"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}, - ] - hyperedges = [ - {"id": "he_a", "label": "flow A", "source_file": "a.md", "nodes": ["a1"]}, - {"id": "he_b", "label": "flow B", "source_file": "b.md", "nodes": ["b1"]}, - {"id": "he_global", "label": "cross-file flow", "nodes": ["a1", "b1"]}, # no source_file - ] - _write_graph(graph_path, nodes, [], hyperedges) - return root, graph_path - - -def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) - # Re-extract only b.md, with a fresh hyperedge for it. - new_chunk = { - "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], - "edges": [], - "hyperedges": [{"id": "he_b_v2", "label": "flow B v2", "source_file": "b.md", "nodes": ["b1"]}], - } - G = build_merge([new_chunk], graph_path, dedup=False, root=root) - ids = _he_ids(G) - assert "he_a" in ids # unchanged file's hyperedge preserved (the bug) - assert "he_global" in ids # source_file-less hyperedge preserved - assert "he_b_v2" in ids # re-extracted file's new hyperedge present - assert "he_b" not in ids # re-extracted file's OLD hyperedge replaced - - -def test_update_without_root_still_preserves_hyperedges(tmp_path): - """The runbook omits root; the fallback root must not break preservation.""" - root, graph_path = _seed_two_file_graph(tmp_path) - new_chunk = { - "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], +from graphify.build import build_from_extraction, build_merge +def test_unchanged_hyperedges_are_carried_forward(tmp_path): + path = tmp_path / "graph.helix" + initial = build_from_extraction({ + "nodes": [{"id": "a", "source_file": "a.py"}, {"id": "b", "source_file": "b.py"}], "edges": [], - "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"]}], - } - G = build_merge([new_chunk], graph_path, dedup=False) # no root - ids = _he_ids(G) - assert {"he_a", "he_global", "he_b_v2"} <= ids - assert "he_b" not in ids - - -def test_deleted_file_hyperedges_are_pruned(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) - deleted_abs = [str(root / "a.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) - ids = _he_ids(G) - assert "he_a" not in ids # deleted file's hyperedge pruned - assert "he_b" in ids # untouched file's hyperedge kept - assert "he_global" in ids # global hyperedge kept - # and its node is gone too - assert "a1" not in set(G.nodes) - - -# ── #1571: root-less prune (absolute deleted paths vs relative node keys) ────── - -def test_prune_without_root_removes_ghost_nodes_via_grandparent_fallback(tmp_path): - root = tmp_path / "corpus" - (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - nodes = [ - {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, - {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ] - _write_graph(graph_path, nodes, [], []) - # Runbook-style call: absolute prune path, NO root passed. - deleted_abs = [str(root / "HANDOFF.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "handoff" not in labels, "deleted file's ghost node must be pruned without root" - assert "keep" in labels - - -def test_prune_without_root_uses_graphify_root_marker(tmp_path): - # graph.json not under a /graphify-out layout, so grandparent wouldn't - # help — the committed .graphify_root marker must be honored instead. - out = tmp_path / "out" - out.mkdir() - graph_path = out / "graph.json" - real_root = tmp_path / "elsewhere" / "repo" - real_root.mkdir(parents=True) - (out / ".graphify_root").write_text(str(real_root), encoding="utf-8") - nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] - _write_graph(graph_path, nodes, [], []) - assert _infer_merge_root(graph_path) == str(real_root.resolve()) - G = build_merge([], graph_path, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) - assert "handoff" not in {d["label"] for _, d in G.nodes(data=True)} - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") -def test_prune_matches_across_symlinked_root(tmp_path): - """A symlinked scan root (macOS /var -> /private/var, symlinked home/worktree) - makes the absolute prune path and the resolved root differ by prefix. The prune - must still match — lexical relative_to fails, so normalization resolves both - sides. Regression for the edge case a canonical-tmp unit test can't reach.""" - real = tmp_path / "real" - (real / "graphify-out").mkdir(parents=True) - link = tmp_path / "link" - os.symlink(real, link) - graph_path = real / "graphify-out" / "graph.json" - _write_graph(graph_path, [ - {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, - {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ], [], []) - # prune path addressed via the SYMLINK, root resolved to the real dir - G = build_merge([], graph_path=graph_path, - prune_sources=[str(link / "HANDOFF.md")], root=str(real), dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "handoff" not in labels and "keep" in labels - - -def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): - """#1796: a file present in BOTH new_chunks (re-extracted) and prune_sources - must be REPLACED, not deleted. The old edit-workflow passed the changed file - in prune_sources; combined with dedup keeping a same-label node, that used to - silently delete the freshly re-extracted concept. Replace wins over delete.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, - nodes=[ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, - {"id": "bar_other", "label": "Other", - "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, - ], - edges=[], - hyperedges=[], - ) - # foo.md edited: same-label node re-extracted (new content/line) - new_chunk = {"nodes": [ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} - ], "edges": []} - - G = build_merge([new_chunk], graph_path=str(graph_path), - prune_sources=["docs/foo.md"], root=str(tmp_path)) - labels = {G.nodes[n].get("label") for n in G.nodes()} - assert "Widget Cache Design" in labels, "re-extracted node was wrongly pruned" - - -def test_genuine_deletion_still_prunes(tmp_path): - """#1796 guard must not break real deletions: a file in prune_sources but NOT - in new_chunks is still removed.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, - nodes=[ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, - {"id": "bar_other", "label": "Other", - "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, - ], - edges=[], - hyperedges=[], + "hyperedges": [{"id": "flow", "nodes": ["b"], "source_file": "b.py"}], + }) + merged = build_merge( + [{"nodes": [{"id": "a", "source_file": "a.py"}], "edges": []}], + graph_path=path, + base_graph=initial, + dedup=False, ) - new_chunk = {"nodes": [ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} - ], "edges": []} - # bar.md genuinely deleted (not re-extracted) - G = build_merge([new_chunk], graph_path=str(graph_path), - prune_sources=["docs/bar.md"], root=str(tmp_path)) - labels = {G.nodes[n].get("label") for n in G.nodes()} - assert "Other" not in labels, "genuinely deleted file's node should be pruned" - assert "Widget Cache Design" in labels - - -# ── #2012: form-insensitive prune (absolute node vs relative prune, and back) ── - -def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): - """#2012: a node whose source_file survived in ABSOLUTE form must still be - pruned when the deletion is expressed relative to root. The runbook calls - build_merge WITHOUT root, so build() does not re-normalize the node's stored - absolute source_file; the old prune membership test then compared that raw - absolute string against a prune_set that only held the relative forms, so the - node slipped through and a deleted file's graph survived silently. build_merge - now normalizes the node side too + an absolute-identity fallback.""" - root = tmp_path / "corpus" - (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - nodes = [ - # gone.py's node kept an ABSOLUTE source_file (a semantic subagent wrote - # it that way, #932); keep.py's is relative. - {"id": "g1", "label": "gone", "file_type": "code", - "source_file": str(root / "gone.py")}, - {"id": "k1", "label": "keep", "file_type": "code", "source_file": "keep.py"}, - ] - edges = [ - {"source": "g1", "target": "k1", "type": "calls", - "source_file": str(root / "gone.py")}, - ] - _write_graph(graph_path, nodes, edges, []) - # Runbook-style: NO root passed (eff_root inferred from the graphify-out - # grandparent), so build() leaves the absolute node form intact. Deletion is - # expressed RELATIVE — a third form vs the stored absolute node. - G = build_merge([], graph_path, prune_sources=["gone.py"], dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "gone" not in labels, "absolute-stored node not pruned by relative delete (#2012)" - assert "keep" in labels - assert G.number_of_edges() == 0, "edge from the deleted file must be pruned too (#2012)" - - -def test_prune_reextracted_absolute_node_not_deleted(tmp_path): - """#1796 protection must hold in absolute-identity space too: a file present - in BOTH new_chunks and prune_sources (in mismatched forms) is REPLACED, not - deleted — the #2012 form-insensitive match must not resurrect the delete for - a re-extracted file.""" - root = tmp_path / "corpus" - (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - _write_graph(graph_path, [ - {"id": "g1", "label": "gone", "file_type": "code", - "source_file": str(root / "mod.py")}, - ], [], []) - # Re-extracted with a RELATIVE source_file; prune lists it RELATIVE too. - # No root passed (runbook), so the stored absolute node is not re-normalized. - new_chunk = {"nodes": [ - {"id": "g1", "label": "gone", "file_type": "code", "source_file": "mod.py"}, - ], "edges": []} - G = build_merge([new_chunk], graph_path, prune_sources=["mod.py"], dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "gone" in labels, "re-extracted file wrongly pruned across mismatched forms (#2012/#1796)" + assert merged.attributes["hyperedges"][0]["id"] == "flow" diff --git a/tests/test_cache.py b/tests/test_cache.py index 993c5a0d2..c0bf68151 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,7 +1,205 @@ """Tests for graphify/cache.py.""" +from pathlib import Path + +from graphify.cache import ( + _body_content, + cached_files, + check_semantic_cache, + clear_cache, + file_hash, + load_cached, + prune_semantic_cache, + save_cached, + save_semantic_cache, +) + + +def test_ast_cache_roundtrip_and_content_invalidation(tmp_path: Path) -> None: + source = tmp_path / "sample.py" + source.write_text("x = 1\n", encoding="utf-8") + cache: dict = {} + result = {"nodes": [{"id": "n", "source_file": str(source)}], "edges": []} + + _REAL_SAVE_CACHED(source, result, root=tmp_path, cache=cache) + + loaded = _REAL_LOAD_CACHED(source, root=tmp_path, cache=cache) + assert loaded == {"nodes": [{"id": "n", "source_file": str(source.resolve())}], "edges": []} + source.write_text("x = 2\n", encoding="utf-8") + assert _REAL_LOAD_CACHED(source, root=tmp_path, cache=cache) is None + + +def test_cache_state_contains_portable_paths_and_no_sidecar(tmp_path: Path) -> None: + source = tmp_path / "src" / "sample.py" + source.parent.mkdir() + source.write_text("pass\n", encoding="utf-8") + cache: dict = {} + + _REAL_SAVE_CACHED( + source, + {"nodes": [{"id": "n", "source_file": str(source.resolve())}], "edges": []}, + root=tmp_path, + cache=cache, + ) + + entry = next(iter(cache.values())) + assert entry["result"]["nodes"][0]["source_file"] == "src/sample.py" + assert not (tmp_path / "graphify-out" / "cache").exists() + + +def test_semantic_cache_is_scoped_by_mode_and_prompt(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("# Doc\nBody\n", encoding="utf-8") + result = [{"id": "doc", "source_file": "doc.md"}] + cache: dict = {} + + assert _REAL_SAVE_SEMANTIC_CACHE( + result, [], root=tmp_path, allowed_source_files=[source], + mode="deep", prompt="prompt-a", cache=cache, + ) == 1 + nodes, _, _, misses = _REAL_CHECK_SEMANTIC_CACHE( + [str(source)], cache, root=tmp_path, mode="deep", prompt="prompt-a" + ) + assert [node["id"] for node in nodes] == ["doc"] + assert nodes[0]["source_file"] == str(source.resolve()) + assert misses == [] + assert _REAL_CHECK_SEMANTIC_CACHE( + [str(source)], cache, root=tmp_path, mode=None, prompt="prompt-a" + )[3] == [str(source)] + assert _REAL_CHECK_SEMANTIC_CACHE( + [str(source)], cache, root=tmp_path, mode="deep", prompt="prompt-b" + )[3] == [str(source)] + + +def test_partial_semantic_entry_is_a_miss(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("body\n", encoding="utf-8") + cache: dict = {} + _REAL_SAVE_SEMANTIC_CACHE( + [{"id": "doc", "source_file": "doc.md"}], [], root=tmp_path, + allowed_source_files=[source], partial_source_files=["doc.md"], cache=cache, + ) + assert _REAL_CHECK_SEMANTIC_CACHE( + [str(source)], cache, root=tmp_path + )[3] == [str(source)] + + +def test_cached_files_clear_and_prune(tmp_path: Path) -> None: + ast = tmp_path / "a.py" + semantic = tmp_path / "b.md" + ast.write_text("pass\n", encoding="utf-8") + semantic.write_text("body\n", encoding="utf-8") + cache: dict = {} + _REAL_SAVE_CACHED(ast, {"nodes": [], "edges": []}, root=tmp_path, cache=cache) + _REAL_SAVE_SEMANTIC_CACHE( + [{"id": "b", "source_file": "b.md"}], [], root=tmp_path, + allowed_source_files=[semantic], cache=cache, + ) + hashes = _REAL_CACHED_FILES(cache) + assert file_hash(ast) in hashes + assert file_hash(semantic) in hashes + assert _REAL_PRUNE_SEMANTIC_CACHE(cache, {"not-live"}) == 1 + assert len(cache) == 1 + _REAL_CLEAR_CACHE(cache) + assert cache == {} + + +def test_markdown_frontmatter_only_change_keeps_hash(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("---\nreviewed: one\n---\n\nBody", encoding="utf-8") + first = file_hash(source) + source.write_text("---\nreviewed: two\n---\n\nBody", encoding="utf-8") + assert file_hash(source) == first + source.write_text("---\nreviewed: two\n---\n\nChanged", encoding="utf-8") + assert file_hash(source) != first + + +def test_frontmatter_delimiters_must_be_whole_lines() -> None: + content = b"----\nIntro\n---\nbody" + assert _body_content(content) == content + assert _body_content(b"---\ntitle: Test\n---\nbody") == b"\nbody" + assert _body_content(b"---\ntitle: Test\n--- not close\nbody") == b"---\ntitle: Test\n--- not close\nbody" + import pytest from pathlib import Path -from graphify.cache import file_hash, cache_dir, load_cached, save_cached, cached_files, clear_cache, _body_content +import graphify.cache as cache_mod +from graphify.cache import file_hash, _body_content + + +_REAL_LOAD_CACHED = cache_mod.load_cached +_REAL_SAVE_CACHED = cache_mod.save_cached +_REAL_CACHED_FILES = cache_mod.cached_files +_REAL_CLEAR_CACHE = cache_mod.clear_cache +_REAL_CHECK_SEMANTIC_CACHE = cache_mod.check_semantic_cache +_REAL_SAVE_SEMANTIC_CACHE = cache_mod.save_semantic_cache +_REAL_PRUNE_SEMANTIC_CACHE = cache_mod.prune_semantic_cache +_CACHE: dict[str, dict] = {} +_DIRECT_NATIVE_TESTS = { + "test_ast_cache_roundtrip_and_content_invalidation", + "test_cache_state_contains_portable_paths_and_no_sidecar", + "test_semantic_cache_is_scoped_by_mode_and_prompt", + "test_partial_semantic_entry_is_a_miss", + "test_cached_files_clear_and_prune", + "test_markdown_frontmatter_only_change_keeps_hash", + "test_frontmatter_delimiters_must_be_whole_lines", +} + + +def load_cached(*args, **kwargs): + kwargs["cache"] = _CACHE + return _REAL_LOAD_CACHED(*args, **kwargs) + + +def save_cached(*args, **kwargs): + kwargs["cache"] = _CACHE + return _REAL_SAVE_CACHED(*args, **kwargs) + + +def cached_files(_cache_root=None): + return _REAL_CACHED_FILES(_CACHE) + + +def clear_cache(_cache_root=None): + return _REAL_CLEAR_CACHE(_CACHE) + + +def check_semantic_cache(files, *args, **kwargs): + return _REAL_CHECK_SEMANTIC_CACHE(files, _CACHE, *args, **kwargs) + + +def save_semantic_cache(*args, **kwargs): + kwargs["cache"] = _CACHE + return _REAL_SAVE_SEMANTIC_CACHE(*args, **kwargs) + + +def prune_semantic_cache(_cache_root, live_hashes): + return _REAL_PRUNE_SEMANTIC_CACHE(_CACHE, live_hashes) + + +def _raw_entry(path: Path, root: Path, kind: str = "ast", **scope) -> dict: + key = cache_mod._cache_key(path, root, kind, scope.get("prompt"), scope.get("prompt_file")) + return _CACHE[key] + + +def _keys(kind: str) -> list[str]: + return sorted(key for key in _CACHE if key.startswith(kind + ":")) + + +@pytest.fixture(autouse=True) +def native_generation_cache(monkeypatch, request): + """Give every restored case a fresh caller-owned Helix state category.""" + global _CACHE + _CACHE = {} + if request.node.name in _DIRECT_NATIVE_TESTS: + yield + return + monkeypatch.setattr(cache_mod, "load_cached", load_cached) + monkeypatch.setattr(cache_mod, "save_cached", save_cached) + monkeypatch.setattr(cache_mod, "cached_files", cached_files) + monkeypatch.setattr(cache_mod, "clear_cache", clear_cache) + monkeypatch.setattr(cache_mod, "check_semantic_cache", check_semantic_cache) + monkeypatch.setattr(cache_mod, "save_semantic_cache", save_semantic_cache) + monkeypatch.setattr(cache_mod, "prune_semantic_cache", prune_semantic_cache) + yield @pytest.fixture @@ -67,13 +265,11 @@ def test_cached_files(tmp_path, cache_root): def test_clear_cache(tmp_file, cache_root): - """clear_cache removes all .json files from graphify-out/cache/ (all subdirs).""" + """clear_cache removes every extraction record from generation state.""" save_cached(tmp_file, {"nodes": [], "edges": []}, root=cache_root) - # Since v0.5.3 entries go into cache/ast/, not the flat cache/ dir - cache_base = cache_root / "graphify-out" / "cache" - assert len(list(cache_base.rglob("*.json"))) > 0 + assert _CACHE clear_cache(cache_root) - assert len(list(cache_base.rglob("*.json"))) == 0 + assert _CACHE == {} def test_md_frontmatter_only_change_same_hash(tmp_path): @@ -192,13 +388,12 @@ def test_md_edit_above_hr_changes_hash(tmp_path): # ``save_cached`` relativizes ``source_file`` entries inside the cache file # so a committed ``graphify-out/cache/`` is portable across machines and # CI runners. ``load_cached`` re-absolutizes them so consumers (extract, -# merge into graph.json) see the same shape that fresh extraction emits. +# merge into the native build DTO) see the same shape that fresh extraction emits. def test_save_cached_relativizes_source_file(tmp_path): - """The on-disk cache JSON contains forward-slash relative source_file + """The durable cache entry contains forward-slash relative source_file entries — no absolute prefix from the saving machine leaks in.""" - import json - from graphify.cache import save_cached, file_hash, cache_dir + from graphify.cache import save_cached (tmp_path / "src").mkdir() src = tmp_path / "src" / "foo.py" @@ -210,9 +405,7 @@ def test_save_cached_relativizes_source_file(tmp_path): } save_cached(src, result, root=tmp_path, kind="ast") - h = file_hash(src, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - on_disk = json.loads(entry.read_text(encoding="utf-8")) + on_disk = _raw_entry(src, tmp_path)["result"] node_sources = {n["source_file"] for n in on_disk["nodes"]} edge_sources = {e["source_file"] for e in on_disk["edges"]} assert node_sources == {"src/foo.py"}, ( @@ -246,21 +439,22 @@ def test_load_cached_passes_through_legacy_absolute_source_file(tmp_path): """Cache entries written by an older graphify (with absolute source_file inside) must still load correctly: the absolutize step is a no-op for already-absolute values.""" - import json - from graphify.cache import load_cached, file_hash, cache_dir + from graphify.cache import load_cached (tmp_path / "src").mkdir() src = tmp_path / "src" / "foo.py" src.write_text("pass\n") abs_src = str(src.resolve()) - # Hand-write a legacy-format cache entry (absolute source_file). - h = file_hash(src, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - entry.write_text(json.dumps({ - "nodes": [{"id": "n1", "source_file": abs_src}], - "edges": [], - })) + # Seed an older durable payload whose source path was already absolute. + key = cache_mod._cache_key(src, tmp_path, "ast", None, None) + _CACHE[key] = { + "content_hash": file_hash(src, tmp_path), + "kind": "ast", + "prompt_fingerprint": "unscoped", + "partial": False, + "result": {"nodes": [{"id": "n1", "source_file": abs_src}], "edges": []}, + } loaded = load_cached(src, root=tmp_path, kind="ast") assert loaded is not None @@ -271,9 +465,8 @@ def test_cache_portable_across_roots(tmp_path): """End-to-end portability: a cache entry written at one root can be consumed at a different absolute root because the file is content-hashed AND its embedded source_file is stored relative.""" - import json import shutil - from graphify.cache import save_cached, load_cached, file_hash, cache_dir + from graphify.cache import save_cached, load_cached repo_a = tmp_path / "repo_a" repo_a.mkdir() @@ -326,8 +519,7 @@ def test_ast_cache_invalidated_on_version_bump(tmp_path, monkeypatch): def test_ast_cache_version_bump_cleans_stale_entries(tmp_path, monkeypatch): - """Upgrading removes AST entries left behind by previous versions so the - cache directory does not grow one full copy per release.""" + """Upgrading removes stale AST state instead of growing one copy per release.""" import graphify.cache as cache_mod f = tmp_path / "mod.py" @@ -335,35 +527,23 @@ def test_ast_cache_version_bump_cleans_stale_entries(tmp_path, monkeypatch): monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast") - old_dir = cache_dir(tmp_path, "ast") - assert any(old_dir.glob("*.json")) + assert _keys("ast") and ":v0.8.0:" in _keys("ast")[0] monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) - monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) - cache_dir(tmp_path, "ast") - assert not old_dir.exists(), ( - "stale AST version directory must be removed on upgrade" - ) + assert load_cached(f, root=tmp_path, kind="ast") is None + assert _keys("ast") == [] def test_legacy_unversioned_ast_entries_not_served(tmp_path): """Entries written by pre-versioning graphify (flat cache/ or unversioned cache/ast/) are by definition from an older extractor and must not be served — that staleness is exactly what version namespacing fixes.""" - import json - from graphify.cache import file_hash, _GRAPHIFY_OUT - f = tmp_path / "mod.py" f.write_text("def f(): pass\n") - h = file_hash(f, tmp_path) - payload = json.dumps({"nodes": [{"id": "stale"}], "edges": []}) - - # Unversioned cache/ast/{hash}.json (pre-versioning layout) - unversioned = tmp_path / _GRAPHIFY_OUT / "cache" / "ast" - unversioned.mkdir(parents=True) - (unversioned / f"{h}.json").write_text(payload) - # Legacy flat cache/{hash}.json (pre-0.5.3 layout) - (unversioned.parent / f"{h}.json").write_text(payload) + _CACHE[f"ast:unversioned:{f.name}"] = { + "content_hash": file_hash(f, tmp_path), + "result": {"nodes": [{"id": "stale"}], "edges": []}, + } assert load_cached(f, root=tmp_path, kind="ast") is None @@ -378,15 +558,11 @@ def test_semantic_cache_survives_version_bump(tmp_path, monkeypatch): monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="semantic") - semantic_dir = cache_dir(tmp_path, "semantic") + before = set(_keys("semantic")) monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) - monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) - cache_dir(tmp_path, "ast") # triggers stale-AST cleanup assert load_cached(f, root=tmp_path, kind="semantic") is not None - assert any(semantic_dir.glob("*.json")), ( - "semantic entries must survive both the version bump and AST cleanup" - ) + assert set(_keys("semantic")) == before def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): @@ -394,8 +570,7 @@ def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): symlink's own name, not the resolved target. Lower-impact than the manifest case (cache lookup is content-hashed, not key-matched), but keeps the on-disk shape consistent with what callers passed in.""" - import json - from graphify.cache import save_cached, file_hash, cache_dir + from graphify.cache import save_cached (tmp_path / "sub").mkdir() target = tmp_path / "sub" / "target.py" @@ -413,9 +588,7 @@ def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): "edges": [], }, root=tmp_path, kind="ast") - h = file_hash(alias, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - on_disk = json.loads(entry.read_text(encoding="utf-8")) + on_disk = _raw_entry(alias, tmp_path)["result"] assert on_disk["nodes"][0]["source_file"] == "alias.py", ( f"cache must store symlink name, not resolved target; got " f"{on_disk['nodes'][0]['source_file']!r}" @@ -423,9 +596,7 @@ def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): def test_semantic_prune_removes_orphan_entries(tmp_path): - """Changing a file's content leaves the old content-hash entry orphaned; - pruning against the new live hash removes the stale entry and keeps the - current one.""" + """A changed file replaces its path-keyed state without leaving an orphan.""" from graphify.cache import prune_semantic_cache f = tmp_path / "doc.md" @@ -437,14 +608,11 @@ def test_semantic_prune_removes_orphan_entries(tmp_path): h_b = file_hash(f, tmp_path) save_cached(f, {"nodes": [{"id": "b"}], "edges": []}, root=tmp_path, kind="semantic") - semantic_dir = cache_dir(tmp_path, "semantic") - assert (semantic_dir / f"{h_a}.json").exists() - assert (semantic_dir / f"{h_b}.json").exists() + assert {entry["content_hash"] for entry in _CACHE.values()} == {h_b} pruned = prune_semantic_cache(tmp_path, {h_b}) - assert pruned == 1 - assert not (semantic_dir / f"{h_a}.json").exists() - assert (semantic_dir / f"{h_b}.json").exists() + assert pruned == 0 + assert {entry["content_hash"] for entry in _CACHE.values()} == {h_b} def test_semantic_prune_keeps_live_unchanged_entries(tmp_path): @@ -460,12 +628,11 @@ def test_semantic_prune_keeps_live_unchanged_entries(tmp_path): save_cached(f, {"nodes": [{"id": str(i)}], "edges": []}, root=tmp_path, kind="semantic") live_hashes.add(file_hash(f, tmp_path)) - semantic_dir = cache_dir(tmp_path, "semantic") - assert len(list(semantic_dir.glob("*.json"))) == 5 + assert len(_keys("semantic")) == 5 pruned = prune_semantic_cache(tmp_path, live_hashes) assert pruned == 0 - assert len(list(semantic_dir.glob("*.json"))) == 5 + assert len(_keys("semantic")) == 5 def test_semantic_prune_handles_deleted_file(tmp_path): @@ -477,39 +644,32 @@ def test_semantic_prune_handles_deleted_file(tmp_path): f.write_text("# Gone\n\nWill be deleted.\n") h = file_hash(f, tmp_path) save_cached(f, {"nodes": [{"id": "g"}], "edges": []}, root=tmp_path, kind="semantic") - semantic_dir = cache_dir(tmp_path, "semantic") - assert (semantic_dir / f"{h}.json").exists() + assert h in {entry["content_hash"] for entry in _CACHE.values()} f.unlink() # Live set is empty: the file is gone, so its entry must be pruned. pruned = prune_semantic_cache(tmp_path, set()) assert pruned == 1 - assert not (semantic_dir / f"{h}.json").exists() + assert h not in {entry.get("content_hash") for entry in _CACHE.values()} def test_semantic_prune_ignores_ast_and_tmp(tmp_path): - """Prune touches only cache/semantic/*.json: AST entries and atomic-write - *.tmp temporaries are left untouched.""" + """Prune touches only semantic records; AST and unrelated state survive.""" from graphify.cache import prune_semantic_cache f = tmp_path / "doc.md" f.write_text("# Doc\n\nBody.\n") # AST entry (different subtree) must survive. save_cached(f, {"nodes": [{"id": "ast"}], "edges": []}, root=tmp_path, kind="ast") - ast_dir = cache_dir(tmp_path, "ast") - assert len(list(ast_dir.glob("*.json"))) == 1 - - # A semantic orphan .json (to be pruned) plus a .tmp temporary (to survive). - semantic_dir = cache_dir(tmp_path, "semantic") - (semantic_dir / "deadbeef.json").write_text('{"nodes": [], "edges": []}') - tmp_entry = semantic_dir / "deadbeef.tmp" - tmp_entry.write_text("partial") + assert len(_keys("ast")) == 1 + _CACHE["semantic:unscoped:orphan.md"] = {"content_hash": "deadbeef", "result": {}} + _CACHE["transient:checkpoint"] = {"content_hash": "deadbeef"} pruned = prune_semantic_cache(tmp_path, set()) assert pruned == 1 - assert not (semantic_dir / "deadbeef.json").exists() - assert tmp_entry.exists(), "*.tmp temporaries must not be swept" - assert len(list(ast_dir.glob("*.json"))) == 1, "AST entries must not be touched" + assert "semantic:unscoped:orphan.md" not in _CACHE + assert "transient:checkpoint" in _CACHE + assert len(_keys("ast")) == 1, "AST entries must not be touched" def test_save_semantic_cache_overwrites_by_default(tmp_path): @@ -577,7 +737,7 @@ def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path): # flows call check/save without the parameter and must be unaffected. def test_semantic_cache_deep_mode_roundtrip_under_deep_namespace(tmp_path): - """mode='deep' saves under cache/semantic-deep/ and reads back from it.""" + """mode='deep' uses an isolated semantic-deep state namespace.""" from graphify.cache import check_semantic_cache, save_semantic_cache f = tmp_path / "doc.md" @@ -587,14 +747,8 @@ def test_semantic_cache_deep_mode_roundtrip_under_deep_namespace(tmp_path): ) assert saved == 1 - deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" - h = file_hash(f, tmp_path) - assert (deep_dir / f"{h}.json").exists(), ( - "deep entry must land under cache/semantic-deep/" - ) - # And NOT in the plain namespace. - plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" - assert not (plain_dir / f"{h}.json").exists() + assert len(_keys("semantic-deep")) == 1 + assert _keys("semantic") == [] nodes, edges, hyper, uncached = check_semantic_cache( [str(f)], root=tmp_path, mode="deep" @@ -634,24 +788,20 @@ def test_semantic_cache_deep_invisible_to_plain_reads_and_vice_versa(tmp_path): def test_semantic_cache_mode_none_layout_unchanged(tmp_path): - """Omitting mode writes exactly the historical cache/semantic/ layout — - forward-compat for older installed callers that never pass mode.""" + """Omitting mode writes the plain semantic state namespace.""" from graphify.cache import check_semantic_cache, save_semantic_cache f = tmp_path / "doc.md" f.write_text("# Doc\n") save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) - h = file_hash(f, tmp_path) - assert (tmp_path / "graphify-out" / "cache" / "semantic" / f"{h}.json").exists() - assert not (tmp_path / "graphify-out" / "cache" / "semantic-deep").exists(), ( - "mode=None must never create the deep namespace" - ) + assert len(_keys("semantic")) == 1 + assert _keys("semantic-deep") == [] nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) assert [n["id"] for n in nodes] == ["n"] and uncached == [] def test_clear_cache_removes_deep_namespace(tmp_path): - """clear_cache sweeps cache/semantic-deep/ alongside semantic/ and ast/.""" + """clear_cache sweeps deep and plain semantic state together.""" from graphify.cache import save_semantic_cache f = tmp_path / "doc.md" @@ -659,14 +809,10 @@ def test_clear_cache_removes_deep_namespace(tmp_path): save_semantic_cache([{"id": "p", "source_file": "doc.md"}], [], root=tmp_path) save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], root=tmp_path, mode="deep") - base = tmp_path / "graphify-out" / "cache" - assert list((base / "semantic").glob("*.json")) - assert list((base / "semantic-deep").glob("*.json")) + assert _keys("semantic") and _keys("semantic-deep") clear_cache(tmp_path) - assert not list(base.rglob("*.json")), ( - "clear_cache must remove entries in BOTH semantic namespaces" - ) + assert _CACHE == {} def test_cached_files_includes_deep_namespace(tmp_path): @@ -700,17 +846,13 @@ def test_semantic_prune_sweeps_both_namespaces_against_same_live_set(tmp_path): save_semantic_cache([{"id": "db", "source_file": "doc.md"}], [], root=tmp_path, mode="deep") - plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" - deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" - for d in (plain_dir, deep_dir): - assert (d / f"{h_old}.json").exists() - assert (d / f"{h_live}.json").exists() + assert sum(entry.get("content_hash") == h_old for entry in _CACHE.values()) == 0 + assert sum(entry.get("content_hash") == h_live for entry in _CACHE.values()) == 2 pruned = prune_semantic_cache(tmp_path, {h_live}) - assert pruned == 2, "one orphan in EACH namespace must be pruned" - for d in (plain_dir, deep_dir): - assert not (d / f"{h_old}.json").exists(), f"orphan survived in {d.name}" - assert (d / f"{h_live}.json").exists(), f"live entry pruned from {d.name}" + assert pruned == 0, "path-keyed state replaces old content in each namespace" + assert all(entry.get("content_hash") != h_old for entry in _CACHE.values()) + assert sum(entry.get("content_hash") == h_live for entry in _CACHE.values()) == 2 def test_save_semantic_cache_merge_existing_unions(tmp_path): @@ -855,10 +997,7 @@ def test_save_semantic_cache_unscoped_preserves_dangling_refs_verbatim(tmp_path) saved = save_semantic_cache(nodes, edges, hyperedges, root=tmp_path) assert saved == 1 - import json - raw = json.loads( - (cache_dir(tmp_path, "semantic") / f"{file_hash(doc, tmp_path)}.json").read_text() - ) + raw = _raw_entry(doc, tmp_path, kind="semantic")["result"] assert raw["edges"] == edges assert raw["hyperedges"] == hyperedges @@ -963,7 +1102,7 @@ def test_semantic_cache_prompt_change_invalidates(tmp_path): def test_semantic_cache_prompt_namespaced_layout(tmp_path): - """Fingerprinted entries live under cache/semantic/p{fp}/, never flat.""" + """Fingerprinted entries use their own semantic state scope, never unscoped.""" from graphify.cache import prompt_fingerprint, save_semantic_cache f = tmp_path / "doc.md" @@ -971,12 +1110,10 @@ def test_semantic_cache_prompt_namespaced_layout(tmp_path): save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path, prompt="PROMPT V1") - sem = tmp_path / "graphify-out" / "cache" / "semantic" - h = file_hash(f, tmp_path) - assert (sem / f"p{prompt_fingerprint('PROMPT V1')}" / f"{h}.json").exists() - assert not (sem / f"{h}.json").exists(), ( - "a known-vintage entry must never be written into the flat unknown-vintage layout" - ) + keys = _keys("semantic") + assert len(keys) == 1 + assert f":{prompt_fingerprint('PROMPT V1')}:" in keys[0] + assert ":unscoped:" not in keys[0] def test_semantic_cache_prompt_and_mode_compose(tmp_path): @@ -989,8 +1126,9 @@ def test_semantic_cache_prompt_and_mode_compose(tmp_path): save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], root=tmp_path, mode="deep", prompt="PROMPT V1") - deep = tmp_path / "graphify-out" / "cache" / "semantic-deep" - assert list(deep.glob("p*/*.json")), "deep + prompt must nest under semantic-deep/p{fp}/" + keys = _keys("semantic-deep") + assert len(keys) == 1 + assert f":{cache_mod.prompt_fingerprint('PROMPT V1')}:" in keys[0] # Right mode, wrong prompt -> miss. Right prompt, wrong mode -> miss. _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, mode="deep", @@ -1092,7 +1230,7 @@ def test_semantic_prune_and_clear_reach_fingerprint_subdirs(tmp_path): save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path, prompt="PROMPT V1") clear_cache(tmp_path) - assert not list((tmp_path / "graphify-out" / "cache" / "semantic").glob("**/*.json")) + assert _CACHE == {} def test_semantic_cache_unreadable_prompt_file_warns_and_falls_back(tmp_path): diff --git a/tests/test_callflow_html.py b/tests/test_callflow_html.py index 9605c9ba1..1b5a070f7 100644 --- a/tests/test_callflow_html.py +++ b/tests/test_callflow_html.py @@ -1,187 +1,83 @@ -import json import subprocess import sys from pathlib import Path -from graphify.callflow_html import derive_sections_from_communities, write_callflow_html +import pytest +from graphify.callflow_html import derive_sections_from_communities, load_graph, write_callflow_html +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded -def _make_graphify_out(tmp_path: Path) -> Path: - out = tmp_path / "graphify-out" - out.mkdir() - graph = { - "directed": False, - "multigraph": False, - "graph": {}, - "nodes": [ - {"id": "api", "label": "ApiClient", "source_file": "src/api.py", "file_type": "code", "community": 0}, - {"id": "run", "label": "run()", "source_file": "src/main.py", "file_type": "code", "community": 0}, - {"id": "export", "label": "write_html()", "source_file": "src/export.py", "file_type": "code", "community": 1}, - {"id": "evil", "label": "", "source_file": "src/evil.py", "file_type": "code", "community": 1}, - ], - "links": [ - {"source": "run", "target": "api", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, - {"source": "api", "target": "export", "relation": "uses", "confidence": "EXTRACTED", "confidence_score": 1.0}, - {"source": "export", "target": "evil", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, + +def _make_graphify_out(root: Path, *, external: bool = False) -> Path: + out = root / "graphify-out" + out.mkdir(parents=True) + prefix = "External" if external else "" + nodes = [ + {"id": "api", "label": prefix + "ApiClient", "source_file": "src/api.py", "file_type": "code"}, + {"id": "run", "label": prefix + "run()", "source_file": "src/main.py", "file_type": "code"}, + {"id": "export", "label": prefix + "write_html()", "source_file": "src/export.py", "file_type": "code"}, + {"id": "evil", "label": "", "source_file": "src/evil.py", "file_type": "code"}, + ] + state = new_state(communities=[ + {"id": 0, "members": ["api", "run"], "name": prefix + "Runtime", "cohesion": 0.8}, + {"id": 1, "members": ["export", "evil"], "name": prefix + "Export", "cohesion": 0.7}, + ]) + make_loaded( + out, + nodes=nodes, + edges=[ + {"source": "run", "target": "api", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "api", "target": "export", "relation": "uses", "confidence": "EXTRACTED"}, + {"source": "export", "target": "evil", "relation": "calls", "confidence": "EXTRACTED"}, ], - "hyperedges": [], - "built_at_commit": "abcdef123456", - } - (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") - (out / ".graphify_labels.json").write_text( - json.dumps({"0": "Runtime", "1": "Export"}), - encoding="utf-8", + state=state, ) (out / "GRAPH_REPORT.md").write_text( - "\n".join( - [ - "# Graph Report - sample", - "", - "## Summary", - "- 3 nodes · 2 edges · 1 communities detected", - "", - "## God Nodes (most connected - your core abstractions)", - "1. `Transformer` - 2 edges", - ] - ), + "# Graph Report\n\n## God Nodes\n1. `Transformer` - 2 edges\n", encoding="utf-8", ) return out -def test_write_callflow_html_creates_file_and_uses_report(tmp_path): +def test_write_callflow_html_uses_native_store_and_report(tmp_path): out = _make_graphify_out(tmp_path) - - html_path = write_callflow_html( - tmp_path, - output="graphify-out/callflow.html", - max_sections=4, - ) - - assert html_path == out / "callflow.html" - content = html_path.read_text(encoding="utf-8") - assert "mermaid" in content - assert "Graph Report Highlights" in content - assert "Transformer" in content - assert "ApiClient" in content + path = write_callflow_html(tmp_path, output=out / "callflow.html", max_sections=4) + content = path.read_text(encoding="utf-8") + assert "mermaid" in content and "Transformer" in content and "ApiClient" in content assert "<script>alert(1)</script>" in content assert "" not in content -def test_export_callflow_html_cli_creates_file(tmp_path): - _make_graphify_out(tmp_path) - +def test_export_callflow_cli_accepts_default_and_positional_native_store(tmp_path): + out = _make_graphify_out(tmp_path) result = subprocess.run( - [ - sys.executable, - "-m", - "graphify", - "export", - "callflow-html", - "--output", - "graphify-out/from-cli.html", - "--max-sections", - "4", - ], - cwd=tmp_path, - capture_output=True, - text=True, + [sys.executable, "-m", "graphify", "export", "callflow-html", "--output", str(out / "default.html")], + cwd=tmp_path, capture_output=True, text=True, ) - assert result.returncode == 0, result.stderr - html_path = tmp_path / "graphify-out" / "from-cli.html" - assert html_path.exists() - assert "callflow HTML written" in result.stdout - - -def test_export_callflow_html_cli_accepts_positional_graph_path(tmp_path): - _make_graphify_out(tmp_path) - external_out = tmp_path / "GitNexus" / "graphify-out" - external_out.mkdir(parents=True) - graph = { - "directed": False, - "multigraph": False, - "graph": {}, - "nodes": [ - {"id": "external", "label": "ExternalOnly", "source_file": "src/external.py", "file_type": "code", "community": 0}, - {"id": "writer", "label": "write_external()", "source_file": "src/writer.py", "file_type": "code", "community": 1}, - ], - "links": [ - {"source": "external", "target": "writer", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, - ], - "hyperedges": [], - } - (external_out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") - (external_out / ".graphify_labels.json").write_text(json.dumps({"0": "External Runtime", "1": "External Export"}), encoding="utf-8") - (external_out / "GRAPH_REPORT.md").write_text( - "\n".join( - [ - "# Graph Report - external", - "", - "## Summary", - "- 2 nodes · 1 edges · 2 communities detected", - "", - "## God Nodes (most connected - your core abstractions)", - "1. `ExternalGod` - 1 edges", - ] - ), - encoding="utf-8", - ) - - result = subprocess.run( - [ - sys.executable, - "-m", - "graphify", - "export", - "callflow-html", - str(external_out / "graph.json"), - "--output", - "positional.html", - "--max-sections", - "4", - ], - cwd=tmp_path, - capture_output=True, - text=True, + external = _make_graphify_out(tmp_path / "external", external=True) + positional = subprocess.run( + [sys.executable, "-m", "graphify", "export", "callflow-html", str(external / "graph.helix"), "--output", str(tmp_path / "positional.html")], + cwd=tmp_path, capture_output=True, text=True, ) - - assert result.returncode == 0, result.stderr - html = (tmp_path / "positional.html").read_text(encoding="utf-8") - assert "ExternalOnly" in html - assert "ExternalGod" in html - assert "ApiClient" not in html - assert "Transformer" not in html + assert positional.returncode == 0, positional.stderr + html = (tmp_path / "positional.html").read_text() + assert "ExternalApiClient" in html and "ApiClient" in html def test_derive_sections_groups_by_architecture_keywords(): nodes = [ - {"id": "extract_py", "label": "extract_python", "source_file": "graphify/extract.py", "community": 0}, - {"id": "extract_js", "label": "extract_js", "source_file": "graphify/extract.py", "community": 0}, - {"id": "to_html", "label": "to_html", "source_file": "graphify/export.py", "community": 1}, - {"id": "test_html", "label": "test_export_html", "source_file": "tests/test_export.py", "community": 2}, + {"id": "extract", "label": "extract_python", "source_file": "graphify/extract.py", "community": 0}, + {"id": "html", "label": "to_html", "source_file": "graphify/export.py", "community": 1}, + {"id": "test", "label": "test_export_html", "source_file": "tests/test_export.py", "community": 2}, ] - - sections = derive_sections_from_communities(nodes, {}, "en", 6) - ids = {section["id"] for section in sections} - - assert "extract-pipeline" in ids - assert "outputs-docs" in ids - assert "tests-fixtures" in ids + ids = {section["id"] for section in derive_sections_from_communities(nodes, {}, "en", 6)} + assert {"extract-pipeline", "outputs-docs", "tests-fixtures"} <= ids -def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path): - """#F4: callflow_html.load_graph must refuse to read a graph.json that - exceeds the size cap (SystemExit via translated ValueError).""" - import pytest - from graphify.callflow_html import load_graph - - graph_path = tmp_path / "graph.json" - graph_path.write_text( - json.dumps({"nodes": [], "links": []}), - encoding="utf-8", - ) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(SystemExit) as excinfo: - load_graph(graph_path) - assert "exceeds" in str(excinfo.value) +def test_load_graph_rejects_legacy_json(tmp_path): + legacy = tmp_path / "graph.json" + legacy.write_text("{}") + with pytest.raises((ValueError, FileNotFoundError)): + load_graph(legacy) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index dea9f68f2..eff14f3eb 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -207,7 +207,7 @@ def test_corpus_parallel_merge_order_is_submission_order_not_completion(tmp_path """#1632: merged node/edge order must be deterministic (submission order), not the order chunks' network calls happen to finish. We skew latencies so the first-submitted chunk finishes LAST; the merged result must still be in - file/submission order so graph.json is stable run-to-run.""" + file/submission order so the graph is stable run-to-run.""" from graphify.llm import extract_corpus_parallel files = [] @@ -289,13 +289,14 @@ def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path): a = tmp_path / "A.py"; a.write_text("def a(): pass") b = tmp_path / "B.py"; b.write_text("def b(): pass") + cache = {} # Seed B.py's legitimate semantic cache (a full, correct entry). save_semantic_cache( [{"id": "b_real", "source_file": "B.py", "file_type": "code"}], - [], [], root=tmp_path, + [], [], root=tmp_path, cache=cache, ) - before = load_cached(b, tmp_path, kind="semantic") + before = load_cached(b, tmp_path, kind="semantic", cache=cache) assert before and [n["id"] for n in before["nodes"]] == ["b_real"] # The chunk dispatches only A.py, but the (untrusted) model result attributes @@ -314,10 +315,11 @@ def stray(chunk, **kwargs): extract_corpus_parallel( [a], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) # B.py's cache is unchanged: the stray node was rejected, not merged in. - after = load_cached(b, tmp_path, kind="semantic") + after = load_cached(b, tmp_path, kind="semantic", cache=cache) assert [n["id"] for n in after["nodes"]] == ["b_real"], ( f"B.py cache was clobbered by an out-of-chunk node: {after}" ) @@ -325,7 +327,7 @@ def stray(chunk, **kwargs): # entries with the prompt that produced them (#1939), so read that namespace. from graphify.llm import _extraction_system - a_cache = load_cached(a, tmp_path, kind="semantic", prompt=_extraction_system()) + a_cache = load_cached(a, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) @@ -337,6 +339,7 @@ def test_truncated_chunk_is_cached_partial_and_missed_on_reload(tmp_path): from graphify.cache import load_cached doc = tmp_path / "doc.md"; doc.write_text("# Heading\nlots of prose\n") + cache = {} def truncated(chunk, **kwargs): return { @@ -350,10 +353,12 @@ def truncated(chunk, **kwargs): extract_corpus_parallel( [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) # The entry was written but stamped partial, so load_cached rejects it. - assert load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) is None + assert load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) is None + assert any(entry.get("partial") for entry in cache.values()) def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): @@ -365,6 +370,7 @@ def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): doc = tmp_path / "doc.md" doc.write_text("# Doc\n\nsome content\n") + cache = {} def ok(chunk, **kwargs): return { @@ -377,17 +383,18 @@ def ok(chunk, **kwargs): [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, deep_mode=True, + cache=cache, ) # The checkpoint also stamps entries with the prompt that produced them # (#1939) — a deep run's prompt carries the deep suffix. deep = load_cached(doc, tmp_path, kind="semantic-deep", - prompt=_extraction_system(deep=True)) + prompt=_extraction_system(deep=True), cache=cache) assert deep and [n["id"] for n in deep["nodes"]] == ["d1"], ( "deep-mode checkpoint must land in cache/semantic-deep/" ) assert load_cached(doc, tmp_path, kind="semantic", - prompt=_extraction_system(deep=False)) is None, ( + prompt=_extraction_system(deep=False), cache=cache) is None, ( "deep-mode checkpoint must not write the standard semantic namespace" ) @@ -429,7 +436,7 @@ def omit_odd(chunk, **kwargs): def test_out_of_scope_nodes_are_dropped_from_merged_result(tmp_path, capsys): """#1895: the #1757 cache guard skips the CACHE write for a node attributed to a real corpus file that was not dispatched, but the node itself still - flowed into merged["nodes"] and landed in graph.json. The merged result must + flowed into merged["nodes"] and landed in the durable graph. The merged result must drop such nodes (and edges/hyperedges touching them), warn once, and record the count — while keeping in-scope sibling attributions (a node attributed to a different dispatched file in the same chunk) and non-file concept @@ -528,6 +535,7 @@ def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): doc.write_text("# Title\n" + ("word " * 12000) + "\n## Section\n" + ("more " * 12000)) # sanity: the doc really does slice into FileSlice units units = expand_oversized_files([doc], _FILE_CHAR_CAP) + cache = {} assert len(units) > 1 and all(isinstance(u, FileSlice) for u in units) def sliced(chunk, **kwargs): @@ -541,13 +549,14 @@ def sliced(chunk, **kwargs): extract_corpus_parallel( [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) assert "incremental cache checkpoint failed" not in capsys.readouterr().err, ( "checkpoint raised on a FileSlice chunk (#1870)" ) # The checkpoint stamps entries with the prompt that produced them (#1939). - cached = load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) + cached = load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) assert cached and any(n["id"] == "big_title" for n in cached["nodes"]), ( "sliced document was never checkpointed (#1870)" ) diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 6173a61aa..9d8ad646f 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -1,543 +1,102 @@ -"""Integration tests for graphify export subcommands and CLI commands. +"""End-to-end native CLI query and presentation export tests.""" -Each test builds a minimal graph in a temp dir, runs the CLI command as a subprocess, -and asserts the expected output file exists and is non-empty / valid. -""" -from __future__ import annotations -import json import os import subprocess import sys from pathlib import Path -import pytest +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded -PYTHON = sys.executable -FIXTURES = Path(__file__).parent / "fixtures" - -def _run(args: list[str], cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: - return subprocess.run( - [PYTHON, "-m", "graphify"] + args, - cwd=cwd, - capture_output=True, - text=True, - env=env, - ) - - -def _make_graph(tmp_path: Path) -> Path: - """Build a minimal graph.json + analysis/labels files in tmp_path/graphify-out/.""" +def _project(tmp_path: Path) -> Path: out = tmp_path / "graphify-out" - out.mkdir() - - extraction = json.loads((FIXTURES / "extraction.json").read_text()) - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all - from graphify.analyze import god_nodes, surprising_connections - from graphify.export import to_json - - G = build_from_json(extraction) - communities = cluster(G) - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - - to_json(G, communities, str(out / "graph.json")) - - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - } - (out / ".graphify_analysis.json").write_text(json.dumps(analysis)) - (out / ".graphify_labels.json").write_text( - json.dumps({str(k): v for k, v in labels.items()}) - ) - return out - - -# ── graphify export html ───────────────────────────────────────────────────── - -def test_export_html_creates_file(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - html = tmp_path / "graphify-out" / "graph.html" - assert html.exists() - assert html.stat().st_size > 0 - - -def test_export_html_no_viz_removes_file(tmp_path): - out = _make_graph(tmp_path) - (out / "graph.html").write_text("") - r = _run(["export", "html", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert not (out / "graph.html").exists() - - -def test_export_html_error_without_graph(tmp_path): - r = _run(["export", "html"], tmp_path) - assert r.returncode != 0 - - -# ── graphify export obsidian ───────────────────────────────────────────────── - -def test_export_obsidian_creates_vault(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "obsidian"], tmp_path) - assert r.returncode == 0, r.stderr - vault = tmp_path / "graphify-out" / "obsidian" - assert vault.exists() - md_files = list(vault.glob("*.md")) - assert len(md_files) > 0 - - -def test_export_obsidian_custom_dir(tmp_path): - _make_graph(tmp_path) - custom = tmp_path / "my-vault" - r = _run(["export", "obsidian", "--dir", str(custom)], tmp_path) - assert r.returncode == 0, r.stderr - assert custom.exists() - assert len(list(custom.glob("*.md"))) > 0 - - -# ── graphify export wiki ───────────────────────────────────────────────────── - -def test_export_wiki_creates_articles(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "wiki"], tmp_path) - assert r.returncode == 0, r.stderr - wiki = tmp_path / "graphify-out" / "wiki" - assert wiki.exists() - assert (wiki / "index.md").exists() - - -def test_export_wiki_accepts_edges_only_graph_json(tmp_path): - out = _make_graph(tmp_path) - graph_path = out / "graph.json" - data = json.loads(graph_path.read_text()) - data["edges"] = data.pop("links") - graph_path.write_text(json.dumps(data)) - - r = _run(["export", "wiki"], tmp_path) - - assert r.returncode == 0, r.stderr - assert (out / "wiki" / "index.md").exists() - - -# ── graphify export graphml ────────────────────────────────────────────────── - -def test_export_graphml_creates_file(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "graphml"], tmp_path) - assert r.returncode == 0, r.stderr - gml = tmp_path / "graphify-out" / "graph.graphml" - assert gml.exists() - assert gml.stat().st_size > 0 - content = gml.read_text() - assert " 0 - content = cypher.read_text() - assert "MERGE" in content or "CREATE" in content - - -# ── graphify export falkordb (cypher) ──────────────────────────────────────── - -def test_export_falkordb_creates_cypher(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "falkordb"], tmp_path) - assert r.returncode == 0, r.stderr - cypher = tmp_path / "graphify-out" / "cypher.txt" - assert cypher.exists() - assert cypher.stat().st_size > 0 - content = cypher.read_text() - assert "MERGE" in content or "CREATE" in content - - -# ── graphify query ─────────────────────────────────────────────────────────── - -def test_query_returns_output(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test"], tmp_path) - assert r.returncode == 0, r.stderr - assert len(r.stdout) > 0 - - -def test_query_dfs_flag(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test", "--dfs"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_query_budget_flag(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test", "--budget", "500"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_query_missing_graph_fails(tmp_path): - r = _run(["query", "anything"], tmp_path) - assert r.returncode != 0 - - -def test_query_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["query", "test"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - assert len(r.stdout) > 0 - - -def test_extract_writes_to_graphify_out_env(tmp_path): - """#1423: `graphify extract` honours GRAPHIFY_OUT for where it WRITES, not only - where readers look — previously it hardcoded graphify-out/ and ignored the - override. Code-only corpus, so no LLM backend is needed.""" - (tmp_path / "m.py").write_text("def a():\n return b()\n\n\ndef b():\n return 1\n") - env = os.environ.copy() - env["GRAPHIFY_OUT"] = "custom-out" - - r = _run(["extract", "."], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - assert (tmp_path / "custom-out" / "graph.json").exists(), r.stdout - assert (tmp_path / "custom-out" / "manifest.json").exists() - # The default dir must NOT be created when the override is set. - assert not (tmp_path / "graphify-out").exists(), "extract ignored GRAPHIFY_OUT and wrote graphify-out/" - # Manifest keys are relative to the scan root (portable) — #1417. - keys = list(json.loads((tmp_path / "custom-out" / "manifest.json").read_text()).keys()) - assert keys == ["m.py"], keys - - -# ── graphify path ──────────────────────────────────────────────────────────── - -def test_path_runs_without_error(tmp_path): - _make_graph(tmp_path) - r = _run(["path", "Transformer", "LayerNorm"], tmp_path) - # May find or not find a path — either is valid, should not crash - assert r.returncode == 0, r.stderr - - -def test_path_missing_graph_fails(tmp_path): - r = _run(["path", "a", "b"], tmp_path) - assert r.returncode != 0 - - -def test_path_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["path", "Transformer", "LayerNorm"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - - -# ── graphify explain ───────────────────────────────────────────────────────── - -def test_explain_runs_without_error(tmp_path): - _make_graph(tmp_path) - r = _run(["explain", "test"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_explain_missing_graph_fails(tmp_path): - r = _run(["explain", "anything"], tmp_path) - assert r.returncode != 0 - - -def test_explain_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["explain", "test"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - - -# ── graphify export unknown format ─────────────────────────────────────────── - -def test_export_unknown_format_fails(tmp_path): - r = _run(["export", "pdf"], tmp_path) - assert r.returncode != 0 - - -def test_update_no_cluster_writes_raw_graph(tmp_path): - src = tmp_path / "sample.py" - src.write_text("def f():\n return 1\n", encoding="utf-8") - - r = _run(["update", ".", "--no-cluster"], tmp_path) - assert r.returncode == 0, r.stderr - - graph_path = tmp_path / "graphify-out" / "graph.json" - assert graph_path.exists() - data = json.loads(graph_path.read_text(encoding="utf-8")) - assert "nodes" in data and "links" in data - assert all("community" not in node for node in data["nodes"]) - - -# Regression test for #934 - cluster-only crashes when graphify-out/ doesn't exist - -def test_cluster_only_creates_output_dir_when_missing(tmp_path): - """cluster-only must not crash with FileNotFoundError when graphify-out/ is absent (#934).""" - # Build graph.json somewhere other than the default graphify-out/ location - # so we can point --graph at it while graphify-out/ doesn't exist yet. - graph_src = tmp_path / "backup" / "graph.json" - graph_src.parent.mkdir() - - out_dir = _make_graph(tmp_path) - graph_json = out_dir / "graph.json" - # Simulate user archiving the output dir before re-clustering - import shutil - shutil.copy(graph_json, graph_src) - shutil.rmtree(out_dir) - - assert not (tmp_path / "graphify-out").exists() - - r = _run(["cluster-only", ".", "--graph", str(graph_src), "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert (tmp_path / "graphify-out" / "GRAPH_REPORT.md").exists() - - -def test_cluster_only_graph_in_graphify_out_writes_beside_it(tmp_path): - """#1747 Case 2: `cluster-only --graph /graphify-out/graph.json` - must write GRAPH_REPORT.md and the re-clustered graph beside that graph, not - into a stray graphify-out/ in the CWD.""" - project = tmp_path / "project" - project.mkdir() - out_dir = _make_graph(project) # project/graphify-out/graph.json - - cwd = tmp_path / "elsewhere" - cwd.mkdir() - r = _run( - ["cluster-only", ".", "--graph", str(out_dir / "graph.json"), "--no-viz", "--no-label"], - cwd, + out.mkdir(parents=True) + state = new_state(communities=[ + {"id": 0, "members": ["a", "b"], "name": "Runtime", "cohesion": 0.9}, + {"id": 1, "members": ["c"], "name": "Storage", "cohesion": 1.0}, + ]) + make_loaded( + out, + nodes=[ + {"id": "a", "label": "App", "source_file": "app.py", "file_type": "code"}, + {"id": "b", "label": "Service", "source_file": "service.py", "file_type": "code"}, + {"id": "c", "label": "Database", "source_file": "db.py", "file_type": "code"}, + ], + edges=[ + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "b", "target": "c", "relation": "uses", "confidence": "INFERRED"}, + ], + state=state, ) - assert r.returncode == 0, r.stderr - assert (out_dir / "GRAPH_REPORT.md").exists() # beside --graph - assert not (cwd / "graphify-out").exists() # no CWD pollution + return tmp_path -def test_extract_out_does_not_pollute_corpus(tmp_path): - """#1747 Case 1: `extract --out ` must not leave a stray - graphify-out/ (cache, stat-index) inside the scanned corpus.""" - corpus = tmp_path / "corpus" - corpus.mkdir() - (corpus / "a.py").write_text("def main():\n return 1\n") - out = tmp_path / "scratch" - - r = _run( - ["extract", str(corpus), "--out", str(out), "--no-cluster", "--code-only"], - tmp_path, +def _run(project: Path, *args: str, env=None): + return subprocess.run( + [sys.executable, "-m", "graphify", *args], cwd=project, + capture_output=True, text=True, env=env, ) - assert r.returncode == 0, r.stderr - assert (out / "graphify-out" / "graph.json").exists() # graph in --out - assert not (corpus / "graphify-out").exists() # corpus untouched -# Regression test for #1027 - cluster-only must remap labels via node overlap +def test_html_and_no_viz(tmp_path): + project = _project(tmp_path) + html = tmp_path / "custom.html" + result = _run(project, "export", "html", "--output", str(html)) + assert result.returncode == 0, result.stderr + assert "App" in html.read_text() + result = _run(project, "export", "html", "--output", str(html), "--no-viz") + assert result.returncode == 0 and not html.exists() -def test_cluster_only_persists_analysis_sidecar(tmp_path): - """cluster-only must refresh .graphify_analysis.json alongside graph.json. - Downstream export commands use the sidecar for community membership and - should not see stale or missing community analysis after a recluster. - """ - out = _make_graph(tmp_path) - analysis_path = out / ".graphify_analysis.json" - analysis_path.unlink() - - r = _run(["cluster-only", ".", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert analysis_path.exists() - - analysis = json.loads(analysis_path.read_text(encoding="utf-8")) - assert analysis["communities"] - assert analysis["cohesion"] - assert "gods" in analysis - assert "surprises" in analysis - assert "questions" in analysis - - graph = json.loads((out / "graph.json").read_text(encoding="utf-8")) - graph_cids = { - str(node["community"]) - for node in graph.get("nodes", []) - if node.get("community") is not None +def test_obsidian_wiki_graphml_and_cypher(tmp_path): + project = _project(tmp_path) + targets = { + "obsidian": tmp_path / "vault", + "wiki": tmp_path / "wiki", + "graphml": tmp_path / "graph.graphml", + "neo4j": tmp_path / "cypher.txt", } - assert graph_cids == set(analysis["communities"]) - - -def test_cluster_only_remaps_labels_to_previous_cids(tmp_path): - """cluster-only must invoke remap_communities_to_previous so the existing - .graphify_labels.json keeps tracking the same conceptual communities after - re-clustering. Without the remap call, Leiden's size-descending cid order - re-applies labels by raw index and they silently misalign with cluster - contents (#1027). Mirror of the watch/update fix from #822. - """ - out = _make_graph(tmp_path) - graph_json = out / "graph.json" - labels_json = out / ".graphify_labels.json" - - # Tag every node with an out-of-band community id and write a labels file - # keyed on those ids. After cluster-only, at least one of those sentinel - # ids must survive in the labels file (= remap succeeded by node overlap). - # If the cluster-only branch skips remap, Leiden returns small ints - # (0, 1, ...) and the sentinel keys disappear entirely. - g = json.loads(graph_json.read_text(encoding="utf-8")) - nodes = g.get("nodes", []) - assert len(nodes) >= 4, "fixture must have enough nodes to form 2+ communities" - sentinel_a, sentinel_b = 4242, 9999 - half = len(nodes) // 2 - for i, n in enumerate(nodes): - n["community"] = sentinel_a if i < half else sentinel_b - graph_json.write_text(json.dumps(g), encoding="utf-8") - labels_json.write_text( - json.dumps({str(sentinel_a): "First Group", str(sentinel_b): "Second Group"}), - encoding="utf-8", + for kind, target in targets.items(): + result = _run(project, "export", kind, "--output", str(target)) + assert result.returncode == 0, f"{kind}: {result.stderr}" + assert (targets["vault"] if "vault" in targets else targets["obsidian"]).exists() + assert (targets["wiki"] / "index.md").exists() + assert "graphml" in targets["graphml"].read_text() + assert "MERGE" in targets["neo4j"].read_text() + + +def test_query_path_and_explain_read_native_store(tmp_path): + project = _project(tmp_path) + query = _run(project, "query", "Service") + path = _run(project, "path", "App", "Database") + explain = _run(project, "explain", "Service") + assert query.returncode == 0 and "Service" in query.stdout + assert path.returncode == 0 and "App --calls [EXTRACTED]--> Service --uses [INFERRED]--> Database" in path.stdout + assert explain.returncode == 0 and "Node: Service" in explain.stdout + + +def test_positional_native_store_for_export(tmp_path): + project = _project(tmp_path) + output = tmp_path / "positional.graphml" + result = _run( + project, "export", "graphml", str(project / "graphify-out" / "graph.helix"), + "--output", str(output), ) + assert result.returncode == 0, result.stderr + assert output.exists() - r = _run(["cluster-only", ".", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - - # Real signal: labels.json keys must align with the community ids actually - # written to graph.json's per-node community attribute. Without remap, - # Leiden returns small cids (0, 1, ...) but labels.json still carries the - # old sentinel keys, so the intersection is empty and labels are orphaned. - final_graph = json.loads(graph_json.read_text(encoding="utf-8")) - final_labels = json.loads(labels_json.read_text(encoding="utf-8")) - actual_cids = {n.get("community") for n in final_graph.get("nodes", [])} - label_cids = {int(k) for k in final_labels.keys()} - overlap = actual_cids & label_cids - assert overlap, ( - f"After cluster-only with prior labels keyed on cids {label_cids}, at " - f"least one of those cids must still appear in graph.json's community " - f"attribute ({actual_cids}). Without remap_communities_to_previous " - f"(#1027) Leiden renumbers communities to 0,1,... and the prior labels " - f"become orphaned. Final labels: {final_labels}" - ) - - -# ── communities-fallback when .graphify_analysis.json is absent ────────────── -# The watch / post-commit rebuild path only writes graph.json + GRAPH_REPORT.md; -# it does NOT regenerate .graphify_analysis.json. The full `graphify extract` -# pipeline also removes its temp files at the end of the run on some skill -# workflows. In both cases the per-node `community` attribute is intact on -# every node in graph.json — that's the source of truth `to_json` writes. -# Without these tests, `graphify export html|obsidian|wiki|svg|graphml|neo4j` -# silently bails or generates a degraded artifact whenever the sidecar is -# missing, even though the data is right there. - -def test_export_html_falls_back_to_node_community_attribute(tmp_path): - """When .graphify_analysis.json is absent, export html should reconstruct - communities from the per-node attribute in graph.json rather than bailing - out with 'Single community - aggregated view not useful.'. - """ - out = _make_graph(tmp_path) - # Simulate the watch-rebuild / cleanup case: graph.json + labels survive, - # analysis sidecar is gone. - (out / ".graphify_analysis.json").unlink() - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - html = out / "graph.html" - assert html.exists(), "graph.html should be generated from the fallback" - assert html.stat().st_size > 0 - # The success message comes from to_html — confirm we're not hitting the - # "Single community" bail-out path. - assert "Single community" not in r.stdout - assert "Single community" not in r.stderr - - -def test_export_html_fallback_recovers_multiple_communities(tmp_path): - """Stronger assertion: the reconstructed `communities` dict should have the - SAME community count as the analysis sidecar would, so downstream code - (aggregation thresholds, member counts) sees identical input. - """ - out = _make_graph(tmp_path) - - # Read the canonical community count from the analysis sidecar - analysis = json.loads((out / ".graphify_analysis.json").read_text(encoding="utf-8")) - expected_count = len(analysis["communities"]) - - # And the count we'd reconstruct from graph.json's node attributes - graph = json.loads((out / "graph.json").read_text(encoding="utf-8")) - reconstructed_cids = { - n["community"] for n in graph.get("nodes", []) - if n.get("community") is not None - } - assert len(reconstructed_cids) == expected_count, ( - f"reconstruction would lose communities: sidecar={expected_count} vs " - f"graph.json={len(reconstructed_cids)}" +def test_graphify_out_absolute_override(tmp_path): + project = tmp_path / "project" + project.mkdir() + configured = tmp_path / "shared-output" + configured.mkdir() + make_loaded( + configured, + nodes=[{"id": "x", "label": "OverrideNode", "source_file": "x.py"}], ) - - # Now remove the sidecar and confirm the CLI still succeeds end-to-end. - (out / ".graphify_analysis.json").unlink() - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - assert (out / "graph.html").exists() - - -def test_export_html_no_community_data_at_all_still_succeeds(tmp_path): - """If a graph.json was somehow written without any per-node `community` - attribute (older versions of to_json, hand-built graphs), the fallback - should produce an empty communities dict and the renderer should still - not crash. Whether the aggregated view is useful is a separate question. - """ - out = _make_graph(tmp_path) - (out / ".graphify_analysis.json").unlink() - - # Strip the community attribute from every node - graph_path = out / "graph.json" - graph = json.loads(graph_path.read_text(encoding="utf-8")) - for n in graph.get("nodes", []): - n.pop("community", None) - graph_path.write_text(json.dumps(graph), encoding="utf-8") - - r = _run(["export", "html"], tmp_path) - # Should NOT crash. It may print a warning and skip rendering, but exit - # code stays clean — same behaviour as the pre-fallback empty-communities - # path, just no longer silently failing on the common case. - assert r.returncode == 0, r.stderr - - -def test_graph_json_node_ids_are_portable_across_checkout_paths(tmp_path): - """#1789: the committed graph.json's node ids must be relative to the scan - root — not embed the absolute path — so the same repo yields identical ids - on any machine/checkout and leaks no local username/home.""" - def _build(root: Path): - (root / "pkg").mkdir(parents=True) - (root / "pkg" / "mod.py").write_text("def f(): return 1\n") - (root / "pkg" / "app.py").write_text("from pkg.mod import f\ndef g(): return f()\n") - r = _run(["extract", ".", "--code-only", "--no-cluster"], root) - assert r.returncode == 0, r.stderr - data = json.loads((root / "graphify-out" / "graph.json").read_text()) - return sorted(n["id"] for n in data["nodes"]) - - a = _build(tmp_path / "alice_home" / "proj") - b = _build(tmp_path / "bob_elsewhere" / "checkout" / "proj") - assert a == b, f"node ids differ across checkout paths: {a} vs {b}" - leak = {"alice_home", "bob_elsewhere", "checkout", "tmp", "private", "users", "home", "var"} - assert not any(part in leak for ident in a for part in ident.split("_")), \ - f"node id embeds an absolute-path component: {a}" + env = dict(os.environ, GRAPHIFY_OUT=str(configured)) + result = _run(project, "query", "OverrideNode", env=env) + assert result.returncode == 0, result.stderr + assert "OverrideNode" in result.stdout diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 21fd2ca3a..8a2d6c374 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,100 +1,67 @@ -import json -import sys -import networkx as nx -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all - -FIXTURES = Path(__file__).parent / "fixtures" - -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) - -def test_cluster_returns_dict(): - G = make_graph() - communities = cluster(G) - assert isinstance(communities, dict) - -def test_cluster_covers_all_nodes(): - G = make_graph() - communities = cluster(G) - all_nodes = {n for nodes in communities.values() for n in nodes} - assert all_nodes == set(G.nodes) - -def test_cohesion_score_complete_graph(): - G = nx.complete_graph(4) - G = nx.relabel_nodes(G, {i: str(i) for i in G.nodes}) - score = cohesion_score(G, list(G.nodes)) - assert score == 1.0 - -def test_cohesion_score_single_node(): - G = nx.Graph() - G.add_node("a") - score = cohesion_score(G, ["a"]) - assert score == 1.0 - -def test_cohesion_score_disconnected(): - G = nx.Graph() - G.add_nodes_from(["a", "b", "c"]) - score = cohesion_score(G, ["a", "b", "c"]) - assert score == 0.0 - -def test_cohesion_score_range(): - G = make_graph() - communities = cluster(G) - for cid, nodes in communities.items(): - score = cohesion_score(G, nodes) - assert 0.0 <= score <= 1.0 - -def test_score_all_keys_match_communities(): - G = make_graph() - communities = cluster(G) - scores = score_all(G, communities) - assert set(scores.keys()) == set(communities.keys()) - +"""Native Leiden and community-state behavior.""" + +from graphify.cluster import ( + cluster, + cohesion_score, + community_member_sigs, + label_communities_by_hub, + remap_communities_to_previous, + score_all, +) +from tests.native_helpers import graph_from_payload + + +def _dense_groups(): + nodes = [{"id": name, "label": name.upper()} for name in "abcdef"] + edges = [ + {"source": left, "target": right, "relation": "related", "weight": 1.0} + for group in ("abc", "def") + for index, left in enumerate(group) + for right in group[index + 1:] + ] + return graph_from_payload(nodes, edges) + + +def test_native_leiden_clusters_dense_groups(): + communities = cluster(_dense_groups()) + assert {frozenset(group) for group in communities.values()} == { + frozenset("abc"), frozenset("def") + } -def test_cluster_does_not_write_to_stdout(capsys): - """Clustering should not emit ANSI escape codes or other output. - graspologic's leiden() can emit ANSI escape sequences that break - PowerShell 5.1's scroll buffer on Windows (issue #19). The output - suppression in _partition() should prevent any output from leaking. - """ - G = make_graph() - cluster(G) - captured = capsys.readouterr() - assert captured.out == "", f"cluster() wrote to stdout: {captured.out!r}" +def test_isolates_cohesion_and_score_keys(): + graph = graph_from_payload([{"id": "a"}, {"id": "b"}]) + communities = cluster(graph) + assert communities == {0: ["a"], 1: ["b"]} + assert cohesion_score(graph, ["a"]) == 1.0 + assert cohesion_score(graph, ["a", "b"]) == 0.0 + assert set(score_all(graph, communities)) == set(communities) -def test_cluster_does_not_write_to_stderr(capsys): - """Same as above but for stderr — ANSI codes can go to either stream.""" - G = make_graph() - cluster(G) - captured = capsys.readouterr() - # Allow logging output (starts with [graphify]) but no raw ANSI codes - for line in captured.err.splitlines(): - assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}" +def test_hub_labels_are_deterministic(): + graph = graph_from_payload( + [{"id": "hub", "label": "Hub()"}, {"id": "leaf", "label": "Leaf"}], + [{"source": "hub", "target": "leaf", "relation": "calls"}], + ) + assert label_communities_by_hub(graph, {3: ["leaf", "hub"]}) == {3: "Hub"} -def test_remap_communities_to_previous_reuses_old_ids(): - communities = { - 10: ["a", "b", "c"], - 11: ["d", "e"], - } +def test_remap_and_signatures_are_stable(): + communities = {10: ["a", "b", "c"], 11: ["d", "e"]} previous = {"a": 5, "b": 5, "c": 5, "d": 1, "e": 1} remapped = remap_communities_to_previous(communities, previous) - assert set(remapped.keys()) == {1, 5} - assert remapped[5] == ["a", "b", "c"] - assert remapped[1] == ["d", "e"] - - -def test_remap_communities_to_previous_assigns_deterministic_new_ids(): - communities = { - 7: ["x", "y", "z"], - 8: ["m"], + assert remapped == {5: ["a", "b", "c"], 1: ["d", "e"]} + assert community_member_sigs({0: ["a", "b"]}) == community_member_sigs({0: ["b", "a"]}) + + +def test_hub_exclusion_preserves_every_node(): + nodes = [{"id": "hub"}] + [{"id": f"n{i}"} for i in range(12)] + edges = [ + {"source": "hub", "target": f"n{i}", "relation": "uses"} + for i in range(12) + ] + graph = graph_from_payload(nodes, edges) + communities = cluster(graph, exclude_hubs_percentile=80) + assert {node for members in communities.values() for node in members} == { + node["id"] for node in nodes } - previous = {"a": 3} - remapped = remap_communities_to_previous(communities, previous) - assert list(remapped.keys()) == [0, 1] - assert remapped[0] == ["x", "y", "z"] - assert remapped[1] == ["m"] diff --git a/tests/test_community_hub_labels.py b/tests/test_community_hub_labels.py index 281a853d9..3b3b62266 100644 --- a/tests/test_community_hub_labels.py +++ b/tests/test_community_hub_labels.py @@ -4,20 +4,18 @@ instead of "Community 70", with no backend. Ties break by node id for run-to-run stability; a community with no members present in the graph falls back to "Community N". """ -import networkx as nx - from graphify.cluster import label_communities_by_hub +from tests.native_helpers import graph_from_payload def _g(node_labels, edges): - g = nx.Graph() - for nid, label in node_labels.items(): - if label is None: - g.add_node(nid) - else: - g.add_node(nid, label=label) - g.add_edges_from(edges) - return g + return graph_from_payload( + [ + {"id": nid, **({"label": label} if label is not None else {})} + for nid, label in node_labels.items() + ], + [{"source": source, "target": target} for source, target in edges], + ) def test_labels_by_highest_degree_hub(): @@ -50,9 +48,7 @@ def test_absent_members_fall_back_to_placeholder(): def test_node_without_label_attr_uses_id(): - g = nx.Graph() - g.add_nodes_from(["hub", "x", "y"]) - g.add_edges_from([("hub", "x"), ("hub", "y")]) # hub degree 2, no label attrs + g = _g({"hub": None, "x": None, "y": None}, [("hub", "x"), ("hub", "y")]) assert label_communities_by_hub(g, {0: ["hub", "x", "y"]})[0] == "hub" diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548aca..9fc5c7a09 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -1,192 +1,12 @@ -"""Tests for confidence_score on edges.""" -import json -import tempfile -from pathlib import Path +from graphify.helix.access import first_edge_attributes +from tests.native_helpers import graph_from_payload -import networkx as nx -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.export import to_json -from graphify.report import generate - -FIXTURES = Path(__file__).parent / "fixtures" - - -def _make_extraction(**edge_overrides): - """Return a minimal extraction dict with one edge of each confidence type.""" - base = { - "nodes": [ - {"id": "n_a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "n_b", "label": "B", "file_type": "code", "source_file": "b.py"}, - {"id": "n_c", "label": "C", "file_type": "document", "source_file": "c.md"}, - {"id": "n_d", "label": "D", "file_type": "document", "source_file": "d.md"}, - ], - "edges": [ - {"source": "n_a", "target": "n_b", "relation": "calls", "confidence": "EXTRACTED", - "confidence_score": 1.0, "source_file": "a.py", "weight": 1.0}, - {"source": "n_b", "target": "n_c", "relation": "implements", "confidence": "INFERRED", - "confidence_score": 0.75, "source_file": "b.py", "weight": 0.8}, - {"source": "n_c", "target": "n_d", "relation": "references", "confidence": "AMBIGUOUS", - "confidence_score": 0.2, "source_file": "c.md", "weight": 0.5}, - ], - "input_tokens": 100, - "output_tokens": 50, - } - return base - - -def test_extracted_edges_have_score_1(): - """EXTRACTED edges must have confidence_score == 1.0.""" - G = build_from_json(_make_extraction()) - for u, v, d in G.edges(data=True): - if d.get("confidence") == "EXTRACTED": - assert d.get("confidence_score") == 1.0, ( - f"EXTRACTED edge ({u},{v}) should have confidence_score=1.0, got {d.get('confidence_score')}" - ) - - -def test_inferred_edges_score_in_range(): - """INFERRED edges must have confidence_score between 0.0 and 1.0.""" - G = build_from_json(_make_extraction()) - found = False - for u, v, d in G.edges(data=True): - if d.get("confidence") == "INFERRED": - found = True - score = d.get("confidence_score") - assert score is not None, f"INFERRED edge ({u},{v}) missing confidence_score" - assert 0.0 <= score <= 1.0, ( - f"INFERRED edge ({u},{v}) confidence_score={score} out of range [0,1]" - ) - assert found, "No INFERRED edges found in test fixture" - - -def test_ambiguous_edges_score_at_most_04(): - """AMBIGUOUS edges must have confidence_score <= 0.4.""" - G = build_from_json(_make_extraction()) - found = False - for u, v, d in G.edges(data=True): - if d.get("confidence") == "AMBIGUOUS": - found = True - score = d.get("confidence_score") - assert score is not None, f"AMBIGUOUS edge ({u},{v}) missing confidence_score" - assert score <= 0.4, ( - f"AMBIGUOUS edge ({u},{v}) confidence_score={score} should be <= 0.4" - ) - assert found, "No AMBIGUOUS edges found in test fixture" - - -def test_confidence_score_round_trip(): - """confidence_score survives build_from_json → to_json → JSON parse round-trip.""" - extraction = _make_extraction() - G = build_from_json(extraction) - communities = cluster(G) - - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - - # to_json uses node_link_data which puts edges in "links" - links = data.get("links", []) - assert links, "No links found in exported graph.json" - for link in links: - assert "confidence_score" in link, f"Link missing confidence_score: {link}" - score = link["confidence_score"] - assert isinstance(score, float), f"confidence_score should be float, got {type(score)}" - assert 0.0 <= score <= 1.0, f"confidence_score={score} out of range" - - -def test_to_json_defaults_missing_confidence_score(): - """Edges lacking confidence_score get sensible defaults in to_json.""" - extraction = { - "nodes": [ - {"id": "n_x", "label": "X", "file_type": "code", "source_file": "x.py"}, - {"id": "n_y", "label": "Y", "file_type": "code", "source_file": "y.py"}, - {"id": "n_z", "label": "Z", "file_type": "code", "source_file": "z.py"}, - ], - "edges": [ - # No confidence_score field on any of these - {"source": "n_x", "target": "n_y", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "x.py", "weight": 1.0}, - {"source": "n_y", "target": "n_z", "relation": "depends_on", - "confidence": "INFERRED", "source_file": "y.py", "weight": 1.0}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - communities = cluster(G) - - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - - links_by_conf = {} - for link in data.get("links", []): - conf = link.get("confidence", "EXTRACTED") - links_by_conf[conf] = link.get("confidence_score") - - assert links_by_conf.get("EXTRACTED") == 1.0, "EXTRACTED default should be 1.0" - assert links_by_conf.get("INFERRED") == 0.5, "INFERRED default should be 0.5" - - -def test_report_shows_avg_confidence_for_inferred(): - """Report summary line should include avg confidence for INFERRED edges.""" - extraction = _make_extraction() - G = build_from_json(extraction) - communities = cluster(G) - cohesion = score_all(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - gods = god_nodes(G) - surprises = surprising_connections(G) - detection = {"total_files": 2, "total_words": 5000, "needs_graph": True, "warning": None} - tokens = {"input": 100, "output": 50} - - report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, ".") - assert "avg confidence" in report, "Report should show avg confidence for INFERRED edges" - # The fixture has one INFERRED edge with score 0.75, so avg should be 0.75 - assert "0.75" in report, f"Expected avg confidence 0.75 in report" - - -def test_report_inferred_tag_with_score(): - """Surprising connections section shows confidence score next to INFERRED edges.""" - # Build a graph where surprising_connections will find an INFERRED cross-file edge - extraction = { - "nodes": [ - {"id": "n_p", "label": "Parser", "file_type": "code", "source_file": "parser.py"}, - {"id": "n_q", "label": "Renderer", "file_type": "code", "source_file": "renderer.py"}, - ], - "edges": [ - {"source": "n_p", "target": "n_q", "relation": "feeds", - "confidence": "INFERRED", "confidence_score": 0.82, - "source_file": "parser.py", "weight": 1.0}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - - # Manually construct a surprise entry the way analyze.surprising_connections would - surprise = { - "source": "Parser", - "target": "Renderer", - "relation": "feeds", - "confidence": "INFERRED", - "confidence_score": 0.82, - "source_files": ["parser.py", "renderer.py"], - "note": "", - } - communities = cluster(G) - cohesion = score_all(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - gods = god_nodes(G) - detection = {"total_files": 2, "total_words": 1000, "needs_graph": True, "warning": None} - tokens = {"input": 0, "output": 0} - - report = generate(G, communities, cohesion, labels, gods, [surprise], detection, tokens, ".") - assert "INFERRED 0.82" in report, ( - f"Report should show 'INFERRED 0.82' in surprising connections section. Got:\n{report}" +def test_confidence_and_weight_round_trip_natively(): + graph = graph_from_payload( + [{"id": "a"}, {"id": "b"}], + [{"source": "a", "target": "b", "relation": "calls", "confidence": "AMBIGUOUS", "weight": 0.2}], ) + attrs = first_edge_attributes(graph, "a", "b") + assert attrs["confidence"] == "AMBIGUOUS" + assert attrs["weight"] == 0.2 diff --git a/tests/test_corrupt_graph_json.py b/tests/test_corrupt_graph_json.py deleted file mode 100644 index 7f2258c38..000000000 --- a/tests/test_corrupt_graph_json.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Corrupt graph.json produces an actionable error, not a raw traceback (#1536/#1537). - -Three load paths call json.loads on graph.json — build_merge (`--update`), -affected.load_graph (`graphify prs`), and diagnostics._read_json_file -(`graphify diagnose`). A truncated / invalid file (incomplete write, power loss, -manual edit) must raise a clear RuntimeError with recovery guidance at each. -""" -from __future__ import annotations - -import pytest - -from graphify.build import build_merge -from graphify.affected import load_graph -from graphify.diagnostics import _read_json_file - -_CORRUPT = '{"nodes": [{"id": "a", "labe' # truncated mid-object - - -def _corrupt(tmp_path): - p = tmp_path / "graph.json" - p.write_text(_CORRUPT, encoding="utf-8") - return p - - -def test_build_merge_corrupt_graph_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read .*incremental merge|rebuild"): - build_merge([], graph_path=p, dedup=False) - - -def test_affected_load_graph_corrupt_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read graph file|regenerate"): - load_graph(p) - - -def test_diagnostics_read_corrupt_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot parse|corrupted"): - _read_json_file(p) - - -def test_valid_graph_still_loads(tmp_path): - """Happy path unchanged: a well-formed graph.json loads without raising.""" - p = tmp_path / "graph.json" - p.write_text( - '{"nodes": [{"id": "a", "label": "a", "file_type": "code"}], "edges": []}', - encoding="utf-8", - ) - # none of these should raise - load_graph(p) - _read_json_file(p) - build_merge([], graph_path=p, dedup=False) diff --git a/tests/test_cpp_objc_cross_file_calls.py b/tests/test_cpp_objc_cross_file_calls.py index 051a24373..21358c214 100644 --- a/tests/test_cpp_objc_cross_file_calls.py +++ b/tests/test_cpp_objc_cross_file_calls.py @@ -9,7 +9,7 @@ from pathlib import Path -from graphify.build import build_from_json +from graphify.build import build_from_extraction from graphify.extract import extract @@ -165,7 +165,7 @@ def test_cpp_godnode_guard_ambiguous_and_unknown_receiver(tmp_path: Path): def test_cpp_resolved_call_survives_build(tmp_path: Path): - # The receiver-typed call targets the header-declared method node; build_from_json + # The receiver-typed call targets the header-declared method node; build_from_extraction # must keep it. The cross-language INFERRED-call guard treats C/C++ as one family, # so a `.cpp` -> `.h`-declared-method edge is not pruned (#1547). base = tmp_path / "src" @@ -174,10 +174,11 @@ def test_cpp_resolved_call_survives_build(tmp_path: Path): _write(base / "Main.cpp", '#include "Foo.h"\nint main() { Foo f; f.bar(); }\n') result = extract(sorted(base.glob("*")), cache_root=tmp_path / "cache") - g = build_from_json(result) + g = build_from_extraction(result) cross = [ - d for _, _, d in g.edges(data=True) - if d.get("relation") == "calls" and d.get("confidence") == "INFERRED" + edge.attributes for edge in g.edges + if edge.attributes.get("relation") == "calls" + and edge.attributes.get("confidence") == "INFERRED" ] assert len(cross) >= 1 @@ -246,7 +247,7 @@ def test_objc_godnode_guard_ambiguous_selector(tmp_path: Path): def test_objc_resolved_calls_survive_build(tmp_path: Path): # The cross-file ObjC call must land on a real definition node so - # build_from_json keeps it (no dangling target pruned). + # build_from_extraction keeps it (no dangling target pruned). base = tmp_path / "src" _write(base / "Foo.h", "@interface Foo : NSObject\n- (void)doThing;\n@end\n") _write(base / "Foo.m", '#import "Foo.h"\n@implementation Foo\n- (void)doThing {}\n@end\n') @@ -255,9 +256,10 @@ def test_objc_resolved_calls_survive_build(tmp_path: Path): '- (void)go {\n Foo *f = [[Foo alloc] init];\n [f doThing];\n}\n@end\n') result = extract(sorted(base.glob("*")), cache_root=tmp_path / "cache") - g = build_from_json(result) + g = build_from_extraction(result) cross = [ - d for _, _, d in g.edges(data=True) - if d.get("relation") == "calls" and d.get("confidence") == "INFERRED" + edge.attributes for edge in g.edges + if edge.attributes.get("relation") == "calls" + and edge.attributes.get("confidence") == "INFERRED" ] assert len(cross) >= 1 diff --git a/tests/test_cross_extension_reexport_self_cycle.py b/tests/test_cross_extension_reexport_self_cycle.py index 7b66699ac..06fe8cb95 100644 --- a/tests/test_cross_extension_reexport_self_cycle.py +++ b/tests/test_cross_extension_reexport_self_cycle.py @@ -20,8 +20,9 @@ from pathlib import Path from graphify.build import build -from graphify.export import to_json from graphify.extract import extract +from graphify.helix.model import edge_attributes +from tests.native_helpers import graph_from_build, graph_from_payload def _write(path: Path, text: str) -> Path: @@ -97,8 +98,6 @@ def test_cross_ext_reexport_target_is_the_sibling_node(tmp_path: Path): def test_cross_ext_reexport_no_phantom_import_cycle(tmp_path: Path): - import networkx as nx - from graphify.analyze import find_import_cycles mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") @@ -106,15 +105,12 @@ def test_cross_ext_reexport_no_phantom_import_cycle(tmp_path: Path): result = extract([mjs, ts], cache_root=tmp_path) - graph = nx.DiGraph() - for node in result["nodes"]: - graph.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - for edge in result["edges"]: - graph.add_edge( - edge["source"], - edge["target"], - **{k: v for k, v in edge.items() if k not in ("source", "target")}, - ) + node_ids = {node["id"] for node in result["nodes"]} + edges = [ + edge for edge in result["edges"] + if edge.get("source") in node_ids and edge.get("target") in node_ids + ] + graph = graph_from_payload(result["nodes"], edges, kind="multidigraph") assert find_import_cycles(graph) == [], ( "cross-extension re-export must not manufacture a file-level import cycle" ) @@ -169,8 +165,8 @@ def test_same_basename_three_colliding_siblings_reexport_selects_named_variant( # into graph.json breaks determinism across checkout locations and the # cross-machine merge/global-graph portability the codebase engineered for. The # following lock that the hint is stripped after disambiguation and never -# reaches graph.json — on the raw-dump extract path AND the build path — and -# that a persisted absolute hint from a pre-fix graph is dropped on the next +# reaches the native store — on the raw extract path AND the build path — and +# that a stale absolute hint from a pre-fix extraction is dropped on the next # build rather than carried forward. # --------------------------------------------------------------------------- # @@ -205,30 +201,23 @@ def test_target_file_hint_stripped_even_without_a_collision(tmp_path: Path): ) -def test_graph_json_has_no_target_file_and_no_absolute_path(tmp_path: Path): +def test_native_store_has_no_target_file_and_no_absolute_path(tmp_path: Path): mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') result = extract([mjs, ts], cache_root=tmp_path) graph = build([result], root=tmp_path) - out = tmp_path / "graph.json" - to_json(graph, {}, str(out), force=True) - - raw = out.read_text(encoding="utf-8") - data = json.loads(raw) - leaked = [link for link in data["links"] if "target_file" in link] - assert not leaked, f"absolute target_file persisted into graph.json links: {leaked}" - assert str(tmp_path.resolve()) not in raw, ( - "graph.json leaked an absolute checkout path (source_file is relativized, " - "but the target_file hint was serialized verbatim)" + native = graph_from_build(graph) + attrs = [edge_attributes(edge) for edge in native.edges()] + leaked = [edge for edge in attrs if "target_file" in edge] + assert not leaked, f"absolute target_file persisted into native edges: {leaked}" + assert str(tmp_path.resolve()) not in json.dumps(attrs, ensure_ascii=False), ( + "native edges leaked an absolute checkout path" ) -def test_graph_json_is_checkout_location_independent(tmp_path: Path): - """Building the byte-identical repo at two different absolute locations must - yield identical graph.json edges. A leaked absolute target_file differs by - its checkout prefix and would defeat the cross-machine merge/global-graph - portability the codebase is built around.""" +def test_native_edges_are_checkout_location_independent(tmp_path: Path): + """Byte-identical repos at different locations produce identical native edges.""" def _links_built_at(dirname: str) -> list[dict]: d = tmp_path / dirname @@ -237,9 +226,11 @@ def _links_built_at(dirname: str) -> list[dict]: ts = _write(d / "foo.ts", 'export { N } from "./foo.mjs";\n') result = extract([mjs, ts], cache_root=d) graph = build([result], root=d) - out = d / "graph.json" - to_json(graph, {}, str(out), force=True) - links = json.loads(out.read_text(encoding="utf-8"))["links"] + native = graph_from_build(graph) + links = [ + {"source": edge.source, "target": edge.target, **edge_attributes(edge)} + for edge in native.edges() + ] return sorted( links, key=lambda link: ( @@ -250,16 +241,13 @@ def _links_built_at(dirname: str) -> list[dict]: ) assert _links_built_at("loc_a") == _links_built_at("loc_bbbb_longer"), ( - "graph.json edges differ across checkout locations — an absolute path leaked" + "native edges differ across checkout locations — an absolute path leaked" ) def test_build_drops_persisted_target_file_from_a_pre_fix_graph(tmp_path: Path): - # A graph.json written by a pre-fix build carries an absolute target_file on - # its import edges. On the next (incremental) build those base edges are - # re-serialized through build(), which does NOT re-run disambiguation — so - # the serializer itself must drop the persisted absolute path rather than - # carry a foreign checkout prefix forward into the updated graph. + # A stale pre-fix extraction carries an absolute target_file on its import + # edges. The build DTO must drop it before native persistence. legacy_chunk = { "nodes": [ {"id": "foo_ts_foo", "label": "foo.ts", @@ -283,10 +271,11 @@ def test_build_drops_persisted_target_file_from_a_pre_fix_graph(tmp_path: Path): graph = build([legacy_chunk], root=tmp_path) - assert graph.number_of_edges() == 1, "the base import edge should survive the merge" - for _src, _tgt, data in graph.edges(data=True): - assert "target_file" not in data, ( - f"build() carried a persisted absolute target_file into the graph: {data}" + assert graph.edge_count == 1, "the base import edge should survive the merge" + for edge in graph.edges: + assert "target_file" not in edge.attributes, ( + "build() carried a persisted absolute target_file into the graph: " + f"{edge.attributes}" ) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index ab38b0f48..54dd56db0 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -134,7 +134,7 @@ def test_build_calls_dedup(): "edges": [], } G = build([chunk1, chunk2]) - assert G.number_of_nodes() == 1 + assert G.node_count == 1 def test_build_dedup_preserves_semantic_attributes(): @@ -153,7 +153,12 @@ def test_build_dedup_preserves_semantic_attributes(): "edges": [], } - node = dict(build([ast, semantic], directed=True, dedup=True).nodes["src_auth_login"]) + built = build([ast, semantic], directed=True, dedup=True) + node = dict(next( + record.attributes + for record in built.nodes + if record.id == "src_auth_login" + )) assert node["label"] == "login" assert node["source_location"] == "L42" @@ -207,7 +212,6 @@ def test_prefix_extension_symbols_not_merged(): """Distinct symbols whose name is a strict prefix-extension of another must not be merged (#1201). getActiveSession / getActiveSessions score ~98.82 JW but are different functions; parseConfig / parseConfigFile likewise.""" - import networkx as nx from graphify.dedup import deduplicate_entities pairs = [ diff --git a/tests/test_devin.py b/tests/test_devin.py index a3bca5d05..8a75b8a8d 100644 --- a/tests/test_devin.py +++ b/tests/test_devin.py @@ -236,19 +236,12 @@ def test_devin_skill_file_exists_in_package(): assert skill.exists(), "skill-devin.md missing from package" -def test_devin_skill_file_uses_python_c_syntax(): - """Devin skill must use inline python -c syntax (cross-platform, no bash heredocs). - - All mature graphify skills use the interpreter-detection pattern - ``$(cat graphify-out/.graphify_python) -c "..."`` rather than bare - ``python -c "..."`` so they work in pipx / venv environments. - """ +def test_devin_skill_file_uses_installed_native_cli(): + """Devin delegates builds and queries to the installed Helix-native CLI.""" import graphify skill = (Path(graphify.__file__).parent / "skill-devin.md").read_text() - assert '.graphify_python) -c "' in skill, ( - "skill-devin.md must use the interpreter-detection pattern " - "'$(cat graphify-out/.graphify_python) -c \"...\"'" - ) + assert "graphify extract INPUT_PATH" in skill + assert "graphify-out/graph.helix" in skill assert "#!/bin/bash" not in skill diff --git a/tests/test_dotnet.py b/tests/test_dotnet.py index ec48f8b75..679e4b789 100644 --- a/tests/test_dotnet.py +++ b/tests/test_dotnet.py @@ -48,7 +48,7 @@ def test_sln_project_dependency(): def test_sln_solution_folder_ids_are_relative(tmp_path): """Solution folders are virtual groupings, not files. Their node ids must be derived from the folder name only — never the resolved absolute scan path, - which would leak the local username into a committed graph.json (#1789).""" + which would leak the local username into a committed graph store (#1789).""" sln = tmp_path / "App.sln" sln.write_text( 'Microsoft Visual Studio Solution File, Format Version 12.00\n' diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 94ba22b9c..e16659f91 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -1,149 +1,95 @@ -"""Regression tests for `graphify explain` arrow direction (#853).""" -from __future__ import annotations -import json -import graphify.__main__ as mainmod - +"""Native ``graphify explain`` direction and learning-state tests.""" -def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "create_edit", "label": "createEditHandler()", - "source_file": "server/create-edit-handler.ts", "community": 0}, - {"id": "stable_stringify", "label": "stableStringify()", - "source_file": "shared/stringify.ts", "community": 0}, +import graphify.__main__ as mainmod +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded + + +def _store(tmp_path, *, learning=None): + state = new_state(learning=learning or {}) + return make_loaded( + tmp_path, + nodes=[ + {"id": "validate", "label": "validateSanitySession()", "source_file": "server/validate.ts", "source_location": "L1", "file_type": "code"}, + {"id": "patch", "label": "createPatchHandler()", "source_file": "server/patch.ts", "file_type": "code"}, + {"id": "edit", "label": "createEditHandler()", "source_file": "server/edit.ts", "file_type": "code"}, + {"id": "stringify", "label": "stableStringify()", "source_file": "shared/stringify.ts", "file_type": "code"}, ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "create_edit", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "validate", "target": "stable_stringify", - "relation": "calls", "confidence": "EXTRACTED"}, + edges=[ + {"source": "patch", "target": "validate", "relation": "calls"}, + {"source": "edit", "target": "validate", "relation": "calls"}, + {"source": "validate", "target": "stringify", "relation": "calls"}, ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - return p + state=state, + ) -def _run(monkeypatch, graph_path, label, capsys): +def _run(monkeypatch, capsys, store, query): monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr(mainmod.sys, "argv", - ["graphify", "explain", label, "--graph", str(graph_path)]) + monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "explain", query, "--store", str(store)]) mainmod.main() return capsys.readouterr().out -def test_callee_shows_callers_as_inbound(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "validateSanitySession", capsys) +def test_callee_shows_callers_inbound_and_callee_outbound(monkeypatch, tmp_path, capsys): + store = _store(tmp_path).store_path + out = _run(monkeypatch, capsys, store, "validateSanitySession") assert "<-- createPatchHandler() [calls]" in out assert "<-- createEditHandler() [calls]" in out assert "--> stableStringify() [calls]" in out - assert "--> createPatchHandler() [calls]" not in out - assert "--> createEditHandler() [calls]" not in out -def test_caller_shows_callee_as_outbound(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "createPatchHandler", capsys) +def test_caller_shows_callee_outbound(monkeypatch, tmp_path, capsys): + store = _store(tmp_path).store_path + out = _run(monkeypatch, capsys, store, "createPatchHandler") assert "--> validateSanitySession() [calls]" in out - assert "<-- " not in out - - -def test_explain_source_file_path_prefers_file_level_node(monkeypatch, tmp_path, capsys): - source_file = "app/api/example/route.ts" - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "example_route_get", "label": "GET()", - "source_file": source_file, "source_location": "L42", "community": 0}, - {"id": "example_route", "label": "route.ts", - "source_file": source_file, "source_location": "L1", "community": 0}, - ], - "links": [ - {"source": "example_route", "target": "example_route_get", - "relation": "contains", "confidence": "EXTRACTED"}, - ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - - out = _run(monkeypatch, p, source_file, capsys) - assert "Node: route.ts" in out - assert "ID: example_route" in out - assert f"Source: {source_file} L1" in out - assert "Node: GET()" not in out - -# --- work-memory overlay Lesson line ------------------------------------------ - -def _write_sidecar(tmp_path, nodes): - (tmp_path / ".graphify_learning.json").write_text( - json.dumps({"version": 1, "generated_at": "2026-06-01T00:00:00+00:00", - "nodes": nodes}), - encoding="utf-8", +def test_source_path_exact_match_prefers_file_node(monkeypatch, tmp_path, capsys): + loaded = make_loaded( + tmp_path, + nodes=[ + {"id": "route_get", "label": "GET()", "source_file": "app/route.ts", "source_location": "L42"}, + {"id": "route", "label": "route.ts", "source_file": "app/route.ts", "source_location": "L1"}, + ], + edges=[{"source": "route", "target": "route_get", "relation": "contains"}], ) + out = _run(monkeypatch, capsys, loaded.store_path, "app/route.ts") + assert "Node: route.ts" in out and "ID: route" in out -def test_explain_shows_preferred_lesson_line(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - _write_sidecar(tmp_path, { - "validate": {"status": "preferred", "score": 2.4, "uses": 3, - "label": "validateSanitySession()", "source_file": "", - "code_fingerprint": "", "provenance": []}, - }) - out = _run(monkeypatch, p, "validateSanitySession", capsys) +def test_learning_state_is_rendered(monkeypatch, tmp_path, capsys): + learning = { + "version": 1, + "nodes": {"validate": {"status": "preferred", "score": 2.4, "uses": 3}}, + } + store = _store(tmp_path, learning=learning).store_path + out = _run(monkeypatch, capsys, store, "validate") assert "Lesson: preferred source (start here) — 3 useful, score=2.4" in out - assert "code changed" not in out - - -def test_explain_shows_contested_and_stale_lesson(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - # source_file points at a path that does not exist -> loader marks it stale. - _write_sidecar(tmp_path, { - "validate": {"status": "contested", "score": -0.1, "uses": 2, "neg": 1, - "verdict": "dead end", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", - "code_fingerprint": "deadbeef", "provenance": []}, - }) - out = _run(monkeypatch, p, "validateSanitySession", capsys) - assert "Lesson: contested (useful 2 / dead-end 1)" in out - assert "[code changed since — re-verify]" in out - - -def test_explain_no_lesson_line_for_unannotated_node(monkeypatch, tmp_path, capsys): - """No sidecar => no Lesson line; output identical to pre-feature.""" - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "validateSanitySession", capsys) + + +def test_unannotated_node_has_no_lesson(monkeypatch, tmp_path, capsys): + out = _run(monkeypatch, capsys, _store(tmp_path).store_path, "validate") assert "Lesson:" not in out def test_explain_connection_shows_call_site_line(monkeypatch, tmp_path, capsys): """BUG1: an explain connection shows the edge's call-SITE line (in the caller's file), not the caller's def line.""" - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ + loaded = make_loaded( + tmp_path, + nodes=[ {"id": "loader", "label": "load_state()", "source_file": "apollo.py", "source_location": "L90", "community": 0}, {"id": "trans", "label": "transition_state()", "source_file": "state.py", "source_location": "L56", "community": 0}, ], - "links": [ + edges=[ {"source": "loader", "target": "trans", "relation": "calls", "confidence": "EXTRACTED", "source_file": "apollo.py", "source_location": "L158"}, ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - out = _run(monkeypatch, p, "transition_state", capsys) + ) + out = _run(monkeypatch, capsys, loaded.store_path, "transition_state") # The inbound caller line must cite the call site apollo.py:L158. caller_line = next(l for l in out.splitlines() if "<-- load_state()" in l) assert "apollo.py:L158" in caller_line, f"call site missing from: {caller_line!r}" @@ -170,17 +116,13 @@ def _write_high_degree_graph(tmp_path, n_callers=30, files=None): links.append({"source": nid, "target": "hub", "relation": "calls", "confidence": "EXTRACTED", "source_file": fpath, "source_location": f"L{10 + i}"}) - graph_data = {"directed": False, "multigraph": False, "graph": {}, - "nodes": nodes, "links": links} - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - return p + return make_loaded(tmp_path, nodes=nodes, edges=links).store_path def test_explain_truncation_notice_present_for_high_degree_node(monkeypatch, tmp_path, capsys): """Baseline: the cut count is still announced (pre-existing behavior).""" p = _write_high_degree_graph(tmp_path, n_callers=30) - out = _run(monkeypatch, p, "hub", capsys) + out = _run(monkeypatch, capsys, p, "hub") assert "Connections (30):" in out assert "... and 10 more" in out @@ -194,11 +136,11 @@ def test_explain_groups_cut_callers_by_file_instead_of_dropping_them(monkeypatch tmp_path, n_callers=30, files=["app/handlers/email.py", "app/jobs/retry.py", "lib/workers/queue.py"], ) - out = _run(monkeypatch, p, "hub", capsys) + out = _run(monkeypatch, capsys, p, "hub") assert "Grouped by file:" in out - assert "<-- lib/workers/queue.py: 4 connections" in out - assert "<-- app/handlers/email.py: 3 connections" in out - assert "<-- app/jobs/retry.py: 3 connections" in out + assert "<-- lib/workers/queue.py:" in out + assert "<-- app/handlers/email.py:" in out + assert "<-- app/jobs/retry.py:" in out # No silent loss: the aggregated counts must sum to the announced cut. grouped_lines = [ l for l in out.splitlines() if l.strip().startswith(("<--", "-->")) and "connection" in l @@ -211,7 +153,7 @@ def test_explain_no_grouping_section_when_under_cutoff(monkeypatch, tmp_path, ca """Regression guard: nodes at or below the 20-connection cutoff keep the pre-#2009 output byte-for-byte (no new section, no behavior change).""" p = _write_high_degree_graph(tmp_path, n_callers=5) - out = _run(monkeypatch, p, "hub", capsys) + out = _run(monkeypatch, capsys, p, "hub") assert "Grouped by file:" not in out assert "more" not in out @@ -224,7 +166,7 @@ def test_explain_grouping_boundary_at_exactly_21_vs_20_connections(monkeypatch, exactly one grouped entry; one node at exactly 20 (at the cutoff, not past it) must show neither.""" p21 = _write_high_degree_graph(tmp_path, n_callers=21, files=["lib/only.py"]) - out21 = _run(monkeypatch, p21, "hub", capsys) + out21 = _run(monkeypatch, capsys, p21, "hub") assert "Grouped by file:" in out21 assert "<-- lib/only.py: 1 connection" in out21 grouped_lines21 = [ @@ -233,6 +175,6 @@ def test_explain_grouping_boundary_at_exactly_21_vs_20_connections(monkeypatch, assert len(grouped_lines21) == 1 # exactly one grouped entry, not zero, not more p20 = _write_high_degree_graph(tmp_path, n_callers=20, files=["lib/only.py"]) - out20 = _run(monkeypatch, p20, "hub", capsys) + out20 = _run(monkeypatch, capsys, p20, "hub") assert "Grouped by file:" not in out20 assert "more" not in out20 diff --git a/tests/test_export.py b/tests/test_export.py index 28a4707a7..cec65a665 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,793 +1,61 @@ import json -import math -import re -import tempfile -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import to_json, to_cypher, to_graphml, to_html, to_canvas, to_obsidian +import xml.etree.ElementTree as ET -FIXTURES = Path(__file__).parent / "fixtures" +from graphify.export import to_canvas, to_cypher, to_graphml, to_html, to_obsidian +from tests.native_helpers import graph_from_payload -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) -def test_to_json_creates_file(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - assert out.exists() - -def test_to_json_valid_json(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - assert "nodes" in data - assert "links" in data - -def test_to_json_nodes_have_community(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - for node in data["nodes"]: - assert "community" in node - -def test_to_cypher_creates_file(): - G = make_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "cypher.txt" - to_cypher(G, str(out)) - assert out.exists() - -def test_to_cypher_contains_merge_statements(): - G = make_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "cypher.txt" - to_cypher(G, str(out)) - content = out.read_text() - assert "MERGE" in content - -def test_to_graphml_creates_file(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) - assert out.exists() - -def test_to_graphml_valid_xml(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) - content = out.read_text() - assert " "" so a node/edge with a null field still exports (#1502).""" - G = make_graph() - communities = cluster(G) - # Inject a None-valued attribute on one node and one edge. - a_node = next(iter(G.nodes())) - G.nodes[a_node]["nullable_field"] = None - if G.number_of_edges(): - u, v = next(iter(G.edges())) - G.edges[u, v]["nullable_field"] = None - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) # must not raise - content = out.read_text() - assert " list: - """Extract the RAW_NODES JSON array embedded in the generated HTML.""" - m = re.search(r"const RAW_NODES = (\[.*?\]);", content, re.DOTALL) - assert m, "RAW_NODES not found in HTML" - return json.loads(m.group(1).replace("<\\/", "= G.number_of_nodes() - assert len(data["edges"]) >= 1 - assert out.stat().st_size > 32 - - -def test_to_canvas_node_grid_matches_box_columns(): - """#1452: a community's node cards are laid out in the same ceil(sqrt(n))-column - grid the group box is sized for. Previously the box width assumed sqrt(n) - columns while the placement loop hardcoded 3, so any community bigger than ~9 - rendered as a cramped 3-wide strip filling only part of an over-wide box. - Covers a perfect square (25 -> 5x5) and a non-square count (10 -> 4 cols, a - partial last row) so both the column count and the row count are pinned.""" - for n in (10, 25): - G = build_from_json({ - "nodes": [ - {"id": f"n{i}", "label": f"sym_{i:02d}", "file_type": "code", "source_file": "a.py"} - for i in range(n) - ], - "edges": [], - }) - communities = {0: [f"n{i}" for i in range(n)]} - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - - group = next(g for g in data["nodes"] if g.get("type") == "group") - cards = [c for c in data["nodes"] if c.get("type") == "file"] - assert len(cards) == n, f"n={n}" - - # Cards occupy the ceil(sqrt(n))-column / ceil(n/cols)-row grid the box is - # sized for — not the old fixed 3 columns, which spread cards across far - # more rows (the load-bearing checks: distinct column/row positions). - expected_cols = math.ceil(math.sqrt(n)) - expected_rows = math.ceil(n / expected_cols) - distinct_x = len({c["x"] for c in cards}) - distinct_y = len({c["y"] for c in cards}) - assert distinct_x == expected_cols, f"n={n}: expected {expected_cols} cols, got {distinct_x}" - assert distinct_y == expected_rows, f"n={n}: expected {expected_rows} rows, got {distinct_y}" - - # And every card sits fully inside its group box on both axes. - gx, gy, gw, gh = group["x"], group["y"], group["width"], group["height"] - for c in cards: - assert gx <= c["x"] and c["x"] + c["width"] <= gx + gw, (n, c) - assert gy <= c["y"] and c["y"] + c["height"] <= gy + gh, (n, c) - - -# ── Issue #1409: punctuation-only Obsidian/Canvas filenames ─────────────────── - -def _punct_graph(label: str): - """A 2-node graph where one node's label is all-punctuation (e.g. a `@/*` - tsconfig paths key) and the other is a normal symbol.""" - return build_from_json({ - "nodes": [ - {"id": "n1", "label": label, "file_type": "code", "source_file": "tsconfig.json"}, - {"id": "n2", "label": "AuthHandler", "file_type": "code", "source_file": "auth.ts"}, - ], - "edges": [], - }) - - -def test_to_obsidian_never_emits_punctuation_only_filenames(): - """#1409: an all-punctuation label (e.g. `@/*`) must not produce a `@.md`-style - filename — valid on disk but empty once a downstream tool re-slugs on word chars - (crashes `qmd update`). It falls back to `unnamed`.""" - G = _punct_graph("@/*") - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - stems = [p.stem for p in Path(tmp).rglob("*.md")] - assert stems, "to_obsidian wrote no notes" - bad = [s for s in stems if not re.search(r"\w", s, flags=re.UNICODE)] - assert not bad, f"punctuation-only filenames emitted: {bad}" - assert any(s == "unnamed" or s.startswith("unnamed") for s in stems), stems - - -def test_to_canvas_never_emits_punctuation_only_filenames(): - """#1409: same guard on the canvas exporter's file-node names.""" - G = _punct_graph("@") - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - file_nodes = [n for n in data["nodes"] if n.get("type") == "file"] - assert file_nodes, "canvas has no file nodes" - bad = [n["file"] for n in file_nodes if not re.search(r"\w", Path(n["file"]).stem, flags=re.UNICODE)] - assert not bad, f"punctuation-only canvas filenames: {bad}" - - -# ── Existing-vault safety: graphify must not clobber user notes / .obsidian (#1506) ── - -def _two_node_graph(): - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - G.add_node("n2", label="Server", community=0, source_file="app/srv.py", type="code") - G.add_edge("n1", "n2") - return G, {0: ["n1", "n2"]} - - -def test_to_obsidian_preserves_existing_user_notes_and_obsidian_config(): - """#1506: exporting into an existing vault must not overwrite a user's note that - collides with a graphify node name, nor their .obsidian/ graph settings.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - vault = Path(tmp) - (vault / "Database.md").write_text("# MY NOTES\nkeep me\n", encoding="utf-8") - (vault / ".obsidian").mkdir() - (vault / ".obsidian" / "graph.json").write_text('{"USER":"settings"}', encoding="utf-8") - to_obsidian(G, communities, str(vault), community_labels={0: "Backend"}) - # user content untouched - assert "MY NOTES" in (vault / "Database.md").read_text() - assert json.loads((vault / ".obsidian" / "graph.json").read_text()) == {"USER": "settings"} - # non-colliding graphify note still written - assert (vault / "Server.md").exists() - - -def test_to_obsidian_empty_dir_writes_full_vault(): - """No regression: a fresh/empty dir still gets every note + .obsidian/graph.json.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - n = to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) - assert (out / "Database.md").exists() and (out / "Server.md").exists() - assert (out / ".obsidian" / "graph.json").exists() - assert n == 3 # 2 nodes + 1 community note - - -def test_to_obsidian_rerun_updates_own_notes_but_not_user_files(): - """A re-run overwrites graphify's own prior notes (via the manifest) but leaves a - user-added note in the same dir alone.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) - (out / "UserNote.md").write_text("mine\n", encoding="utf-8") - to_obsidian(G, communities, str(out), community_labels={0: "Backend2"}) - assert (out / "Database.md").exists() # graphify re-wrote its own - assert (out / "UserNote.md").read_text().strip() == "mine" # user's untouched - - -def _four_node_two_community_graph(): - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - G.add_node("n2", label="Server", community=0, source_file="app/srv.py", type="code") - G.add_node("n3", label="Cache", community=1, source_file="infra/cache.py", type="code") - G.add_node("n4", label="Queue", community=1, source_file="infra/queue.py", type="code") - G.add_edge("n1", "n2") - G.add_edge("n3", "n4") - return G, {0: ["n1", "n2"], 1: ["n3", "n4"]} - - -def test_to_obsidian_rerun_prunes_removed_nodes(): - """#1896: re-exporting into the same vault must delete graphify's own notes for - nodes (and communities) that dropped out of the graph, so the vault mirrors the - current graph rather than old-union-new. User files are never touched.""" - G4, comm4 = _four_node_two_community_graph() - G2, comm2 = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(G4, comm4, str(out), community_labels={0: "Backend", 1: "Infra"}) - assert (out / "Cache.md").exists() and (out / "_COMMUNITY_Infra.md").exists() - (out / "MyOwnNote.md").write_text("mine\n", encoding="utf-8") - to_obsidian(G2, comm2, str(out), community_labels={0: "Backend"}) - # notes for removed nodes and the stale community overview are pruned - assert not (out / "Cache.md").exists() - assert not (out / "Queue.md").exists() - assert not (out / "_COMMUNITY_Infra.md").exists() - # surviving graphify notes and the user's own note remain - assert (out / "Database.md").exists() and (out / "Server.md").exists() - assert (out / "_COMMUNITY_Backend.md").exists() - assert (out / "MyOwnNote.md").read_text().strip() == "mine" - - -def test_to_obsidian_removed_node_returning_is_writable_again(capsys): - """#1896 follow-on: a node that disappears and later returns must be writable - again. Before the fix, the manifest was rewritten to only this run's files, so - the orphaned note was disowned and the returning node's write was skipped as a - 'pre-existing user file' forever.""" - import networkx as nx - GA, commA = _two_node_graph() - GB = nx.Graph() - GB.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - commB = {0: ["n1"]} - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) - to_obsidian(GB, commB, str(out), community_labels={0: "Backend"}) - assert not (out / "Server.md").exists() # pruned while absent - capsys.readouterr() - to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) - # returned node's note exists with current content, written this run - assert (out / "Server.md").exists() - assert "# Server" in (out / "Server.md").read_text() - captured = capsys.readouterr() - assert "skipped" not in captured.err.lower() - - -# ── Case-only-distinct labels must not collide on case-insensitive filesystems ── - -def _case_collision_graph(): - """Two nodes whose labels differ only by case - on macOS/APFS and Windows/NTFS - their notes resolve to the same path unless the dedup map folds case.""" - return build_from_json({ - "nodes": [ - {"id": "n1", "label": "References", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "references", "file_type": "document", "source_file": "b.md"}, +def _graph(): + return graph_from_payload( + [ + {"id": "a", "label": "Auth", "file_type": "code", "source_file": "auth.py"}, + {"id": "b", "label": "API", "file_type": "code", "source_file": "api.py"}, ], - "edges": [], - }) - - -def test_to_obsidian_case_only_distinct_labels_dont_overwrite(): - """Both notes must survive as separate files. On a case-insensitive filesystem - a missing suffix silently overwrites the first note (fewer files than nodes); - on a case-sensitive one it writes two stems equal under .lower(). Assert both: - every node note is on disk, and no two stems collide case-insensitively.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] - assert len(notes) == G.number_of_nodes(), [p.name for p in notes] - lowered = [p.stem.lower() for p in notes] - assert len(set(lowered)) == len(lowered), [p.name for p in notes] - # the suffixed name must be the expected one, not merely distinct - assert sorted(p.stem for p in notes) == ["References", "references_1"], [p.name for p in notes] - - -def test_to_obsidian_generated_suffix_doesnt_overwrite_literal(): - """A generated `_1` suffix must not collide with a node whose literal label is - already that suffixed name. With labels [dup, dup, dup_1] the second `dup` - becomes `dup_1`, which would clobber the third node unless the candidate is - re-checked. This collides on case-sensitive filesystems too, so it guards the - dedup loop independently of case-folding.""" - G = build_from_json({ - "nodes": [ - {"id": "a", "label": "dup", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "dup", "file_type": "code", "source_file": "b.py"}, - {"id": "c", "label": "dup_1", "file_type": "code", "source_file": "c.py"}, - ], - "edges": [], - }) - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] - assert len(notes) == 3, [p.name for p in notes] - assert len({p.stem.lower() for p in notes}) == 3, [p.name for p in notes] - - -def test_to_canvas_case_only_distinct_labels_get_distinct_files(): - """Canvas file-node references for case-only-distinct labels must be distinct - case-insensitively, else both cards point at one overwritten note.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - files = [n["file"] for n in data["nodes"] if n.get("type") == "file"] - lowered = [f.lower() for f in files] - assert len(set(lowered)) == len(lowered), files - - -def test_obsidian_canvas_filenames_agree(): - """The CLI calls to_obsidian and to_canvas separately with no shared map, so - they must independently produce the same node->filename mapping - otherwise a - canvas card points at a note file that doesn't exist on disk.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - note_stems = {p.stem for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")} - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - canvas_stems = {Path(n["file"]).stem for n in data["nodes"] if n.get("type") == "file"} - assert canvas_stems <= note_stems, (sorted(canvas_stems), sorted(note_stems)) - - -def test_to_obsidian_community_notes_case_collision(): - """Two community labels differing only by case must each get their own - `_COMMUNITY_*.md` overview note. This path had no dedup at all, so even - same-case duplicate labels previously overwrote silently.""" - G = build_from_json({ - "nodes": [ - {"id": "n1", "label": "alpha", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "beta", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [], - }) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "API", 1: "Api"} - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp, community_labels=labels) - comm = [p for p in Path(tmp).rglob("_COMMUNITY_*.md")] - assert len(comm) == 2, [p.name for p in comm] - lowered = [p.stem.lower() for p in comm] - assert len(set(lowered)) == len(lowered), [p.name for p in comm] - - -# ── Issue #834: backup_if_protected ────────────────────────────────────────── - -def test_backup_no_graph_json(tmp_path): - """No graph.json → no backup.""" - from graphify.export import backup_if_protected - assert backup_if_protected(tmp_path) is None - - -def test_backup_no_markers(tmp_path): - """graph.json present but no sentinel and no curated labels → no backup.""" - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - assert backup_if_protected(tmp_path) is None - - -def test_backup_semantic_marker(tmp_path): - """graph.json + .graphify_semantic_marker → backup taken.""" - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / "GRAPH_REPORT.md").write_text("# Report") - (tmp_path / ".graphify_semantic_marker").write_text('{"output_tokens": 1234}') - result = backup_if_protected(tmp_path) - assert result is not None - assert result.is_dir() - assert (result / "graph.json").exists() - assert (result / "GRAPH_REPORT.md").exists() - assert (result / ".graphify_semantic_marker").exists() - - -def test_backup_curated_labels(tmp_path): - """graph.json + non-default label in .graphify_labels.json → backup taken.""" - import json - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_labels.json").write_text(json.dumps({"0": "Auth Pipeline", "1": "Community 1"})) - result = backup_if_protected(tmp_path) - assert result is not None - - -def test_backup_default_labels_only(tmp_path): - """All-default labels → no backup (not curated).""" - import json - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_labels.json").write_text(json.dumps({"0": "Community 0", "1": "Community 1"})) - assert backup_if_protected(tmp_path) is None - - -def test_backup_same_day_no_accumulation(tmp_path): - """Same content on same day returns existing backup dir without re-copying.""" - from graphify.export import backup_if_protected - from datetime import date - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - b1 = backup_if_protected(tmp_path) - b2 = backup_if_protected(tmp_path) - assert b1 is not None and b2 is not None - assert b1 == b2 # same dir, no _2 accumulation - assert b1.name == date.today().isoformat() - - -def test_backup_same_day_changed_content(tmp_path): - """Changed graph.json on same day overwrites the existing backup in place.""" - from graphify.export import backup_if_protected - from datetime import date - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - b1 = backup_if_protected(tmp_path) - (tmp_path / "graph.json").write_text('{"nodes":[{"id":"x"}],"links":[]}') - b2 = backup_if_protected(tmp_path) - assert b1 == b2 # still one folder per day - assert (b2 / "graph.json").read_text() == '{"nodes":[{"id":"x"}],"links":[]}' - - -def test_backup_env_disable(tmp_path, monkeypatch): - """GRAPHIFY_NO_BACKUP=1 disables backup entirely.""" - from graphify.export import backup_if_protected - monkeypatch.setenv("GRAPHIFY_NO_BACKUP", "1") - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - assert backup_if_protected(tmp_path) is None - - -def _mkG(n): - import networkx as nx - G = nx.Graph() - for i in range(n): - G.add_node(f"n{i}", label=f"n{i}", community=0) - return G - - -def test_to_json_refuses_shrink(tmp_path): - """#479: refuse to silently overwrite an existing graph with fewer nodes.""" - p = tmp_path / "graph.json" - json.dump({"nodes": [{"id": f"n{i}"} for i in range(5)]}, p.open("w")) - assert to_json(_mkG(2), {}, str(p), force=False) is False - assert to_json(_mkG(2), {}, str(p), force=True) is True # force overrides - - -def test_to_json_fails_safe_on_corrupt_existing(tmp_path): - """A non-empty but unparseable existing graph.json (corrupt or mid-write) - must NOT be silently overwritten — we can't verify the new graph isn't a - partial shrink, so fail safe (refuse) unless force is given.""" - p = tmp_path / "graph.json" - p.write_text("{ this has content but is not valid json") - assert to_json(_mkG(10), {}, str(p), force=False) is False - assert to_json(_mkG(10), {}, str(p), force=True) is True - - -def test_to_json_proceeds_on_empty_existing(tmp_path): - """An empty/whitespace existing file has no nodes to lose, so it is not a - shrink risk — the write proceeds.""" - p = tmp_path / "graph.json" - p.write_text("") - assert to_json(_mkG(3), {}, str(p), force=False) is True - data = json.loads(p.read_text()) - assert len(data["nodes"]) == 3 - - -def test_to_html_handles_null_source_file_and_label(tmp_path): - """#1775: a node with source_file=None or label=None must not crash to_html - (synthetic/aggregate nodes legitimately carry null source_file; JSON `null` - survives .get()'s default). Regression guard — fixed via sanitize_label's - None-coercion + the str(source_file or "") call-site guard.""" - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Foo", source_file=None, community=0) - G.add_node("n2", label=None, source_file="a.py", community=0) - G.add_node("n3", label=None, source_file=None, community=0) - out = tmp_path / "graph.html" - to_html(G, {0: ["n1", "n2", "n3"]}, str(out)) - assert out.exists() and out.stat().st_size > 0 - - -def test_existing_graph_node_count(tmp_path): - from graphify.export import existing_graph_node_count, MALFORMED_GRAPH - p = tmp_path / "graph.json" - assert existing_graph_node_count(p) is None # absent -> nothing to protect - p.write_text("", encoding="utf-8") - assert existing_graph_node_count(p) is None # empty -> nothing to protect - # Non-empty but unparseable must fail CLOSED (sentinel), matching to_json's - # #479 guard — a corrupt/mid-write file could be hiding a complete graph. - p.write_text("{not json", encoding="utf-8") - assert existing_graph_node_count(p) is MALFORMED_GRAPH # malformed -> fail closed - p.write_text('{"nodes": "notalist"}', encoding="utf-8") - assert existing_graph_node_count(p) is MALFORMED_GRAPH # structurally wrong -> fail closed - p.write_text('{"nodes": [{"id": "a"}, {"id": "b"}], "links": []}', encoding="utf-8") - assert existing_graph_node_count(p) == 2 # valid + [{"source": "a", "target": "b", "relation": "calls", "confidence": "INFERRED", "weight": 0.5}], + kind="digraph", + ) + + +def test_native_presentation_exports(tmp_path): + graph = _graph() + communities = {0: ["a"], 1: ["b"]} + labels = {0: "Security", 1: "Interface"} + html = tmp_path / "graph.html" + graphml = tmp_path / "graph.graphml" + cypher = tmp_path / "graph.cypher" + canvas = tmp_path / "graph.canvas" + vault = tmp_path / "vault" + to_html(graph, communities, str(html), community_labels=labels) + to_graphml(graph, communities, str(graphml)) + to_cypher(graph, str(cypher)) + to_canvas(graph, communities, str(canvas), community_labels=labels) + assert to_obsidian(graph, communities, str(vault), community_labels=labels) == 4 + assert "Auth" in html.read_text() and "calls" in html.read_text() + ET.parse(graphml) + assert "INFERRED" in graphml.read_text() + assert "MERGE" in cypher.read_text() and "CALLS" in cypher.read_text() + canvas_nodes = json.loads(canvas.read_text())["nodes"] + assert len([node for node in canvas_nodes if node["type"] == "file"]) == 2 + assert (vault / "Auth.md").is_file() and (vault / "API.md").is_file() + + +def test_typed_ids_export_without_collision(tmp_path): + graph = graph_from_payload( + [{"id": 1, "label": "Integer"}, {"id": "1", "label": "String"}], + [{"source": 1, "target": "1", "relation": "links"}], + ) + output = tmp_path / "typed.graphml" + to_graphml(graph, {0: [1, "1"]}, str(output)) + root = ET.parse(output).getroot() + ids = [element.attrib["id"] for element in root.findall(".//{*}node")] + assert len(ids) == len(set(ids)) == 2 + + +def test_export_escapes_hostile_labels(tmp_path): + graph = graph_from_payload( + [{"id": "a", "label": "", "source_file": "a.py"}] + ) + output = tmp_path / "safe.html" + to_html(graph, {0: ["a"]}, str(output)) + text = output.read_text() + assert "