Add deterministic R extraction

graphify/extract.py

@@ -47,6 +47,7 @@
 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form  # noqa: F401
 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest  # noqa: F401
 from graphify.extractors.razor import extract_razor  # noqa: F401
+from graphify.extractors.r import extract_r  # noqa: F401
 from graphify.extractors.rust import extract_rust  # noqa: F401
 from graphify.extractors.sln import extract_sln  # noqa: F401
 from graphify.extractors.sql import extract_sql  # noqa: F401
@@ -4024,6 +4025,7 @@ def add_existing_edge(edge: dict) -> None:
     ".sv": extract_verilog,
     ".svh": extract_verilog,
     ".sql": extract_sql,
+    ".r": extract_r,
     ".md": extract_markdown,
     ".mdx": extract_markdown,
     ".qmd": extract_markdown,
@@ -4080,7 +4082,7 @@ def add_existing_edge(edge: dict) -> None:
 # routes them to the CODE path via _shebang_interpreter; _get_extractor must
 # honor the same signal or these files are classified as code and then silently
 # dropped by extraction. Only interpreters with a real extractor are mapped —
-# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped.
+# detect's wider set (perl, fish, tcsh) stays unmapped and skipped.
 _SHEBANG_DISPATCH: dict[str, Any] = {
     "python": extract_python,
     "python2": extract_python,
@@ -4096,6 +4098,7 @@ def add_existing_edge(edge: dict) -> None:
     "lua": extract_lua,
     "php": extract_php,
     "julia": extract_julia,
+    "Rscript": extract_r,
 }
 
 
@@ -4545,8 +4548,8 @@ def extract(
         )
 
     # #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST
-    # extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently
-    # contributes zero nodes. The #1666 warning above deliberately skips these (it
+    # extractor wired up silently contributes zero nodes. The #1666 warning above
+    # deliberately skips these (it
     # only fires when an extractor exists), so surface them explicitly, grouped by
     # extension, rather than reporting success as if the language were mapped.
     from graphify.detect import CODE_EXTENSIONS as _CODE_EXTS

graphify/extractors/__init__.py

@@ -26,6 +26,7 @@
 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form
 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest
 from graphify.extractors.razor import extract_razor
+from graphify.extractors.r import extract_r
 from graphify.extractors.rust import extract_rust
 from graphify.extractors.sln import extract_sln
 from graphify.extractors.sql import extract_sql
@@ -55,6 +56,7 @@
     "powershell": extract_powershell,
     "powershell_manifest": extract_powershell_manifest,
     "razor": extract_razor,
+    "r": extract_r,
     "rust": extract_rust,
     "sln": extract_sln,
     "sql": extract_sql,

graphify/extractors/r.py

@@ -0,0 +1,244 @@
+"""Deterministic R extraction shared with Compass.
+
+R is intentionally source-driven: Graphify does not require an optional R
+grammar at runtime, while Compass keeps its grammar bundle static. Keeping the
+small structural contract here avoids a classified-as-code file disappearing
+from one implementation while preserving the useful R relationships: package
+imports, named functions, and direct calls between local functions.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from graphify.extractors.base import _file_stem, _make_id
+
+_FUNCTION = re.compile(
+    r"(?m)^[ \t]*([A-Za-z.][A-Za-z0-9._]*)[ \t]*(?:<<-|<-|=)[ \t]*function[ \t]*\("
+)
+_IMPORT = re.compile(
+    r"(?m)^[ \t]*(?:library|require|requireNamespace)[ \t]*\("
+)
+_PACKAGE = re.compile(
+    r"\s*(?:package\s*=\s*)?[\"']?([A-Za-z][A-Za-z0-9._-]*)"
+)
+_CALL = re.compile(
+    r"([A-Za-z.][A-Za-z0-9._]*(?:(?:::|\$)[A-Za-z.][A-Za-z0-9._]*)?)[ \t]*\("
+)
+_NON_CALLS = frozenset({"function", "if", "for", "while", "repeat", "switch", "return"})
+
+
+def _mask_non_code(source: str) -> str:
+    """Blank comments and strings without changing byte offsets or line numbers."""
+    out = list(source)
+    quote: str | None = None
+    escaped = False
+    comment = False
+    for index, char in enumerate(source):
+        if comment:
+            if char == "\n":
+                comment = False
+            else:
+                out[index] = " "
+            continue
+        if quote is not None:
+            if char == "\n":
+                continue
+            if escaped:
+                escaped = False
+                out[index] = " "
+            elif char == "\\":
+                escaped = True
+                out[index] = " "
+            elif char == quote:
+                quote = None
+            else:
+                out[index] = " "
+            continue
+        if char == "#":
+            comment = True
+            out[index] = " "
+        elif char in ("'", '"'):
+            quote = char
+    return "".join(out)
+
+
+def _matching_delimiter(text: str, start: int, opening: str, closing: str) -> int | None:
+    depth = 1
+    for index in range(start, len(text)):
+        if text[index] == opening:
+            depth += 1
+        elif text[index] == closing:
+            depth -= 1
+            if depth == 0:
+                return index
+    return None
+
+
+def extract_r(path: Path) -> dict:
+    """Extract R package imports, named functions, and local direct calls."""
+    try:
+        source = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        return {"nodes": [], "edges": [], "error": str(exc)}
+
+    masked = _mask_non_code(source)
+    str_path = str(path)
+    stem = _file_stem(path)
+    file_nid = _make_id(str_path)
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    raw_calls: list[dict] = []
+    seen_ids: set[str] = set()
+
+    def line_at(offset: int) -> int:
+        return source.count("\n", 0, offset) + 1
+
+    def add_node(nid: str, label: str, line: int) -> None:
+        if nid not in seen_ids:
+            seen_ids.add(nid)
+            nodes.append(
+                {
+                    "id": nid,
+                    "label": label,
+                    "file_type": "code",
+                    "source_file": str_path,
+                    "source_location": f"L{line}",
+                }
+            )
+
+    def add_edge(src: str, tgt: str, relation: str, line: int) -> None:
+        edges.append(
+            {
+                "source": src,
+                "target": tgt,
+                "relation": relation,
+                "confidence": "EXTRACTED",
+                "source_file": str_path,
+                "source_location": f"L{line}",
+                "weight": 1.0,
+            }
+        )
+
+    add_node(file_nid, path.name, 1)
+
+    for match in _IMPORT.finditer(masked):
+        arguments_end = _matching_delimiter(masked, match.end(), "(", ")")
+        if arguments_end is None:
+            continue
+        tail = source[match.end():arguments_end]
+        package = _PACKAGE.match(tail)
+        if package:
+            add_edge(file_nid, _make_id(package.group(1)), "imports", line_at(match.start()))
+
+    function_specs: list[tuple[int, int, str]] = []
+    for match in _FUNCTION.finditer(masked):
+        parameters_end = _matching_delimiter(masked, match.end(), "(", ")")
+        if parameters_end is None:
+            continue
+        body_start = parameters_end + 1
+        while body_start < len(masked) and masked[body_start] in " \t\r\n":
+            body_start += 1
+        if body_start < len(masked) and masked[body_start] == "{":
+            end = _matching_delimiter(masked, body_start + 1, "{", "}")
+            if end is None:
+                continue
+        else:
+            newline = masked.find("\n", body_start)
+            end = len(masked) - 1 if newline < 0 else newline
+        function_specs.append((match.start(), end, match.group(1)))
+
+    functions: list[dict] = []
+    for start, end, name in function_specs:
+        parent = min(
+            (
+                function
+                for function in functions
+                if function["start"] < start and end <= function["end"]
+            ),
+            key=lambda function: function["end"] - function["start"],
+            default=None,
+        )
+        parent_id = parent["id"] if parent else None
+        nid = _make_id(parent_id or stem, name)
+        line = line_at(start)
+        add_node(nid, f"{name}()", line)
+        add_edge(parent_id or file_nid, nid, "contains", line)
+        functions.append(
+            {
+                "start": start,
+                "end": end,
+                "id": nid,
+                "name": name,
+                "parent": parent_id,
+            }
+        )
+
+    labels: dict[str, list[dict]] = {}
+    for function in functions:
+        labels.setdefault(function["name"], []).append(function)
+
+    def call_target(callee: str, caller: dict) -> str | None:
+        candidates = [
+            candidate
+            for candidate in labels.get(callee, [])
+            if candidate["id"] != caller["id"]
+        ]
+        children = [
+            candidate for candidate in candidates if candidate["parent"] == caller["id"]
+        ]
+        if len(children) == 1:
+            return children[0]["id"]
+        same_scope = [
+            candidate for candidate in candidates if candidate["parent"] == caller["parent"]
+        ]
+        if len(same_scope) == 1:
+            return same_scope[0]["id"]
+        if len(candidates) == 1:
+            return candidates[0]["id"]
+        return None
+
+    seen_calls: set[tuple[str, str]] = set()
+    for function in functions:
+        start = function["start"]
+        end = function["end"]
+        caller = function["id"]
+        body = list(masked[start:end + 1])
+        for nested in functions:
+            nested_start = nested["start"]
+            nested_end = nested["end"]
+            if start < nested_start and nested_end <= end:
+                for index in range(nested_start, nested_end + 1):
+                    if body[index - start] != "\n":
+                        body[index - start] = " "
+        body_text = "".join(body)
+        for call in _CALL.finditer(body_text):
+            expression = call.group(1)
+            callee = re.split(r"::|\$", expression)[-1]
+            if callee in _NON_CALLS:
+                continue
+            absolute = start + call.start(1)
+            target = call_target(callee, function)
+            if target:
+                pair = (caller, target)
+                if pair not in seen_calls:
+                    seen_calls.add(pair)
+                    add_edge(caller, target, "calls", line_at(absolute))
+            elif callee:
+                raw_calls.append(
+                    {
+                        "caller_nid": caller,
+                        "callee": callee,
+                        "is_member_call": "$" in expression,
+                        "source_file": str_path,
+                        "source_location": f"L{line_at(absolute)}",
+                    }
+                )
+
+    clean_edges = [
+        edge
+        for edge in edges
+        if edge["source"] in seen_ids
+        and (edge["target"] in seen_ids or edge["relation"] == "imports")
+    ]
+    return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}

tests/fixtures/sample.r

@@ -0,0 +1,32 @@
+library(
+  package = dplyr
+)
+requireNamespace("jsonlite")
+
+normalize_values <- function(
+  values,
+  trim = 0
+) {
+  mean(values, trim = trim)
+}
+
+identity_value <<- function(value) value
+
+make_scaler <- function(scale) {
+  scale_one <- function(value) {
+    value * scale
+  }
+  scale_one(scale)
+}
+
+analyze <- function(values) {
+  normalized <- normalize_values(values)
+  identity_value(normalized)
+  stats::median(normalized)
+}
+
+# Calls in comments and strings are not executable dependencies:
+description <- "normalize_values(fake)
+fake <- function(value) {
+  identity_value(value)
+}"

tests/test_extract.py

@@ -2268,18 +2268,18 @@ def test_case_insensitive_suffix_filtering(tmp_path):
 
 
 def test_extract_warns_on_code_files_with_no_ast_extractor(tmp_path, capsys):
-    # #1689: .r/.R is in CODE_EXTENSIONS (counted as code) but has no AST extractor,
-    # so R files silently contribute nothing. extract() must surface that instead of
-    # reporting success as if the language were mapped.
-    r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n")
-    r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n")
+    # #1689: .ejs is in CODE_EXTENSIONS (counted as code) but has no AST extractor,
+    # so those files silently contribute nothing. extract() must surface that instead
+    # of reporting success as if the language were mapped.
+    ejs1 = tmp_path / "analysis.ejs"; ejs1.write_text("<%= result %>\n")
+    ejs2 = tmp_path / "helper.ejs"; ejs2.write_text("<%= helper() %>\n")
     py = tmp_path / "main.py"; py.write_text("def main():\n    return 1\n")
 
-    result = extract([r1, r2, py], cache_root=tmp_path)
+    result = extract([ejs1, ejs2, py], cache_root=tmp_path)
     err = capsys.readouterr().err
 
     assert "no AST extractor" in err
-    assert ".r (2)" in err            # both R files grouped under the lowercased ext
+    assert ".ejs (2)" in err
     assert "#1689" in err
     # the Python file still extracts normally
     labels = [n.get("label") for n in result["nodes"]]
@@ -2330,8 +2330,8 @@ def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsy
     for i in range(100):
         (tmp_path / f"m{i}.py").write_text(f"def f{i}():\n    return {i}\n")
     for i in range(5):
-        (tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n")  # no extractor
-    paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r"))  # total 105
+        (tmp_path / f"s{i}.ejs").write_text("<%= result %>\n")  # no extractor
+    paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.ejs"))  # total 105
 
     extract(paths, cache_root=tmp_path, parallel=False)
     out = capsys.readouterr().out