diff --git a/graphify/extract.py b/graphify/extract.py index e5a48dc04..e84b9a7b8 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -47,6 +47,7 @@ from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 +from graphify.extractors.r import extract_r # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 from graphify.extractors.sql import extract_sql # noqa: F401 @@ -4024,6 +4025,7 @@ def add_existing_edge(edge: dict) -> None: ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".r": extract_r, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, @@ -4080,7 +4082,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (perl, fish, tcsh) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -4096,6 +4098,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "Rscript": extract_r, } @@ -4545,8 +4548,8 @@ def extract( ) # #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST - # extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently - # contributes zero nodes. The #1666 warning above deliberately skips these (it + # extractor wired up silently contributes zero nodes. The #1666 warning above + # deliberately skips these (it # only fires when an extractor exists), so surface them explicitly, grouped by # extension, rather than reporting success as if the language were mapped. from graphify.detect import CODE_EXTENSIONS as _CODE_EXTS diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..452350878 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -26,6 +26,7 @@ from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor +from graphify.extractors.r import extract_r from graphify.extractors.rust import extract_rust from graphify.extractors.sln import extract_sln from graphify.extractors.sql import extract_sql @@ -55,6 +56,7 @@ "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, + "r": extract_r, "rust": extract_rust, "sln": extract_sln, "sql": extract_sql, diff --git a/graphify/extractors/r.py b/graphify/extractors/r.py new file mode 100644 index 000000000..b0438a241 --- /dev/null +++ b/graphify/extractors/r.py @@ -0,0 +1,244 @@ +"""Deterministic R extraction shared with Compass. + +R is intentionally source-driven: Graphify does not require an optional R +grammar at runtime, while Compass keeps its grammar bundle static. Keeping the +small structural contract here avoids a classified-as-code file disappearing +from one implementation while preserving the useful R relationships: package +imports, named functions, and direct calls between local functions. +""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_FUNCTION = re.compile( + r"(?m)^[ \t]*([A-Za-z.][A-Za-z0-9._]*)[ \t]*(?:<<-|<-|=)[ \t]*function[ \t]*\(" +) +_IMPORT = re.compile( + r"(?m)^[ \t]*(?:library|require|requireNamespace)[ \t]*\(" +) +_PACKAGE = re.compile( + r"\s*(?:package\s*=\s*)?[\"']?([A-Za-z][A-Za-z0-9._-]*)" +) +_CALL = re.compile( + r"([A-Za-z.][A-Za-z0-9._]*(?:(?:::|\$)[A-Za-z.][A-Za-z0-9._]*)?)[ \t]*\(" +) +_NON_CALLS = frozenset({"function", "if", "for", "while", "repeat", "switch", "return"}) + + +def _mask_non_code(source: str) -> str: + """Blank comments and strings without changing byte offsets or line numbers.""" + out = list(source) + quote: str | None = None + escaped = False + comment = False + for index, char in enumerate(source): + if comment: + if char == "\n": + comment = False + else: + out[index] = " " + continue + if quote is not None: + if char == "\n": + continue + if escaped: + escaped = False + out[index] = " " + elif char == "\\": + escaped = True + out[index] = " " + elif char == quote: + quote = None + else: + out[index] = " " + continue + if char == "#": + comment = True + out[index] = " " + elif char in ("'", '"'): + quote = char + return "".join(out) + + +def _matching_delimiter(text: str, start: int, opening: str, closing: str) -> int | None: + depth = 1 + for index in range(start, len(text)): + if text[index] == opening: + depth += 1 + elif text[index] == closing: + depth -= 1 + if depth == 0: + return index + return None + + +def extract_r(path: Path) -> dict: + """Extract R package imports, named functions, and local direct calls.""" + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + + masked = _mask_non_code(source) + str_path = str(path) + stem = _file_stem(path) + file_nid = _make_id(str_path) + nodes: list[dict] = [] + edges: list[dict] = [] + raw_calls: list[dict] = [] + seen_ids: set[str] = set() + + def line_at(offset: int) -> int: + return source.count("\n", 0, offset) + 1 + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append( + { + "id": nid, + "label": label, + "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}", + } + ) + + def add_edge(src: str, tgt: str, relation: str, line: int) -> None: + edges.append( + { + "source": src, + "target": tgt, + "relation": relation, + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + } + ) + + add_node(file_nid, path.name, 1) + + for match in _IMPORT.finditer(masked): + arguments_end = _matching_delimiter(masked, match.end(), "(", ")") + if arguments_end is None: + continue + tail = source[match.end():arguments_end] + package = _PACKAGE.match(tail) + if package: + add_edge(file_nid, _make_id(package.group(1)), "imports", line_at(match.start())) + + function_specs: list[tuple[int, int, str]] = [] + for match in _FUNCTION.finditer(masked): + parameters_end = _matching_delimiter(masked, match.end(), "(", ")") + if parameters_end is None: + continue + body_start = parameters_end + 1 + while body_start < len(masked) and masked[body_start] in " \t\r\n": + body_start += 1 + if body_start < len(masked) and masked[body_start] == "{": + end = _matching_delimiter(masked, body_start + 1, "{", "}") + if end is None: + continue + else: + newline = masked.find("\n", body_start) + end = len(masked) - 1 if newline < 0 else newline + function_specs.append((match.start(), end, match.group(1))) + + functions: list[dict] = [] + for start, end, name in function_specs: + parent = min( + ( + function + for function in functions + if function["start"] < start and end <= function["end"] + ), + key=lambda function: function["end"] - function["start"], + default=None, + ) + parent_id = parent["id"] if parent else None + nid = _make_id(parent_id or stem, name) + line = line_at(start) + add_node(nid, f"{name}()", line) + add_edge(parent_id or file_nid, nid, "contains", line) + functions.append( + { + "start": start, + "end": end, + "id": nid, + "name": name, + "parent": parent_id, + } + ) + + labels: dict[str, list[dict]] = {} + for function in functions: + labels.setdefault(function["name"], []).append(function) + + def call_target(callee: str, caller: dict) -> str | None: + candidates = [ + candidate + for candidate in labels.get(callee, []) + if candidate["id"] != caller["id"] + ] + children = [ + candidate for candidate in candidates if candidate["parent"] == caller["id"] + ] + if len(children) == 1: + return children[0]["id"] + same_scope = [ + candidate for candidate in candidates if candidate["parent"] == caller["parent"] + ] + if len(same_scope) == 1: + return same_scope[0]["id"] + if len(candidates) == 1: + return candidates[0]["id"] + return None + + seen_calls: set[tuple[str, str]] = set() + for function in functions: + start = function["start"] + end = function["end"] + caller = function["id"] + body = list(masked[start:end + 1]) + for nested in functions: + nested_start = nested["start"] + nested_end = nested["end"] + if start < nested_start and nested_end <= end: + for index in range(nested_start, nested_end + 1): + if body[index - start] != "\n": + body[index - start] = " " + body_text = "".join(body) + for call in _CALL.finditer(body_text): + expression = call.group(1) + callee = re.split(r"::|\$", expression)[-1] + if callee in _NON_CALLS: + continue + absolute = start + call.start(1) + target = call_target(callee, function) + if target: + pair = (caller, target) + if pair not in seen_calls: + seen_calls.add(pair) + add_edge(caller, target, "calls", line_at(absolute)) + elif callee: + raw_calls.append( + { + "caller_nid": caller, + "callee": callee, + "is_member_call": "$" in expression, + "source_file": str_path, + "source_location": f"L{line_at(absolute)}", + } + ) + + clean_edges = [ + edge + for edge in edges + if edge["source"] in seen_ids + and (edge["target"] in seen_ids or edge["relation"] == "imports") + ] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} diff --git a/tests/fixtures/sample.r b/tests/fixtures/sample.r new file mode 100644 index 000000000..5a8911643 --- /dev/null +++ b/tests/fixtures/sample.r @@ -0,0 +1,32 @@ +library( + package = dplyr +) +requireNamespace("jsonlite") + +normalize_values <- function( + values, + trim = 0 +) { + mean(values, trim = trim) +} + +identity_value <<- function(value) value + +make_scaler <- function(scale) { + scale_one <- function(value) { + value * scale + } + scale_one(scale) +} + +analyze <- function(values) { + normalized <- normalize_values(values) + identity_value(normalized) + stats::median(normalized) +} + +# Calls in comments and strings are not executable dependencies: +description <- "normalize_values(fake) +fake <- function(value) { + identity_value(value) +}" diff --git a/tests/test_extract.py b/tests/test_extract.py index 824419edd..8d6000eb7 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2268,18 +2268,18 @@ def test_case_insensitive_suffix_filtering(tmp_path): def test_extract_warns_on_code_files_with_no_ast_extractor(tmp_path, capsys): - # #1689: .r/.R is in CODE_EXTENSIONS (counted as code) but has no AST extractor, - # so R files silently contribute nothing. extract() must surface that instead of - # reporting success as if the language were mapped. - r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") - r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + # #1689: .ejs is in CODE_EXTENSIONS (counted as code) but has no AST extractor, + # so those files silently contribute nothing. extract() must surface that instead + # of reporting success as if the language were mapped. + ejs1 = tmp_path / "analysis.ejs"; ejs1.write_text("<%= result %>\n") + ejs2 = tmp_path / "helper.ejs"; ejs2.write_text("<%= helper() %>\n") py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") - result = extract([r1, r2, py], cache_root=tmp_path) + result = extract([ejs1, ejs2, py], cache_root=tmp_path) err = capsys.readouterr().err assert "no AST extractor" in err - assert ".r (2)" in err # both R files grouped under the lowercased ext + assert ".ejs (2)" in err assert "#1689" in err # the Python file still extracts normally labels = [n.get("label") for n in result["nodes"]] @@ -2330,8 +2330,8 @@ def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsy for i in range(100): (tmp_path / f"m{i}.py").write_text(f"def f{i}():\n return {i}\n") for i in range(5): - (tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n") # no extractor - paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r")) # total 105 + (tmp_path / f"s{i}.ejs").write_text("<%= result %>\n") # no extractor + paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.ejs")) # total 105 extract(paths, cache_root=tmp_path, parallel=False) out = capsys.readouterr().out diff --git a/tests/test_r_extractor.py b/tests/test_r_extractor.py new file mode 100644 index 000000000..23b7e2639 --- /dev/null +++ b/tests/test_r_extractor.py @@ -0,0 +1,70 @@ +"""Tests for deterministic R extraction and extensionless Rscript routing.""" +from pathlib import Path + +from graphify.extract import _get_extractor, extract_r + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_r_extracts_imports_functions_nested_scope_and_calls() -> None: + result = extract_r(FIXTURES / "sample.r") + + labels = {node["label"] for node in result["nodes"]} + assert { + "sample.r", + "normalize_values()", + "identity_value()", + "make_scaler()", + "scale_one()", + "analyze()", + } <= labels + ids = {node["label"]: node["id"] for node in result["nodes"]} + + relations = { + (edge["source"], edge["target"], edge["relation"]) + for edge in result["edges"] + } + assert (ids["sample.r"], "dplyr", "imports") in relations + assert (ids["sample.r"], "jsonlite", "imports") in relations + assert ( + ids["make_scaler()"], + ids["scale_one()"], + "contains", + ) in relations + assert ( + ids["analyze()"], + ids["normalize_values()"], + "calls", + ) in relations + assert ( + ids["analyze()"], + ids["identity_value()"], + "calls", + ) in relations + + raw_calls = {(call["caller_nid"], call["callee"]) for call in result["raw_calls"]} + assert (ids["analyze()"], "median") in raw_calls + assert not any(callee == "fake" for _, callee in raw_calls) + + +def test_extensionless_rscript_routes_to_r_extractor(tmp_path: Path) -> None: + script = tmp_path / "analyze" + script.write_text( + "#!/usr/bin/env Rscript\n" + "run <- function(values) {\n" + " mean(values)\n" + "}\n" + ) + + assert _get_extractor(script) is extract_r + result = extract_r(script) + assert any(node["label"] == "run()" for node in result["nodes"]) + + +def test_r_invalid_utf8_is_replaced_without_dropping_symbols(tmp_path: Path) -> None: + source = tmp_path / "legacy.r" + source.write_bytes(b"# legacy annotation: \xff\nrun <- function(value) value\n") + + result = extract_r(source) + assert "error" not in result + assert any(node["label"] == "run()" for node in result["nodes"])