diff --git a/docs/user_guide/embedded_code.md b/docs/user_guide/embedded_code.md new file mode 100644 index 000000000..05c45b7dc --- /dev/null +++ b/docs/user_guide/embedded_code.md @@ -0,0 +1,62 @@ +# Embedded Robot Framework code + +Robocop can lint and format Robot Framework code that is embedded in other files: + +- **Markdown** files (`.md`, `.markdown`) – fenced code blocks tagged with ` ```robotframework ` (or ` ```robot `). +- **Python** files (`.py`) – the same fenced blocks placed inside docstrings, as used for runnable examples in + library documentation. + +This mirrors the embedded execution support that Robot Framework added in version 7.5, but Robocop does its own +parsing, so it works regardless of the installed Robot Framework version. + +A single file may contain zero, one, or many code blocks. **Each block is treated as an independent suite** – it is +linted and formatted on its own, and the surrounding prose, fences and indentation are left untouched. + +````markdown +# Example + +Some documentation describing the test below. + +```robotframework +*** Test Cases *** +Example Test + Log message +``` +```` + +## Opting in + +Embedded files are **not** analyzed by default. There are two ways to opt in: + +1. **Pass the file directly** on the command line: + + ```bash + robocop check docs/example.md + robocop format library.py + ``` + +2. **Add the extension to the include patterns** so the files are picked up during directory scans: + + ```bash + robocop check --include "*.md" --include "*.py" . + ``` + + or in the configuration file: + + ```toml + [tool.robocop] + include = ["*.robot", "*.resource", "*.md"] + ``` + +## Positions and fixes + +Reported issues point at the **exact physical line and column** of the code inside the original file, not at a +position within an extracted block. Automatic fixes (`--fix`) and formatting are written back in place, preserving +the block indentation (for example the indentation of a fenced block inside a Python docstring) and the file's line +endings. + +## Notes + +- Because every block is an independent suite, suite-level rules (such as *missing documentation in suite*) are + reported once per block. +- Blocks tagged with any language other than `robotframework`/`robot` are ignored. diff --git a/mkdocs.yml b/mkdocs.yml index 60c399a17..eaf203d9f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,7 @@ nav: - Robocop: index.md - User Guide: - "Getting started": user_guide/intro.md + - "Embedded code (Markdown, Python)": user_guide/embedded_code.md - "Migrate to Robocop 6.0": user_guide/migrate_to_robocop6.md - "Python API Reference": user_guide/python_api.md - Linter: diff --git a/pyproject.toml b/pyproject.toml index f888687bf..e9138b2c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,6 +183,7 @@ lint.unfixable = [ ] extend-exclude = [ "tests/linter/test_data/", + "tests/embedded/test_data/", "docs/" ] [tool.ruff.lint.extend-per-file-ignores] diff --git a/src/robocop/config/manager.py b/src/robocop/config/manager.py index 30a839986..8b7efdd3f 100644 --- a/src/robocop/config/manager.py +++ b/src/robocop/config/manager.py @@ -11,7 +11,7 @@ from robocop.config.builder import ConfigBuilder from robocop.config.parser import read_toml_config from robocop.config.schema import Config, RawConfig -from robocop.source_file import SourceFile +from robocop.source_file import SourceFile, build_source_file if TYPE_CHECKING: from collections.abc import Generator, Sequence @@ -291,4 +291,4 @@ def resolve_paths( if source.is_dir(): self.resolve_paths(list(source.iterdir()), target=target) elif source.is_file(): - target[source] = self._paths.get(source) or SourceFile(path=source, config=config) + target[source] = self._paths.get(source) or build_source_file(source, config) diff --git a/src/robocop/embedded.py b/src/robocop/embedded.py new file mode 100644 index 000000000..ef81cde2c --- /dev/null +++ b/src/robocop/embedded.py @@ -0,0 +1,220 @@ +""" +Extraction of embedded Robot Framework code blocks from Markdown and Python files. + +Robot Framework 7.5 added support for executing Robot code embedded in Markdown files. The code lives in fenced +code blocks tagged with ``robotframework`` (or ``robot``):: + + ```robotframework + *** Test Cases *** + Example + Log message + ``` + +The same convention is used to embed runnable examples in Python docstrings (see Robot Framework's own +``Collections`` library). Robocop supports both sources. + +Unlike Robot Framework -- which concatenates every block and throws away line positions -- Robocop needs to keep +the exact physical position (line and column) of each block so that reported issues and applied fixes point at the +right place in the original file. This module extracts blocks while preserving that information. + +The extraction is intentionally line based (it does not parse Python syntax). A fenced ``robotframework`` block is +recognized wherever it appears, which covers both Markdown prose and Python docstrings. Blocks nested inside +indented docstrings keep their common indentation, which is stripped when the block is turned into a Robot model +and re-applied when the file is written back. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from robot.api.parsing import ModelVisitor + +if TYPE_CHECKING: + from robot.parsing.model import File + from robot.parsing.model.statements import Statement + +# Supported code block languages (case-insensitive), mirroring Robot Framework's Markdown parser. +ROBOT_BLOCK_LANGUAGES = frozenset({"robotframework", "robot"}) + +# File extensions that may contain embedded Robot Framework code blocks. +MARKDOWN_EXTENSIONS = frozenset({".md", ".markdown"}) +PYTHON_EXTENSIONS = frozenset({".py"}) +EMBEDDED_EXTENSIONS = MARKDOWN_EXTENSIONS | PYTHON_EXTENSIONS + +# Opening fence: optional indentation, at least three backticks or tildes, then the language token. +_FENCE_OPEN = re.compile(r"(?P[ \t]*)(?P`{3,}|~{3,})[ \t]*(?P\S+)") + + +@dataclass +class RobotCodeBlock: + """ + A single embedded Robot Framework code block located inside a Markdown or Python file. + + Attributes: + start_line: 1-indexed physical line number of the first code line (line just after the opening fence). + end_line: 1-indexed physical line number of the last code line (inclusive). For an empty block this is + ``start_line - 1`` so that ``range(start_line, end_line + 1)`` is empty. + indent: Common leading whitespace shared by every non-blank code line. It is stripped when building the + Robot model and re-applied when writing changes back to the file. + fence_open_line: 1-indexed physical line number of the opening fence. + fence_close_line: 1-indexed physical line number of the closing fence, or ``None`` if the block is closed + implicitly by the end of the file. + + """ + + start_line: int + end_line: int + indent: str + fence_open_line: int + fence_close_line: int | None + + @property + def is_empty(self) -> bool: + return self.end_line < self.start_line + + def contains(self, lineno: int) -> bool: + """Return whether the given 1-indexed physical line is a code line of this block.""" + return self.start_line <= lineno <= self.end_line + + +def is_embedded_extension(suffix: str) -> bool: + """Return whether files with the given suffix may contain embedded Robot Framework code blocks.""" + return suffix.lower() in EMBEDDED_EXTENSIONS + + +def _common_indent(lines: list[str]) -> str: + """ + Return the longest leading whitespace shared by every non-blank line. + + Blank lines (only whitespace or empty) are ignored, mirroring :func:`textwrap.dedent`. + """ + common: str | None = None + for line in lines: + stripped = line.lstrip(" \t") + if not stripped or stripped in ("\n", "\r\n", "\r"): + continue + indent = line[: len(line) - len(stripped)] + if common is None: + common = indent + continue + # Shrink the common prefix to the shared portion. + shared_len = 0 + for a, b in zip(common, indent, strict=False): + if a != b: + break + shared_len += 1 + common = common[:shared_len] + if not common: + break + return common or "" + + +def extract_robot_blocks(lines: list[str]) -> list[RobotCodeBlock]: + """ + Find all embedded Robot Framework code blocks in the given file lines. + + Args: + lines: The physical file lines, including line endings (as returned by ``file.readlines()``). + + Returns: + List of :class:`RobotCodeBlock`, in file order. Empty if the file has no Robot code blocks. + + """ + blocks: list[RobotCodeBlock] = [] + total = len(lines) + index = 0 + while index < total: + match = _FENCE_OPEN.match(lines[index]) + if not match or match.group("lang").lower() not in ROBOT_BLOCK_LANGUAGES: + index += 1 + continue + fence = match.group("fence") + fence_char = re.escape(fence[0]) + # Closing fence: at least as many of the same fence character, optionally surrounded by whitespace. + close_re = re.compile(rf"[ \t]*{fence_char}{{{len(fence)},}}[ \t]*$") + fence_open_line = index + 1 + code_lines: list[str] = [] + cursor = index + 1 + while cursor < total and not close_re.match(lines[cursor].rstrip("\r\n")): + code_lines.append(lines[cursor]) + cursor += 1 + start_line = index + 2 + end_line = cursor # cursor is the closing fence (or EOF); last code line is cursor - 1 (0-indexed) -> cursor + fence_close_line = cursor + 1 if cursor < total else None + blocks.append( + RobotCodeBlock( + start_line=start_line, + end_line=end_line, + indent=_common_indent(code_lines), + fence_open_line=fence_open_line, + fence_close_line=fence_close_line, + ) + ) + index = cursor + 1 + return blocks + + +def strip_indent(line: str, indent: str) -> str: + """Remove up to ``indent`` leading characters from ``line`` (only if the line starts with that indentation).""" + if indent and line.startswith(indent): + return line[len(indent) :] + # Blank or shorter-indented lines: drop whatever leading whitespace overlaps the common indent. + stripped = line.lstrip(" \t") + leading = line[: len(line) - len(stripped)] + keep = leading[len(indent) :] if len(leading) > len(indent) else "" + return keep + stripped + + +def reconstruct_source(lines: list[str], blocks: list[RobotCodeBlock]) -> str: + """ + Build a Robot Framework source string that preserves physical line numbers. + + Non-code lines (prose, fences, Python code) are replaced with blank lines and code lines are dedented by their + block indentation. The resulting model therefore has line numbers matching the original file, while section + headers and statements start at column 0 as Robot Framework requires. + + Args: + lines: The physical file lines, including line endings. + blocks: Blocks extracted from ``lines`` via :func:`extract_robot_blocks`. + + Returns: + Robot Framework source text with one entry per physical line. + + """ + reconstructed = ["\n"] * len(lines) + for block in blocks: + for lineno in range(block.start_line, block.end_line + 1): + original = lines[lineno - 1] + newline = _line_ending(original) + reconstructed[lineno - 1] = strip_indent(original.rstrip("\r\n"), block.indent) + newline + return "".join(reconstructed) + + +def _line_ending(line: str) -> str: + if line.endswith("\r\n"): + return "\r\n" + if line.endswith("\n"): + return "\n" + if line.endswith("\r"): + return "\r" + return "" + + +class _LineShifter(ModelVisitor): # type: ignore[misc] + """Shift every token line number by a fixed offset, turning block-local lines into physical file lines.""" + + def __init__(self, offset: int) -> None: + self.offset = offset + + def visit_Statement(self, node: Statement) -> None: # noqa: N802 + for token in node.tokens: + token.lineno += self.offset + + +def shift_model_lines(model: File, offset: int) -> File: + """Add ``offset`` to every token line number in the model (in place) and return it.""" + if offset: + _LineShifter(offset).visit(model) + return model diff --git a/src/robocop/formatter/runner.py b/src/robocop/formatter/runner.py index 18996dd82..533aa71e0 100644 --- a/src/robocop/formatter/runner.py +++ b/src/robocop/formatter/runner.py @@ -68,6 +68,18 @@ def run(self) -> int: cached_files += 1 continue previous_changed_files = changed_files + if source_file.is_embedded: + diff, old_text, new_text = self.format_embedded(source_file) + if diff: + self.save_text(source_file.path, new_text) + self.log_formatted_source(source_file.path, stdin) + self.output_diff_text(source_file.path, old_text, new_text) + changed_files += 1 + if not diff or self.config.formatter.overwrite: + self.config_manager.cache.set_formatter_entry( + source_file.path, source_file.config.hash, needs_formatting=False + ) + continue diff, old_model, new_model, model = self.format_until_stable(source_file) # if stdin: # self.print_to_stdout(new_model) @@ -150,6 +162,87 @@ def format( new_model = StatementLinesCollector(model) return new_model != old_model, old_model, new_model + def format_embedded(self, source_file: SourceFile) -> tuple[bool, str, str]: + """ + Format every Robot code block embedded in a Markdown or Python file independently. + + Each block is formatted on its own, re-indented to its original position and spliced back into the physical + file, leaving prose, fences and indentation of the surrounding document untouched. + + Returns: + Tuple of (changed, original_text, new_text) with the full physical file contents. + + """ + resolved_config = self.config_resolver.resolve_config(source_file.config) + original = list(source_file.source_lines) + physical = list(original) + newline = self._detect_newline(original) + # Blocks are spliced bottom-to-top so that changing the length of one block does not shift the others. + for block, model in reversed(source_file.format_blocks()): # type: ignore[attr-defined] + formatted = self.format_block(model, resolved_config) + if formatted is None: + continue + physical[block.start_line - 1 : block.end_line] = self._reindent_block(formatted, block.indent, newline) + new_text = "".join(physical) + old_text = "".join(original) + return new_text != old_text, old_text, new_text + + def format_block(self, model: File, resolved_config: ResolvedConfig) -> str | None: + """ + Run the configured formatters on a single block model. + + Returns: + The formatted block text, or ``None`` if the block is unchanged or fully disabled. + + """ + disabler_finder = disablers.RegisterDisablers(self.config.formatter.start_line, self.config.formatter.end_line) + disabler_finder.visit(model) + if disabler_finder.is_disabled_in_file(disablers.ALL_FORMATTERS): + return None + original_text = StatementLinesCollector(model).text + formatted = self._apply_formatters(model, disabler_finder.disablers, resolved_config) + reruns = self.config.formatter.reruns + while formatted != original_text and reruns: + rerun_model = get_model(formatted) + rerun_disablers = disablers.RegisterDisablers( + self.config.formatter.start_line, self.config.formatter.end_line + ) + rerun_disablers.visit(rerun_model) + new_formatted = self._apply_formatters(rerun_model, rerun_disablers.disablers, resolved_config) + if new_formatted == formatted: + break + formatted = new_formatted + reruns -= 1 + return formatted if formatted != original_text else None + + @staticmethod + def _apply_formatters( + model: File, disablers_in_file: disablers.DisablersInFile, resolved_config: ResolvedConfig + ) -> str: + for name, formatter in resolved_config.formatters.items(): + formatter.disablers = disablers_in_file + if disablers_in_file.is_disabled_in_file(name): + continue + formatter.visit(model) + return StatementLinesCollector(model).text + + @staticmethod + def _detect_newline(lines: list[str]) -> str: + """Return the line ending used by the physical file so formatted blocks keep the surrounding style.""" + for line in lines: + if line.endswith("\r\n"): + return "\r\n" + if line.endswith("\n"): + return "\n" + if line.endswith("\r"): + return "\r" + return os.linesep + + @staticmethod + def _reindent_block(text: str, indent: str, newline: str) -> list[str]: + """Re-apply the block indentation and the file's line ending to formatted lines, leaving blanks unindented.""" + return [f"{indent}{line}{newline}" if line.strip() else f"{line}{newline}" for line in text.splitlines()] + def log_formatted_source(self, source: Path, stdin: bool) -> None: if stdin or self.config.silent: return @@ -171,6 +264,28 @@ def save_model(self, source: Path, model: File) -> None: output = self.config.formatter.output or source misc.ModelWriter(output=str(output), newline=self.get_line_ending(str(source))).write(model) + def save_text(self, source: Path, text: str) -> None: + """Write the full physical file content of an embedded file, preserving its original line endings.""" + if self.config.formatter.overwrite: + output = self.config.formatter.output or source + with open(output, "w", encoding="utf-8", newline="") as f: + f.write(text) + + def output_diff_text(self, path: Path, old_text: str, new_text: str) -> None: + if not self.config.formatter.diff: + return + old = [line + "\n" for line in old_text.splitlines()] + new = [line + "\n" for line in new_text.splitlines()] + lines = list(unified_diff(old, new, fromfile=f"{path}\tbefore", tofile=f"{path}\tafter")) + if not lines: + return + if self.config.formatter.color: + output = misc.decorate_diff_with_color(lines) + else: + output = misc.escape_rich_markup(lines) + for line in output: + console.print(line, end="", highlight=False, soft_wrap=True) + def get_line_ending(self, path: str) -> str: if self.config.formatter.whitespace_config.line_ending == "auto": with open(path) as f: diff --git a/src/robocop/linter/checkers/raw_file.py b/src/robocop/linter/checkers/raw_file.py index 2b84676c6..65c136367 100644 --- a/src/robocop/linter/checkers/raw_file.py +++ b/src/robocop/linter/checkers/raw_file.py @@ -31,7 +31,8 @@ class RawFileRulesChecker(RawFileChecker): def parse_file(self) -> None: self.lines = self.source_file.source_lines - if self.ignored_data.enabled or self.bom_encoding_in_file.enabled: + is_embedded = self.source_file.is_embedded + if not is_embedded and (self.ignored_data.enabled or self.bom_encoding_in_file.enabled): is_bom = detect_bom(self.source_file.path) self.bom_encoding_in_file.check(is_bom) self.ignored_data.check(self.lines, is_bom) @@ -39,11 +40,12 @@ def parse_file(self) -> None: doc_lines: frozenset[int] = frozenset() if self.line_too_long.enabled and self.line_too_long.ignore_docs: doc_lines = self.line_too_long.get_documentation_lines(self.source_file.model) - for lineno, line in enumerate(self.lines, start=1): + for lineno, line in self.source_file.check_lines: self.trailing_whitespace.check(line, lineno) self.line_too_long.check(line, lineno, doc_lines) - self.too_many_trailing_blank_lines.check(self.lines) - self.missing_trailing_blank_line.check(self.lines) + if not is_embedded: + self.too_many_trailing_blank_lines.check(self.lines) + self.missing_trailing_blank_line.check(self.lines) def detect_bom(source: Path) -> bool: diff --git a/src/robocop/linter/fix.py b/src/robocop/linter/fix.py index 7122eeaf9..a887c3409 100644 --- a/src/robocop/linter/fix.py +++ b/src/robocop/linter/fix.py @@ -319,6 +319,7 @@ def apply_fixes(self, source_file: SourceFile, fixes: list[Fix]) -> bool: if not all_edits: return False + all_edits = source_file.prepare_fix_edits(all_edits) sorted_edits = sorted(all_edits, key=lambda e: (e.start_line, e.start_col)) non_overlapping_edits = self._remove_overlapping_edits(sorted_edits) diff --git a/src/robocop/linter/rules/__init__.py b/src/robocop/linter/rules/__init__.py index cfac9e45d..284aadd0f 100644 --- a/src/robocop/linter/rules/__init__.py +++ b/src/robocop/linter/rules/__init__.py @@ -595,7 +595,8 @@ def scan_file(self, source_file: SourceFile, templated: bool = False) -> list[Di self.source_file = source_file self.templated_suite = templated self.context = Context() - self.visit_File(source_file.model) + for model in source_file.models: + self.visit_File(model) return self.issues def visit_File(self, node: File) -> None: # noqa: N802 diff --git a/src/robocop/linter/runner.py b/src/robocop/linter/runner.py index 6fc504dc5..c7e5d978d 100644 --- a/src/robocop/linter/runner.py +++ b/src/robocop/linter/runner.py @@ -145,11 +145,18 @@ def run(self) -> list[Diagnostic]: run_stats = RunStatistic( files_count=files, fix_stats=fix_applier.fix_stats, modified_files=fix_applier.modified_files ) + self.offset_embedded_diagnostics() self.make_reports(run_stats=run_stats) if self.config_manager.default_config.linter.return_result: return self.diagnostics return self.return_with_exit_code(len(self.diagnostics)) + def offset_embedded_diagnostics(self) -> None: + """Translate diagnostic positions of embedded files (Markdown, Python) to physical file coordinates.""" + for diagnostic in self.diagnostics: + if diagnostic.source.is_embedded: + diagnostic.source.offset_range(diagnostic.range) + def run_check(self, source_file: SourceFile, fix_applier: FixApplier | None = None) -> list[Diagnostic]: """ Run all rules on file model and return list of diagnostics. @@ -204,7 +211,7 @@ def run_check(self, source_file: SourceFile, fix_applier: FixApplier | None = No fix_applier.fix_stats.total_fixes += max(prev_fixable - len(fixable_diagnostics), 0) prev_fixable = len(fixable_diagnostics) # Collect fixes from diagnostics - fixes = [diag.fix or diag.rule.fix(diag, source_file.source_lines) for diag in fixable_diagnostics] + fixes = [diag.fix or diag.rule.fix(diag, source_file.fix_source_lines) for diag in fixable_diagnostics] if not fix_applier.apply_fixes(source_file, [fix for fix in fixes if fix]): break if source_file.config.linter.fix and not source_file.config.linter.diff: @@ -317,7 +324,7 @@ def apply_project_fixes( source_file = modified_files.get(resolved_path) or project_files.get(resolved_path) if source_file is None: source_file = source_diagnostics[0].source - fixes = [diag.fix or diag.rule.fix(diag, source_file.source_lines) for diag in source_diagnostics] + fixes = [diag.fix or diag.rule.fix(diag, source_file.fix_source_lines) for diag in source_diagnostics] fixes_before = self.count_applied_fixes(fix_applier) if not fix_applier.apply_fixes(source_file, [fix for fix in fixes if fix]): continue diff --git a/src/robocop/mcp/tools/formatting.py b/src/robocop/mcp/tools/formatting.py index c4758e8fe..3aca87cae 100644 --- a/src/robocop/mcp/tools/formatting.py +++ b/src/robocop/mcp/tools/formatting.py @@ -125,7 +125,7 @@ def _format_file_impl( raise ToolError(f"File not found: {file_path}") if path.suffix not in VALID_EXTENSIONS: - raise ToolError(f"Invalid file type: {path.suffix}. Expected .robot or .resource file.") + raise ToolError(f"Invalid file type: {path.suffix}. Expected one of: {', '.join(sorted(VALID_EXTENSIONS))}.") try: # Read the file content @@ -212,7 +212,9 @@ def _lint_and_format_impl( raise ToolError(f"File not found: {file_path}") if path.suffix not in VALID_EXTENSIONS: - raise ToolError(f"Invalid file type: {path.suffix}. Expected .robot or .resource file.") + raise ToolError( + f"Invalid file type: {path.suffix}. Expected one of: {', '.join(sorted(VALID_EXTENSIONS))}." + ) try: content = path.read_text(encoding="utf-8") diff --git a/src/robocop/mcp/tools/linting.py b/src/robocop/mcp/tools/linting.py index 2bf11e66a..00407e3c9 100644 --- a/src/robocop/mcp/tools/linting.py +++ b/src/robocop/mcp/tools/linting.py @@ -126,7 +126,7 @@ def _lint_file_impl( raise ToolError(f"File not found: {file_path}") if path.suffix not in VALID_EXTENSIONS: - raise ToolError(f"Invalid file type: {path.suffix}. Expected .robot or .resource file.") + raise ToolError(f"Invalid file type: {path.suffix}. Expected one of: {', '.join(sorted(VALID_EXTENSIONS))}.") try: linter_config = _create_linter_config(select, ignore, threshold, configure) diff --git a/src/robocop/mcp/tools/utils/constants.py b/src/robocop/mcp/tools/utils/constants.py index 48e66ca19..7afe511b7 100644 --- a/src/robocop/mcp/tools/utils/constants.py +++ b/src/robocop/mcp/tools/utils/constants.py @@ -2,10 +2,11 @@ from __future__ import annotations +from robocop.embedded import EMBEDDED_EXTENSIONS from robocop.linter.rules import RuleSeverity -# Valid Robot Framework file extensions -VALID_EXTENSIONS = frozenset((".robot", ".resource")) +# Valid Robot Framework file extensions, including files that may embed Robot code (Markdown, Python). +VALID_EXTENSIONS = frozenset((".robot", ".resource")) | EMBEDDED_EXTENSIONS # Threshold string to severity mapping THRESHOLD_MAP = { diff --git a/src/robocop/source_file.py b/src/robocop/source_file.py index aabf9f2d1..17a5a302a 100644 --- a/src/robocop/source_file.py +++ b/src/robocop/source_file.py @@ -11,6 +11,7 @@ except ImportError: Languages = None +from robocop.embedded import extract_robot_blocks, reconstruct_source, shift_model_lines from robocop.files import path_relative_to_cwd, resolve_path from robocop.version_handling import LANG_SUPPORTED @@ -22,6 +23,9 @@ from robot.parsing.model.statements import Statement from robocop.config.schema import Config + from robocop.embedded import RobotCodeBlock + from robocop.linter.diagnostics import Range + from robocop.linter.fix import TextEdit @dataclass @@ -154,6 +158,185 @@ def write_changes(self) -> None: with open(self.path, "w", encoding="utf-8", newline="") as f: f.writelines(self.source_lines) + # Hooks used to support files with embedded Robot Framework code (Markdown, Python). For regular Robot files + # they are no-ops, so the linter and formatter can treat every source file uniformly. + + @property + def is_embedded(self) -> bool: + """Whether the source file holds Robot code embedded in another format (Markdown, Python).""" + return False + + @property + def models(self) -> list[File]: + """Models visited by AST checkers. Regular files expose a single model.""" + return [self.model] + + @property + def fix_source_lines(self) -> list[str]: + """Lines used when generating fixes. They share the coordinate space of :attr:`model`.""" + return self.source_lines + + @property + def check_lines(self) -> list[tuple[int, str]]: + """Physical lines (1-indexed) that raw-file rules should inspect.""" + return list(enumerate(self.source_lines, start=1)) + + def offset_range(self, diag_range: Range) -> None: # noqa: ARG002 + """Translate a diagnostic range from model coordinates to physical file coordinates (in place).""" + return + + def prepare_fix_edits(self, edits: list[TextEdit]) -> list[TextEdit]: + """Translate fix edits from model coordinates to physical file coordinates.""" + return edits + + +class EmbeddedSourceFile(SourceFile): + """ + Source file holding Robot Framework code embedded in Markdown or Python files. + + The physical file (``source_lines``) keeps prose, fences and the original indentation untouched, so writing it + back is a verbatim round-trip. + + Every code block is parsed into its own model whose line numbers are shifted to match the physical file, so AST + checkers see each block independently (via :attr:`models`) and no false positives are produced by the prose + between blocks. A combined model (:attr:`model`), where non-code lines are blanked out and code lines are + dedented, is kept for consumers that need a single model spanning the whole file (disablers, documentation + lines, templated-suite detection). + + Column positions differ between the model space and the physical file by the block indentation, so diagnostics + and fixes generated against a model are translated back to physical coordinates before they are displayed or + applied. + """ + + _blocks: list[RobotCodeBlock] | None = None + _model_lines: list[str] | None = None + _models: list[File] | None = None + + @property + def is_embedded(self) -> bool: + return True + + def _load_model(self, path_or_text: Path | str) -> File: + text = self._physical_text(path_or_text) + lines = text.splitlines(keepends=True) + self._blocks = extract_robot_blocks(lines) + reconstructed = reconstruct_source(lines, self._blocks) + self._model_lines = reconstructed.splitlines(keepends=True) + self._models = [ + self._build_block_model(block, self._model_lines) for block in self._blocks if not block.is_empty + ] + return self._build_model(reconstructed) + + def _build_model(self, source: str) -> File: + from robot.api import get_model as _get_model # noqa: PLC0415 + + if LANG_SUPPORTED: + model = _get_model(source, lang=self.config.languages) + else: + model = _get_model(source) + model.source = self.path + return model + + def _build_block_model(self, block: RobotCodeBlock, model_lines: list[str]) -> File: + source = "".join(model_lines[block.start_line - 1 : block.end_line]) + model = self._build_model(source) + return shift_model_lines(model, block.start_line - 1) + + def format_blocks(self) -> list[tuple[RobotCodeBlock, File]]: + """ + Return every non-empty block paired with a freshly parsed model for formatting. + + The models use block-local line numbers (starting at 1), which is what the formatters and the model writer + expect. Re-indentation and splicing back into the physical file is handled by the caller. + """ + blocks = [] + for block in self.blocks: + if block.is_empty: + continue + source = "".join(self.model_lines[block.start_line - 1 : block.end_line]) + blocks.append((block, self._build_model(source))) + return blocks + + def _physical_text(self, path_or_text: Path | str) -> str: + if isinstance(path_or_text, str): + return path_or_text + with open(path_or_text, encoding="utf-8", newline="") as f: + return f.read() + + @property + def blocks(self) -> list[RobotCodeBlock]: + if self._blocks is None: + _ = self.model # triggers extraction + return self._blocks or [] + + @property + def models(self) -> list[File]: + if self._models is None: + _ = self.model + return self._models or [] + + @property + def model_lines(self) -> list[str]: + if self._model_lines is None: + _ = self.model + return self._model_lines or [] + + @property + def fix_source_lines(self) -> list[str]: + return self.model_lines + + @property + def check_lines(self) -> list[tuple[int, str]]: + """Only the (dedented) lines that belong to a Robot code block are inspected by raw-file rules.""" + lines = self.model_lines + return [ + (lineno, lines[lineno - 1]) + for block in self.blocks + for lineno in range(block.start_line, block.end_line + 1) + if 0 < lineno <= len(lines) + ] + + def base_indent(self, lineno: int) -> int: + """Return the block indentation (number of columns) for the given physical line, or 0 outside any block.""" + for block in self.blocks: + if block.contains(lineno): + return len(block.indent) + return 0 + + def offset_range(self, diag_range: Range) -> None: + start_indent = self.base_indent(diag_range.start.line) + end_indent = self.base_indent(diag_range.end.line) + diag_range.start.character += start_indent + diag_range.end.character += end_indent + + def prepare_fix_edits(self, edits: list[TextEdit]) -> list[TextEdit]: + from robocop.linter.fix import TextEditKind # noqa: PLC0415 + + for edit in edits: + indent = self.base_indent(edit.start_line) + if not indent: + continue + if edit.kind == TextEditKind.REPLACEMENT: + edit.start_col += indent + edit.end_col += indent + elif edit.kind in (TextEditKind.REPLACEMENT_LINES, TextEditKind.INSERTION): + edit.replacement = self._reindent(edit.replacement, indent) + return edits + + @staticmethod + def _reindent(text: str, indent: int) -> str: + prefix = " " * indent + return "".join(f"{prefix}{line}" if line.strip() else line for line in text.splitlines(keepends=True)) + + +def build_source_file(path: Path, config: Config) -> SourceFile: + """Create the right :class:`SourceFile` for the given path, based on its extension.""" + from robocop.embedded import is_embedded_extension # noqa: PLC0415 + + if is_embedded_extension(path.suffix): + return EmbeddedSourceFile(path=path, config=config) + return SourceFile(path=path, config=config) + class VirtualSourceFile(SourceFile): @property diff --git a/tests/embedded/__init__.py b/tests/embedded/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/embedded/test_data/docstring.py b/tests/embedded/test_data/docstring.py new file mode 100644 index 000000000..7dd9ed880 --- /dev/null +++ b/tests/embedded/test_data/docstring.py @@ -0,0 +1,14 @@ +"""Module with a runnable Robot Framework example. + +Example: + + ```robotframework + *** Test Cases *** + Example Test + Log message + ``` +""" + + +def keyword(): + return 1 diff --git a/tests/embedded/test_data/expected/unformatted.md b/tests/embedded/test_data/expected/unformatted.md new file mode 100644 index 000000000..d57607c7f --- /dev/null +++ b/tests/embedded/test_data/expected/unformatted.md @@ -0,0 +1,11 @@ +# Formatting example + +Prose before the block. + +```robotframework +*** Test Cases *** +Example Test + Log message +``` + +Prose after the block. diff --git a/tests/embedded/test_data/expected/unformatted.py b/tests/embedded/test_data/expected/unformatted.py new file mode 100644 index 000000000..b20625da7 --- /dev/null +++ b/tests/embedded/test_data/expected/unformatted.py @@ -0,0 +1,10 @@ +"""Module with an unformatted Robot example. + +Example: + + ```robotframework + *** Test Cases *** + Example Test + Log message + ``` +""" diff --git a/tests/embedded/test_data/no_blocks.md b/tests/embedded/test_data/no_blocks.md new file mode 100644 index 000000000..4c0a4a065 --- /dev/null +++ b/tests/embedded/test_data/no_blocks.md @@ -0,0 +1,9 @@ +# Just a Markdown file + +No Robot Framework code blocks here, only prose and a shell snippet. + +```bash +echo "not robot code" +``` + +Nothing to lint. diff --git a/tests/embedded/test_data/two_blocks.md b/tests/embedded/test_data/two_blocks.md new file mode 100644 index 000000000..c1f330d09 --- /dev/null +++ b/tests/embedded/test_data/two_blocks.md @@ -0,0 +1,19 @@ +# Embedded Robot Framework code + +Some introduction prose describing the examples below. + +```robotframework +*** Test Cases *** +Example Test + Log message +``` + +More prose between the blocks. + +```robot +*** Keywords *** +Example Keyword + Log message +``` + +Closing prose. diff --git a/tests/embedded/test_data/unformatted.md b/tests/embedded/test_data/unformatted.md new file mode 100644 index 000000000..8f89290c7 --- /dev/null +++ b/tests/embedded/test_data/unformatted.md @@ -0,0 +1,11 @@ +# Formatting example + +Prose before the block. + +```robotframework +*** Test Cases *** +Example Test + Log message +``` + +Prose after the block. diff --git a/tests/embedded/test_data/unformatted.py b/tests/embedded/test_data/unformatted.py new file mode 100644 index 000000000..1bed2bbf1 --- /dev/null +++ b/tests/embedded/test_data/unformatted.py @@ -0,0 +1,10 @@ +"""Module with an unformatted Robot example. + +Example: + + ```robotframework + *** Test Cases *** + Example Test + Log message + ``` +""" diff --git a/tests/embedded/test_embedded.py b/tests/embedded/test_embedded.py new file mode 100644 index 000000000..7949802d1 --- /dev/null +++ b/tests/embedded/test_embedded.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +import pytest +import typer + +from robocop.embedded import extract_robot_blocks, is_embedded_extension +from robocop.run import check_files, format_files +from tests import working_directory +from tests.linter.utils import get_result, isolated_output + +TEST_DATA = Path(__file__).parent / "test_data" + + +def read(path: Path) -> str: + with open(path, encoding="utf-8", newline="") as f: + return f.read() + + +class TestExtraction: + def test_markdown_two_blocks(self): + lines = read(TEST_DATA / "two_blocks.md").splitlines(keepends=True) + blocks = extract_robot_blocks(lines) + assert len(blocks) == 2 + first, second = blocks + assert (first.start_line, first.end_line, first.indent) == (6, 8, "") + assert (second.start_line, second.end_line, second.indent) == (14, 16, "") + + def test_python_indented_block(self): + lines = read(TEST_DATA / "docstring.py").splitlines(keepends=True) + blocks = extract_robot_blocks(lines) + assert len(blocks) == 1 + block = blocks[0] + assert block.indent == " " + assert block.start_line == 6 + assert block.end_line == 8 + + def test_no_blocks(self): + lines = read(TEST_DATA / "no_blocks.md").splitlines(keepends=True) + assert extract_robot_blocks(lines) == [] + + def test_empty_block(self): + lines = ["```robotframework\n", "```\n"] + blocks = extract_robot_blocks(lines) + assert len(blocks) == 1 + assert blocks[0].is_empty + + @pytest.mark.parametrize( + ("suffix", "expected"), + [ + (".md", True), + (".markdown", True), + (".py", True), + (".MD", True), + (".robot", False), + (".resource", False), + (".txt", False), + ], + ) + def test_is_embedded_extension(self, suffix, expected): + assert is_embedded_extension(suffix) is expected + + +def run_check(source: str, select: list[str] | None = None, issue_format: str = "default") -> list[str]: + default_format = "{source}:{line}:{col} [{severity}] {rule_id} {desc}" + end_col_format = "{source}:{line}:{col}:{end_line}:{end_col} [{severity}] {rule_id} {desc}" + fmt = end_col_format if issue_format == "end_col" else default_format + with isolated_output() as output, working_directory(TEST_DATA): + with pytest.raises(typer.Exit): + check_files( + sources=[TEST_DATA / source], + select=select, + issue_format=fmt, + configure=["print_issues.output_format=simple"], + ignore_file_config=True, + cache=False, + ) + sys.stdout.flush() + result = get_result(output) + return [line for line in result.splitlines() if line.startswith(source)] + + +class TestLinter: + def test_markdown_physical_line_numbers(self): + issues = run_check("two_blocks.md") + assert "two_blocks.md:7:1 [W] DOC02 Missing documentation in 'Example Test' test case" in issues + assert "two_blocks.md:15:1 [W] DOC01 Missing documentation in 'Example Keyword' keyword" in issues + + def test_python_docstring_column_offset(self): + # The block is indented by 4 spaces, so the reported column is shifted by 4. + issues = run_check("docstring.py", select=["DOC02"], issue_format="end_col") + assert issues == ["docstring.py:7:5:7:17 [W] DOC02 Missing documentation in 'Example Test' test case"] + + def test_no_blocks_no_issues(self): + assert run_check("no_blocks.md") == [] + + +class TestFormatter: + def _format(self, tmp_path: Path, source: str) -> str: + work = tmp_path / source + shutil.copy2(TEST_DATA / source, work) + with pytest.raises(typer.Exit): + format_files(sources=[work], overwrite=True, cache=False) + return read(work) + + def test_markdown_block_formatted_prose_preserved(self, tmp_path): + result = self._format(tmp_path, "unformatted.md") + assert result == read(TEST_DATA / "expected" / "unformatted.md") + + def test_python_docstring_reindented(self, tmp_path): + result = self._format(tmp_path, "unformatted.py") + assert result == read(TEST_DATA / "expected" / "unformatted.py") + + def test_already_formatted_is_noop(self, tmp_path): + result = self._format(tmp_path, "two_blocks.md") + assert result == read(TEST_DATA / "two_blocks.md") + + def test_no_blocks_is_noop(self, tmp_path): + result = self._format(tmp_path, "no_blocks.md") + assert result == read(TEST_DATA / "no_blocks.md") + + def test_idempotent(self, tmp_path): + once = self._format(tmp_path, "unformatted.md") + work = tmp_path / "unformatted.md" + with pytest.raises(typer.Exit): + format_files(sources=[work], overwrite=True, cache=False) + assert read(work) == once