diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ced53a..dded30b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,16 +1,18 @@ name: Test -on: +on: pull_request_target: types: [assigned, opened, synchronize, reopened, ready_for_review] - paths: - - shasta/** + paths: + - shasta/** + - tests/gosh_tests/** push: branches: - main - future - paths: + paths: - libdash/** - shasta/** + - tests/gosh_tests/** jobs: Shasta-Test: strategy: @@ -21,7 +23,7 @@ jobs: runs-on: ${{ matrix.os }} if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} - name: Running Tests diff --git a/shasta/gosh_to_shasta_ast.py b/shasta/gosh_to_shasta_ast.py index 011fa4b..a295316 100644 --- a/shasta/gosh_to_shasta_ast.py +++ b/shasta/gosh_to_shasta_ast.py @@ -2,50 +2,58 @@ import json import subprocess -from typing import Any, Iterable +from typing import Any, Iterable, cast from .ast_node import ( - AstNode, + AArgChar, + AndNode, + ArgChar, + ArithForNode, + ArithNode, AssignNode, - CommandNode, - RedirNode, + AstNode, BackgroundNode, - NotNode, - TimeNode, - RedirectionNode, - FileRedirNode, + BArgChar, + CArgChar, + CaseNode, + Command, + CommandNode, + CondNode, + CondType, + CoprocNode, + DefunNode, DupRedirNode, + FileRedirNode, + ForNode, + GroupNode, HeredocRedirNode, - SingleArgRedirNode, - ArgChar, - CArgChar, - QArgChar, - AArgChar, - BArgChar, + IfNode, + NotNode, + OrNode, PArgChar, - VArgChar, PipeNode, - AndNode, - OrNode, - IfNode, - WhileNode, - ForNode, + QArgChar, + RedirectionNode, + RedirNode, SelectNode, - CaseNode, + SingleArgRedirNode, SubshellNode, - GroupNode, - DefunNode, - ArithForNode, - ArithNode, - CoprocNode, + TimeNode, + VArgChar, + WhileNode, make_typed_semi_sequence, string_of_arg, ) -# NOTE: This adapter targets shfmt's typed JSON output. +# This adapter targets shfmt's typed JSON output. # Use: shfmt -tojson > ast.json # Then: json.load(ast.json) and pass to to_ast_nodes/to_ast_node. +# NOTE: EArgChars are not created by the conversion, instead backslashes are turned into CArgChars +# To accurately detect which backslashes are escapes you need more context (which an interpreter would have) +# NOTE: TArgChars are also not created by the conversion, they are instead turned into CArgChars +# Similar to EArgChars, accurate detection would require more context + _SOURCE_BYTES: bytes | None = None @@ -75,57 +83,61 @@ def parse(path: str, shfmt_path: str | None = None) -> list[AstNode]: # Binary command operators (mvdan/sh BinCmdOperator) -> names BIN_CMD_OPS = { - 10: "AndStmt", - 11: "OrStmt", - 12: "Pipe", - 13: "PipeAll", + 11: "AndStmt", + 12: "OrStmt", + 13: "Pipe", + 14: "PipeAll", } # Case operators (mvdan/sh CaseOperator) -> names CASE_OPS = { - 33: "Break", - 34: "Fallthrough", - 35: "Resume", - 36: "ResumeKorn", + 35: "Break", + 36: "Fallthrough", + 37: "Resume", + 38: "ResumeKorn", } # Redirection operators (mvdan/sh RedirOperator) -> names REDIR_OPS = { - 54: "RdrOut", - 55: "AppOut", - 56: "RdrIn", - 57: "RdrInOut", - 58: "DplIn", - 59: "DplOut", - 60: "RdrClob", - 61: "RdrTrunc", - 62: "AppClob", - 63: "AppTrunc", - 64: "Hdoc", - 65: "DashHdoc", - 66: "WordHdoc", - 67: "RdrAll", - 68: "RdrAllClob", - 69: "RdrAllTrunc", - 70: "AppAll", - 71: "AppAllClob", - 72: "AppAllTrunc", + 63: "RdrOut", + 64: "AppOut", + 65: "RdrIn", + 66: "RdrInOut", + 67: "DplIn", + 68: "DplOut", + 69: "RdrClob", + 70: "AppClob", + 71: "Hdoc", + 72: "DashHdoc", + 73: "WordHdoc", + 74: "RdrAll", + 75: "RdrAllClob", + 76: "AppAll", + 77: "AppAllClob", } # Parameter expansion operators (mvdan/sh ParExpOperator) -> names PAR_EXP_OPS = { - 68: "AlternateUnset", - 69: "AlternateUnsetOrNull", - 70: "DefaultUnset", - 71: "DefaultUnsetOrNull", - 72: "ErrorUnset", - 73: "ErrorUnsetOrNull", - 74: "AssignUnset", - 75: "AssignUnsetOrNull", - 76: "RemSmallSuffix", - 77: "RemLargeSuffix", - 78: "RemSmallPrefix", - 79: "RemLargePrefix", + 81: "AlternateUnset", + 82: "AlternateUnsetOrNull", + 83: "DefaultUnset", + 84: "DefaultUnsetOrNull", + 85: "ErrorUnset", + 86: "ErrorUnsetOrNull", + 87: "AssignUnset", + 88: "AssignUnsetOrNull", + 89: "RemSmallSuffix", + 90: "RemLargeSuffix", + 91: "RemSmallPrefix", + 92: "RemLargePrefix", + 93: "MatchEmpty", + 94: "ArrayExclude", + 95: "ArrayIntersect", + 96: "UpperFirst", + 97: "UpperAll", + 98: "LowerFirst", + 99: "LowerAll", + 100: "OtherParamOps", } PAR_EXP_TO_VAR_TYPE = { @@ -163,111 +175,124 @@ def parse(path: str, shfmt_path: str | None = None) -> list[AstNode]: "RemLargeSuffix": "%%", "RemSmallPrefix": "#", "RemLargePrefix": "##", + "MatchEmpty": ":#", + "ArrayExclude": ":|", + "ArrayIntersect": ":*", + "UpperFirst": "^", + "UpperAll": "^^", + "LowerFirst": ",", + "LowerAll": ",,", + "OtherParamOps": "@", } GLOB_OPS = { - 122: "?(", - 123: "*(", - 124: "+(", - 125: "@(", - 126: "!(", + 139: "?(", + 140: "*(", + 141: "+(", + 142: "@(", + 143: "!(", } PROC_SUBST_OPS = { - 66: "<(", - 67: "=(", - 68: ">(", + 78: "<(", + 79: "=(", + 80: ">(", } UN_TEST_OPS = { - 88: "-e", - 89: "-f", - 90: "-d", - 91: "-c", - 92: "-b", - 93: "-p", - 94: "-S", - 95: "-L", - 96: "-k", - 97: "-g", - 98: "-u", - 99: "-G", - 100: "-O", - 101: "-N", - 102: "-r", - 103: "-w", - 104: "-x", - 105: "-s", - 106: "-t", - 107: "-z", - 108: "-n", - 109: "-o", - 110: "-v", - 111: "-R", - 112: "!", - 113: "(", + 105: "-e", + 106: "-f", + 107: "-d", + 108: "-c", + 109: "-b", + 110: "-p", + 111: "-S", + 112: "-L", + 113: "-k", + 114: "-g", + 115: "-u", + 116: "-G", + 117: "-O", + 118: "-N", + 119: "-r", + 120: "-w", + 121: "-x", + 122: "-s", + 123: "-t", + 124: "-z", + 125: "-n", + 126: "-o", + 127: "-v", + 128: "-R", + 39: "!", + 27: "(", } BIN_TEST_OPS = { - 112: "=~", - 113: "-nt", - 114: "-ot", - 115: "-ef", - 116: "-eq", - 117: "-ne", - 118: "-le", - 119: "-ge", - 120: "-lt", - 121: "-gt", - 122: "&&", - 123: "||", - 124: "=", - 125: "==", - 126: "!=", - 127: "<", - 128: ">", + 129: "=~", + 130: "-nt", + 131: "-ot", + 132: "-ef", + 133: "-eq", + 134: "-ne", + 135: "-le", + 136: "-ge", + 137: "-lt", + 138: "-gt", + 11: "&&", + 12: "||", + 87: "=", + 45: "==", + 46: "!=", + 65: "<", + 63: ">", } # Arithmetic operator tokens (mvdan/sh token values) -> symbols ARITH_TOKEN_STR = { - 68: "+", - 70: "-", - 38: "*", - 85: "/", - 76: "%", - 39: "**", - 40: "==", - 54: ">", - 56: "<", - 41: "!=", - 42: "<=", - 43: ">=", - 9: "&", - 12: "|", - 80: "^", - 55: ">>", - 61: "<<", - 10: "&&", - 11: "||", - 81: "^^", - 82: ",", - 72: "?", - 87: ":", - 74: "=", - 44: "+=", - 45: "-=", - 46: "*=", - 47: "/=", - 48: "%=", - 49: "&=", - 50: "|=", - 51: "^=", - 52: "<<=", - 53: ">>=", - 34: "!", - 35: "~", - 36: "++", - 37: "--", + 81: "+", + 83: "-", + 43: "*", + 101: "/", + 89: "%", + 44: "**", + 45: "==", + 63: ">", + 65: "<", + 46: "!=", + 47: "<=", + 48: ">=", + 10: "&", + 13: "|", + 96: "^", + 64: ">>", + 71: "<<", + 11: "&&", + 12: "||", + 97: "^^", + 98: ",", + 85: "?", + 104: ":", + 87: "=", + 49: "+=", + 50: "-=", + 51: "*=", + 52: "/=", + 53: "%=", + 54: "&=", + 55: "|=", + 56: "^=", + 57: "<<=", + 58: ">>=", + 39: "!", + 40: "~", + 41: "++", + 42: "--", +} + +PAR_NAMES_OP_STR = { + 43: "*", + 100: "@", } @@ -290,7 +315,8 @@ def to_ast_node(obj: Any) -> AstNode: def _stmt_to_command(stmt: dict[str, Any]) -> AstNode: - cmd = _command_to_ast(stmt.get("Cmd")) if stmt.get("Cmd") else _empty_command() + cmd_node = stmt.get("Cmd") + cmd = _command_to_ast(cast(dict[str, Any], cmd_node)) if cmd_node else _empty_command() redirs = _to_redirs(stmt.get("Redirs", [])) if stmt.get("Negated"): @@ -368,7 +394,8 @@ def _call_expr_to_command(node: dict[str, Any]) -> CommandNode: def _binary_cmd_to_ast(node: dict[str, Any]) -> AstNode: - op = BIN_CMD_OPS.get(node.get("Op")) + op_value = node.get("Op") + op = BIN_CMD_OPS.get(op_value) if isinstance(op_value, int) else None left = _stmt_to_command(node["X"]) right = _stmt_to_command(node["Y"]) @@ -377,6 +404,8 @@ def _binary_cmd_to_ast(node: dict[str, Any]) -> AstNode: if op == "OrStmt": return OrNode(left_operand=left, right_operand=right, no_braces=True) if op in ("Pipe", "PipeAll"): + if op == "PipeAll": + left = _apply_pipe_all(left) items = _flatten_pipe_items(left) + _flatten_pipe_items(right) return PipeNode(is_background=False, items=items) @@ -499,14 +528,14 @@ def _arithm_cmd_to_ast(node: dict[str, Any]) -> ArithNode: def _time_clause_to_ast(node: dict[str, Any]) -> TimeNode: stmt = node.get("Stmt") - inner = _stmt_to_command(stmt) if stmt is not None else _empty_command() + inner = _stmt_to_command(cast(dict[str, Any], stmt)) if stmt is not None else _empty_command() return TimeNode(time_posix=bool(node.get("PosixFormat")), command=inner) def _coproc_clause_to_ast(node: dict[str, Any]) -> CoprocNode: name_word = node.get("Name") name = _word_to_arg_chars(name_word) if name_word else [] - inner = _stmt_to_command(node.get("Stmt")) + inner = _stmt_to_command(cast(dict[str, Any], node.get("Stmt"))) return CoprocNode(name=name, body=inner) @@ -518,7 +547,8 @@ def _assign_to_shasta( name = node.get("Name") value = node.get("Value") - var = name.get("Value") if isinstance(name, dict) else "" + name_val = name.get("Value") if isinstance(name, dict) else "" + var = name_val if isinstance(name_val, str) else "" return AssignNode(var=var, val=_word_to_arg_chars(value) if value else []), None @@ -546,15 +576,20 @@ def _word_part_to_arg_chars(part: dict[str, Any]) -> list[ArgChar]: if part_type == "ParamExp": return _param_exp_to_arg_chars(part) if part_type == "CmdSubst": - cmd = _stmts_to_command(part.get("Stmts", [])) - return [BArgChar(cmd)] + cmd = cast(Command, _stmts_to_command(cast(list[dict[str, Any]], part.get("Stmts", [])))) + arg: ArgChar = BArgChar(cmd) + return [arg] if part_type == "ArithmExp": - expr = _arithm_expr_to_string(part.get("X")) + expr = _arithm_expr_to_string(cast(dict[str, Any], part.get("X"))) return [AArgChar(_string_to_arg_chars(expr))] if part_type == "ProcSubst": - op = PROC_SUBST_OPS.get(part.get("Op"), "<(") - cmd = _stmts_to_command(part.get("Stmts", [])) - return [PArgChar(op, cmd)] + op_value = part.get("Op") + op = PROC_SUBST_OPS.get(op_value) if isinstance(op_value, int) else None + if not op: + op = "<(" + cmd = cast(Command, _stmts_to_command(cast(list[dict[str, Any]], part.get("Stmts", [])))) + arg: ArgChar = PArgChar(op, cmd) + return [arg] if part_type == "ExtGlob": return _literal_word_part_chars(_extglob_to_string(part)) if part_type == "BraceExp": @@ -591,7 +626,10 @@ def _param_exp_to_arg_chars(node: dict[str, Any]) -> list[ArgChar]: if exp is None: return [VArgChar(fmt="Normal", null=False, var=var, arg=[])] - op_name = PAR_EXP_OPS.get(exp.get("Op")) + op_value = exp.get("Op") + op_name = PAR_EXP_OPS.get(op_value) if isinstance(op_value, int) else None + if op_name is None: + return _param_exp_literal_fallback(node) fmt = PAR_EXP_TO_VAR_TYPE.get(op_name) if fmt is None: return _param_exp_literal_fallback(node) @@ -629,14 +667,16 @@ def _arithm_expr_to_string(expr: dict[str, Any]) -> str: if expr_type == "Word": return string_of_arg(_word_to_arg_chars(expr)) if expr_type == "BinaryArithm": - op = ARITH_TOKEN_STR.get(expr.get("Op")) + op_value = expr.get("Op") + op = ARITH_TOKEN_STR.get(op_value) if isinstance(op_value, int) else None if not op: raise NotImplementedError(f"Unsupported arithmetic op: {expr.get('Op')}") left = _arithm_expr_to_string(expr["X"]) right = _arithm_expr_to_string(expr["Y"]) return f"{left} {op} {right}" if expr_type == "UnaryArithm": - op = ARITH_TOKEN_STR.get(expr.get("Op")) + op_value = expr.get("Op") + op = ARITH_TOKEN_STR.get(op_value) if isinstance(op_value, int) else None if not op: raise NotImplementedError(f"Unsupported unary arithmetic op: {expr.get('Op')}") inner = _arithm_expr_to_string(expr["X"]) @@ -655,14 +695,15 @@ def _to_redirs(redirs: Iterable[dict[str, Any]]) -> list[RedirectionNode]: def _redir_to_node(redir: dict[str, Any]) -> RedirectionNode: - op_name = REDIR_OPS.get(redir.get("Op")) + op_value = redir.get("Op") + op_name = REDIR_OPS.get(op_value) if isinstance(op_value, int) else None if not op_name: raise NotImplementedError(f"Unsupported redirection op: {redir.get('Op')}") default_fd = 0 if op_name in ("RdrIn", "RdrInOut", "DplIn", "Hdoc", "DashHdoc", "WordHdoc") else 1 fd = _redir_fd(redir.get("N"), default_fd) - word = redir.get("Word") - heredoc = redir.get("Hdoc") + word = cast(dict[str, Any], redir.get("Word")) + heredoc = cast(dict[str, Any], redir.get("Hdoc")) if op_name == "RdrOut": return FileRedirNode("To", fd, _word_to_arg_chars(word)) @@ -672,9 +713,9 @@ def _redir_to_node(redir: dict[str, Any]) -> RedirectionNode: return FileRedirNode("From", fd, _word_to_arg_chars(word)) if op_name == "RdrInOut": return FileRedirNode("FromTo", fd, _word_to_arg_chars(word)) - if op_name in ("RdrClob", "RdrTrunc"): + if op_name == "RdrClob": return FileRedirNode("Clobber", fd, _word_to_arg_chars(word)) - if op_name in ("AppClob", "AppTrunc"): + if op_name == "AppClob": return FileRedirNode("Append", fd, _word_to_arg_chars(word)) if op_name in ("Hdoc", "DashHdoc"): delimiter = string_of_arg(_word_to_arg_chars(word)) @@ -693,9 +734,9 @@ def _redir_to_node(redir: dict[str, Any]) -> RedirectionNode: if _word_lit_equals(word, "-"): return SingleArgRedirNode("CloseThis", fd) return DupRedirNode(dup_type, fd, _redir_arg(word)) - if op_name in ("RdrAll", "RdrAllClob", "RdrAllTrunc"): + if op_name in ("RdrAll", "RdrAllClob"): return SingleArgRedirNode("ErrAndOut", ("var", _word_to_arg_chars(word))) - if op_name in ("AppAll", "AppAllClob", "AppAllTrunc"): + if op_name in ("AppAll", "AppAllClob"): return SingleArgRedirNode("AppendErrAndOut", ("var", _word_to_arg_chars(word))) raise NotImplementedError(f"Unsupported redirection op: {op_name}") @@ -746,7 +787,9 @@ def _word_has_quotes(word: dict[str, Any] | None) -> bool: def _word_to_string(word: dict[str, Any] | None) -> str: - return string_of_arg(_word_to_arg_chars(word)) + if not word: + return "" + return string_of_arg(_word_to_arg_chars(cast(dict[str, Any], word))) def _assign_to_word_text(node: dict[str, Any]) -> str: @@ -850,12 +893,12 @@ def param_text() -> str: with_str = _word_to_string(with_word) if with_word else "" buf += f"/{orig}/{with_str}" if names: - # Names operator is encoded as a token. - op = "*" if names == 1 else "@" + op = PAR_NAMES_OP_STR.get(names, "@") buf = "${!" + name + op if exp: - op_name = PAR_EXP_OPS.get(exp.get("Op")) - op_str = PAR_EXP_OP_STR.get(op_name, "") + op_value = exp.get("Op") + op_name = PAR_EXP_OPS.get(op_value) if isinstance(op_value, int) else None + op_str = PAR_EXP_OP_STR.get(op_name, "") if op_name else "" buf += f"{op_str}{_word_to_string(exp.get('Word'))}" buf += "}" @@ -863,13 +906,15 @@ def param_text() -> str: def _proc_subst_to_string(node: dict[str, Any]) -> str: - op = PROC_SUBST_OPS.get(node.get("Op"), "<(") - cmd = _stmts_to_command(node.get("Stmts", [])) + op_value = node.get("Op") + op = PROC_SUBST_OPS.get(op_value) if isinstance(op_value, int) else "<(" + cmd = _stmts_to_command(cast(list[dict[str, Any]], node.get("Stmts", []))) return f"{op}{cmd.pretty()})" def _extglob_to_string(node: dict[str, Any]) -> str: - op = GLOB_OPS.get(node.get("Op"), "?(") + op_value = node.get("Op") + op = GLOB_OPS.get(op_value) if isinstance(op_value, int) else "?(" pattern = node.get("Pattern", {}).get("Value", "") return f"{op}{pattern})" @@ -924,6 +969,24 @@ def _flatten_pipe_items(node: AstNode) -> list[AstNode]: return [node] +def _apply_pipe_all(node: AstNode) -> AstNode: + dup = DupRedirNode("ToFD", ("fixed", 2), ("fixed", 1), move=False) + return _attach_redirs(node, [dup]) + + +def _attach_redirs(node: AstNode, redirs: list[RedirectionNode]) -> AstNode: + if isinstance(node, PipeNode) and node.items: + node.items[-1] = cast(Command, _attach_redirs(node.items[-1], redirs)) + return node + if isinstance(node, CommandNode): + node.redir_list.extend(redirs) + return node + if isinstance(node, RedirNode): + node.redir_list.extend(redirs) + return node + return RedirNode(line_number=None, node=cast(Command, node), redir_list=redirs) + + def _stmts_to_command(stmts: list[dict[str, Any]]) -> AstNode: nodes = [_stmt_to_command(stmt) for stmt in stmts] if not nodes: @@ -958,37 +1021,89 @@ def _let_clause_to_ast(node: dict[str, Any]) -> CommandNode: return CommandNode(line_number=-1, assignments=[], arguments=args, redir_list=[]) -def _test_clause_to_ast(node: dict[str, Any]) -> CommandNode: - expr = node.get("X") - words = [_string_to_arg_chars("[[")] - words.extend(_test_expr_to_words(expr)) - words.append(_string_to_arg_chars("]]")) - return CommandNode(line_number=-1, assignments=[], arguments=words, redir_list=[]) +def _test_clause_to_ast(node: dict[str, Any]) -> CondNode: + expr = cast(dict[str, Any], node.get("X")) + return _test_expr_to_cond(expr) -def _test_expr_to_words(expr: dict[str, Any]) -> list[list[ArgChar]]: +def _test_expr_to_cond(expr: dict[str, Any]) -> CondNode: etype = expr.get("Type") if etype == "Word": - return [_word_to_arg_chars(expr)] + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_TERM.value, + op=_word_to_arg_chars(expr), + left=None, + right=None, + invert_return=False, + ) if etype == "UnaryTest": - op = UN_TEST_OPS.get(expr.get("Op"), "") - inner = _test_expr_to_words(expr.get("X")) - return [_string_to_arg_chars(op)] + inner + op_value = expr.get("Op") + op = UN_TEST_OPS.get(op_value) if isinstance(op_value, int) else "" + inner = _test_expr_to_cond(cast(dict[str, Any], expr.get("X"))) + if op == "!": + inner.invert_return = not inner.invert_return + return inner + if not op: + raise NotImplementedError(f"Unsupported unary test op: {expr.get('Op')}") + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_UNARY.value, + op=_string_to_arg_chars(op), + left=inner, + right=None, + invert_return=False, + ) if etype == "BinaryTest": - op = BIN_TEST_OPS.get(expr.get("Op"), "") - left = _test_expr_to_words(expr.get("X")) - right = _test_expr_to_words(expr.get("Y")) - return left + [_string_to_arg_chars(op)] + right + op_value = expr.get("Op") + op = BIN_TEST_OPS.get(op_value) if isinstance(op_value, int) else "" + left = _test_expr_to_cond(cast(dict[str, Any], expr.get("X"))) + right = _test_expr_to_cond(cast(dict[str, Any], expr.get("Y"))) + if op == "&&": + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_AND.value, + op=None, + left=left, + right=right, + invert_return=False, + ) + if op == "||": + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_OR.value, + op=None, + left=left, + right=right, + invert_return=False, + ) + if not op: + raise NotImplementedError(f"Unsupported binary test op: {expr.get('Op')}") + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_BINARY.value, + op=_string_to_arg_chars(op), + left=left, + right=right, + invert_return=False, + ) if etype == "ParenTest": - inner = _test_expr_to_words(expr.get("X")) - return [_string_to_arg_chars("(")] + inner + [_string_to_arg_chars(")")] + inner = _test_expr_to_cond(cast(dict[str, Any], expr.get("X"))) + return CondNode( + line_number=_line_from_pos(expr.get("Pos")) or -1, + cond_type=CondType.COND_EXPR.value, + op=None, + left=inner, + right=None, + invert_return=False, + ) raise NotImplementedError(f"Unsupported test expr: {etype}") def _test_decl_to_ast(node: dict[str, Any]) -> CommandNode: desc = _word_to_string(node.get("Description")) - body = _stmt_to_command(node.get("Body")) + body = _stmt_to_command(cast(dict[str, Any], node.get("Body"))) body_str = body.pretty() if body.NodeName != "Group": body_str = "{ " + body_str + " ; }" diff --git a/tests/Makefile b/tests/Makefile index 62b58c6..491ce51 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,4 @@ -.PHONY: test dash_test bash_test gosh_test tutorial_test clean +.PHONY: test dash_test bash_test gosh_test gosh_unit_test tutorial_test clean dash_test: @find libdash_tests/tests libdash_tests/pash_tests -type f | while read f; do libdash_tests/round_trip.sh ./rt.py "$$f"; done | tee python.log @@ -14,10 +14,13 @@ gosh_test: @cat python.log | egrep '^[A-Z0-9_]+:' | cut -d ':' -f 1 | sort | uniq -c @grep ':' python.log && echo "FAILED" && exit 1 || exit 0 +gosh_unit_test: + @python3 gosh_tests/test_gosh_to_shasta_ast.py + tutorial_test: @pytest tutorial_tests/ -v | tee -a python.log -test: dash_test bash_test gosh_test tutorial_test +test: dash_test bash_test gosh_test gosh_unit_test tutorial_test clean: rm -rf $(wildcard *.log) libdash_tests diff --git a/tests/gosh_tests/test_gosh_to_shasta_ast.py b/tests/gosh_tests/test_gosh_to_shasta_ast.py new file mode 100644 index 0000000..4fb1376 --- /dev/null +++ b/tests/gosh_tests/test_gosh_to_shasta_ast.py @@ -0,0 +1,1340 @@ +#!/usr/bin/env python3 +"""Minimal tests for the gosh (shfmt) JSON to shasta bridge.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT_DIR)) + +from shasta.ast_node import ( + AArgChar, + AndNode, + ArgChar, + ArithForNode, + ArithNode, + AssignNode, + AstNode, + BackgroundNode, + BArgChar, + CArgChar, + CaseNode, + CommandNode, + CondNode, + CondType, + CoprocNode, + DefunNode, + DupRedirNode, + FileRedirNode, + ForNode, + GroupNode, + HeredocRedirNode, + IfNode, + NotNode, + OrNode, + PArgChar, + PipeNode, + QArgChar, + RedirNode, + SelectNode, + SingleArgRedirNode, + SubshellNode, + TimeNode, + VArgChar, + WhileNode, + ast_node_to_untyped_deep, +) +from shasta.gosh_to_shasta_ast import to_ast_node, to_ast_nodes + + +def argchars(text: str) -> list[ArgChar]: + return [CArgChar(ord(ch)) for ch in text] + + +def cond_term(text: str, line: int = -1) -> CondNode: + return CondNode( + line_number=line, + cond_type=CondType.COND_TERM.value, + op=argchars(text), + left=None, + right=None, + invert_return=False, + ) + + +def cond_unary(op: str, inner: CondNode, line: int = -1) -> CondNode: + return CondNode( + line_number=line, + cond_type=CondType.COND_UNARY.value, + op=argchars(op), + left=inner, + right=None, + invert_return=False, + ) + + +def cond_binary(op: str, left: CondNode, right: CondNode, line: int = -1) -> CondNode: + return CondNode( + line_number=line, + cond_type=CondType.COND_BINARY.value, + op=argchars(op), + left=left, + right=right, + invert_return=False, + ) + + +def cond_and(left: CondNode, right: CondNode, line: int = -1) -> CondNode: + return CondNode( + line_number=line, + cond_type=CondType.COND_AND.value, + op=None, + left=left, + right=right, + invert_return=False, + ) + + +def cond_or(left: CondNode, right: CondNode, line: int = -1) -> CondNode: + return CondNode( + line_number=line, + cond_type=CondType.COND_OR.value, + op=None, + left=left, + right=right, + invert_return=False, + ) + + +def word_json(text: str) -> dict[str, object]: + return {"Parts": [{"Type": "Lit", "Value": text}]} + + +def lit_json(text: str) -> dict[str, object]: + return {"Type": "Lit", "Value": text} + + +def call_json( + args: list[dict[str, object]], assigns: list[dict[str, object]] | None = None +) -> dict[str, object]: + return { + "Type": "CallExpr", + "Assigns": assigns or [], + "Args": args, + } + + +def stmt_json(cmd: dict[str, object], **kwargs: object) -> dict[str, object]: + out: dict[str, object] = {"Cmd": cmd} + out.update(kwargs) + return out + + +def assert_shasta_equal(actual: object, expected: object) -> None: + assert ast_node_to_untyped_deep(actual) == ast_node_to_untyped_deep(expected) + + +def shfmt_to_shasta_nodes(script: str) -> list[AstNode]: + script_text = ( + script if script.endswith("\n") else f"{script}\n" + ) # Add trailing newline if missing + with tempfile.NamedTemporaryFile("wb", suffix=".sh", delete=False) as handle: + path = handle.name + handle.write(script_text.encode("utf-8")) + try: + proc = subprocess.run( + ["shfmt", "--to-json", "-filename", path], + input=script_text.encode("utf-8"), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise RuntimeError("shfmt not found on PATH") from exc + finally: + os.unlink(path) + payload = json.loads(proc.stdout.decode("utf-8")) + return to_ast_nodes(payload) + + +def test_call_expr_with_assign() -> None: + # A=1 echo + shfmt_node = call_json( + args=[word_json("echo")], + assigns=[{"Name": lit_json("A"), "Value": word_json("1")}], + ) + expected = CommandNode( + line_number=-1, + assignments=[AssignNode("A", argchars("1"))], + arguments=[argchars("echo")], + redir_list=[], + ) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_binary_cmd_ops() -> None: + # true && false + left = stmt_json(call_json([word_json("true")])) + right = stmt_json(call_json([word_json("false")])) + + shfmt_and = { + "Type": "BinaryCmd", + "Op": 11, + "X": left, + "Y": right, + } + expected_and = AndNode( + left_operand=CommandNode(-1, [], [argchars("true")], []), + right_operand=CommandNode(-1, [], [argchars("false")], []), + no_braces=True, + ) + + actual = to_ast_node(shfmt_and) + + assert_shasta_equal(actual, expected_and) + + # true || false + shfmt_or = { + "Type": "BinaryCmd", + "Op": 12, + "X": left, + "Y": right, + } + expected_or = OrNode( + left_operand=CommandNode(-1, [], [argchars("true")], []), + right_operand=CommandNode(-1, [], [argchars("false")], []), + no_braces=True, + ) + + actual = to_ast_node(shfmt_or) + + assert_shasta_equal(actual, expected_or) + + +def test_pipe_all_injects_dup() -> None: + # echo hi |& cat + left = stmt_json(call_json([word_json("echo")])) + right = stmt_json(call_json([word_json("cat")])) + shfmt_node = { + "Type": "BinaryCmd", + "Op": 14, + "X": left, + "Y": right, + } + expected = PipeNode( + is_background=False, + items=[ + CommandNode( + -1, + [], + [argchars("echo")], + [DupRedirNode("ToFD", ("fixed", 2), ("fixed", 1), move=False)], + ), + CommandNode(-1, [], [argchars("cat")], []), + ], + ) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_if_and_while_until() -> None: + # if true; then echo hi; else false; fi + cond_stmt = stmt_json(call_json([word_json("true")])) + then_stmt = stmt_json(call_json([word_json("echo")])) + else_stmt = stmt_json(call_json([word_json("false")])) + shfmt_if = { + "Type": "IfClause", + "Cond": [cond_stmt], + "Then": [then_stmt], + "Else": {"Cond": [], "Then": [else_stmt]}, + } + expected_if = IfNode( + cond=CommandNode(-1, [], [argchars("true")], []), + then_b=CommandNode(-1, [], [argchars("echo")], []), + else_b=CommandNode(-1, [], [argchars("false")], []), + ) + + actual = to_ast_node(shfmt_if) + + assert_shasta_equal(actual, expected_if) + + # while true; do echo hi; done + shfmt_while = { + "Type": "WhileClause", + "Cond": [cond_stmt], + "Do": [then_stmt], + "Until": True, + } + expected_while = WhileNode( + test=NotNode(CommandNode(-1, [], [argchars("true")], []), no_braces=True), + body=CommandNode(-1, [], [argchars("echo")], []), + ) + + actual = to_ast_node(shfmt_while) + + assert_shasta_equal(actual, expected_while) + + +def test_for_clause_variants() -> None: + # for i in a b; do echo hi; done + word_iter = { + "Type": "WordIter", + "Name": lit_json("i"), + "Items": [word_json("a"), word_json("b")], + } + shfmt_for = { + "Type": "ForClause", + "Loop": word_iter, + "Do": [stmt_json(call_json([word_json("echo")]))], + } + expected_for = ForNode( + line_number=-1, + argument=[argchars("a"), argchars("b")], + body=CommandNode(-1, [], [argchars("echo")], []), + variable=argchars("i"), + ) + + actual = to_ast_node(shfmt_for) + + assert_shasta_equal(actual, expected_for) + + # for i select a b; do echo hi; done + shfmt_select = { + "Type": "ForClause", + "Loop": word_iter, + "Select": True, + "Do": [stmt_json(call_json([word_json("echo")]))], + } + expected_select = SelectNode( + line_number=-1, + variable=argchars("i"), + body=CommandNode(-1, [], [argchars("echo")], []), + map_list=[argchars("a"), argchars("b")], + ) + + actual = to_ast_node(shfmt_select) + + assert_shasta_equal(actual, expected_select) + + # for ((i=0; i<2; i++)); do echo hi; done + cstyle = { + "Type": "CStyleLoop", + "Init": {"Type": "Word", "Parts": [lit_json("i=0")]}, + "Cond": {"Type": "Word", "Parts": [lit_json("i<2")]}, + "Post": {"Type": "Word", "Parts": [lit_json("i++")]}, + } + shfmt_cstyle = { + "Type": "ForClause", + "Loop": cstyle, + "Do": [stmt_json(call_json([word_json("echo")]))], + } + expected_cstyle = ArithForNode( + line_number=-1, + init=[argchars("i=0")], + cond=[argchars("i<2")], + step=[argchars("i++")], + action=CommandNode(-1, [], [argchars("echo")], []), + ) + + actual = to_ast_node(shfmt_cstyle) + + assert_shasta_equal(actual, expected_cstyle) + + +def test_case_clause_fallthrough() -> None: + # case x in a) echo hi ;& b) echo bye ;; esac + case_item = { + "Patterns": [word_json("a")], + "Stmts": [stmt_json(call_json([word_json("echo")]))], + "Op": 36, + } + shfmt_node = { + "Type": "CaseClause", + "Word": word_json("x"), + "Items": [case_item], + } + expected = CaseNode( + line_number=None, + argument=argchars("x"), + cases=[ + { + "cpattern": [argchars("a")], + "cbody": CommandNode(-1, [], [argchars("echo")], []), + "fallthrough": True, + } + ], + ) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_redirection_variants() -> None: + # echo out > out 2>> err < inout + shfmt_stmt = stmt_json( + call_json([word_json("echo")]), + Redirs=[ + {"Op": 63, "Word": word_json("out")}, + {"Op": 65, "Word": word_json("in")}, + {"Op": 71, "Word": word_json("EOF"), "Hdoc": word_json("body")}, + {"Op": 73, "Word": word_json("data")}, + {"Op": 67, "Word": word_json("1")}, + {"Op": 68, "Word": word_json("-")}, + {"Op": 74, "Word": word_json("out")}, + ], + ) + expected = CommandNode( + -1, + [], + [argchars("echo")], + [ + FileRedirNode("To", ("fixed", 1), argchars("out")), + FileRedirNode("From", ("fixed", 0), argchars("in")), + HeredocRedirNode("XHere", ("fixed", 0), argchars("body"), False, "EOF"), + FileRedirNode("ReadingString", ("fixed", 0), argchars("data")), + DupRedirNode("FromFD", ("fixed", 0), ("fixed", 1), move=False), + SingleArgRedirNode("CloseThis", ("fixed", 1)), + SingleArgRedirNode("ErrAndOut", ("var", argchars("out"))), + ], + ) + + actual = to_ast_node(shfmt_stmt) + + assert_shasta_equal(actual, expected) + + +def test_word_parts_and_param_exp() -> None: + # echo hi there "you" ${VAR:-fallback} $(echo) $((1+2)) <(echo) ?(*.py) {a,b} + shfmt_word: dict[str, object] = { + "Parts": [ + {"Type": "Lit", "Value": "hi"}, + {"Type": "SglQuoted", "Value": "there"}, + {"Type": "DblQuoted", "Parts": [{"Type": "Lit", "Value": "you"}]}, + { + "Type": "ParamExp", + "Param": lit_json("VAR"), + "Exp": {"Op": 84, "Word": word_json("fallback")}, + }, + {"Type": "CmdSubst", "Stmts": [stmt_json(call_json([word_json("echo")]))]}, + {"Type": "ArithmExp", "X": {"Type": "Word", "Parts": [lit_json("1+2")]}}, + { + "Type": "ProcSubst", + "Op": 78, + "Stmts": [stmt_json(call_json([word_json("echo")]))], + }, + {"Type": "ExtGlob", "Op": 139, "Pattern": lit_json("*.py")}, + { + "Type": "BraceExp", + "Elems": [word_json("a"), word_json("b")], + "Sequence": False, + }, + ] + } + + shfmt_node = call_json([shfmt_word]) + expected = CommandNode( + -1, + [], + [ + argchars("hi") + + [QArgChar(argchars("there"))] + + [QArgChar(argchars("you"))] + + [VArgChar("Minus", True, "VAR", argchars("fallback"))] + + [BArgChar(CommandNode(-1, [], [argchars("echo")], []))] + + [AArgChar(argchars("1+2"))] + + [PArgChar("<(", CommandNode(-1, [], [argchars("echo")], []))] + + argchars("?(*.py)") + + argchars("{a,b}") + ], + [], + ) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_param_exp_names_fallback() -> None: + # echo ${!arr*} + shfmt_word: dict[str, object] = { + "Parts": [{"Type": "ParamExp", "Param": lit_json("arr"), "Names": 43}] + } + shfmt_node = call_json([shfmt_word]) + expected = CommandNode( + -1, + [], + [argchars("${!arr*}")], + [], + ) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_test_clause_binary() -> None: + # [[ a -eq b ]] + shfmt_node = { + "Type": "TestClause", + "X": { + "Type": "BinaryTest", + "Op": 133, + "X": {"Type": "Word", "Parts": [lit_json("a")]}, + "Y": {"Type": "Word", "Parts": [lit_json("b")]}, + }, + } + expected = cond_binary("-eq", cond_term("a"), cond_term("b")) + + actual = to_ast_node(shfmt_node) + + assert_shasta_equal(actual, expected) + + +def test_file_to_nodes() -> None: + shfmt_file = { + "Type": "File", + "Stmts": [stmt_json(call_json([word_json("echo")]))], + } + expected = [CommandNode(-1, [], [argchars("echo")], [])] + + actual = to_ast_nodes(shfmt_file) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_simple_call() -> None: + script = "echo hi" + expected = [CommandNode(1, [], [argchars("echo"), argchars("hi")], [])] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_escape_param() -> None: + script = "echo $HOME\necho \\$HOME" + expected = [ + CommandNode( + 1, + [], + [argchars("echo"), [VArgChar("Normal", False, "HOME", [])]], + [], + ), + CommandNode(2, [], [argchars("echo"), argchars("\\$HOME")], []), + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_escape_glob_literal() -> None: + script = "echo *\necho \\*" + expected = [ + CommandNode(1, [], [argchars("echo"), argchars("*")], []), + CommandNode(2, [], [argchars("echo"), argchars("\\*")], []), + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_pipe_all() -> None: + script = "echo hi |& cat" + expected = [ + PipeNode( + is_background=False, + items=[ + CommandNode( + 1, + [], + [argchars("echo"), argchars("hi")], + [DupRedirNode("ToFD", ("fixed", 2), ("fixed", 1), move=False)], + ), + CommandNode(1, [], [argchars("cat")], []), + ], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_background() -> None: + script = "echo hi &" + expected = [ + BackgroundNode( + line_number=None, + node=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + redir_list=[], + no_braces=True, + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_background_pipe() -> None: + script = "echo hi | cat &" + expected = [ + BackgroundNode( + line_number=None, + node=PipeNode( + is_background=False, + items=[ + CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + CommandNode(1, [], [argchars("cat")], []), + ], + ), + redir_list=[], + no_braces=True, + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_negation() -> None: + script = "! echo hi" + expected = [ + NotNode( + CommandNode(1, [], [argchars("echo"), argchars("hi")], []), no_braces=True + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_time_clause() -> None: + script = "time echo hi" + expected = [ + TimeNode( + time_posix=False, + command=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_time_clause_posix() -> None: + script = "time -p echo hi" + expected = [ + TimeNode( + time_posix=True, + command=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_function_decl() -> None: + script = "foo() { echo hi; }" + expected = [ + DefunNode( + line_number=1, + name=argchars("foo"), + body=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + bash_mode=False, + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_coproc() -> None: + script = "coproc echo hi" + expected = [ + CoprocNode( + name=[], + body=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_subshell() -> None: + script = "(echo hi)" + expected = [ + SubshellNode( + line_number=1, + body=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + redir_list=[], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_group() -> None: + script = "{ echo hi; }" + expected = [GroupNode(CommandNode(1, [], [argchars("echo"), argchars("hi")], []))] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_group_redir() -> None: + script = "{ echo hi; } > out" + expected = [ + RedirNode( + line_number=None, + node=GroupNode(CommandNode(1, [], [argchars("echo"), argchars("hi")], [])), + redir_list=[FileRedirNode("To", ("fixed", 1), argchars("out"))], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_complex_one() -> None: + script = ( + "echo hi\n" + "if true; then echo ok; fi\n" + "{ echo grp; } > out\n" + "(echo sub)\n" + "foo() { echo fn; }\n" + "coproc echo co\n" + "time -p echo tm\n" + "echo a | cat &\n" + "[[ -n x ]]\n" + "((i+=1))\n" + ) + expected = [ + CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + IfNode( + cond=CommandNode(2, [], [argchars("true")], []), + then_b=CommandNode(2, [], [argchars("echo"), argchars("ok")], []), + else_b=None, + ), + RedirNode( + line_number=None, + node=GroupNode(CommandNode(3, [], [argchars("echo"), argchars("grp")], [])), + redir_list=[FileRedirNode("To", ("fixed", 1), argchars("out"))], + ), + SubshellNode( + line_number=4, + body=CommandNode(4, [], [argchars("echo"), argchars("sub")], []), + redir_list=[], + ), + DefunNode( + line_number=5, + name=argchars("foo"), + body=CommandNode(5, [], [argchars("echo"), argchars("fn")], []), + bash_mode=False, + ), + CoprocNode( + name=[], + body=CommandNode(6, [], [argchars("echo"), argchars("co")], []), + ), + TimeNode( + time_posix=True, + command=CommandNode(7, [], [argchars("echo"), argchars("tm")], []), + ), + BackgroundNode( + line_number=None, + node=PipeNode( + is_background=False, + items=[ + CommandNode(8, [], [argchars("echo"), argchars("a")], []), + CommandNode(8, [], [argchars("cat")], []), + ], + ), + redir_list=[], + no_braces=True, + ), + cond_unary("-n", cond_term("x", line=9), line=9), + ArithNode(line_number=10, body=[argchars("i += 1")]), + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_complex_two() -> None: + script = ( + "for i in a b; do echo hi; done\n" + "while false; do echo no; done\n" + "case x in a) echo one ;; esac\n" + "echo ${a:-b} ${#a}\n" + "echo <(echo hi) >(cat)\n" + "echo @(a) ?(b) *(c) +(d) !(e)\n" + "cat <(", CommandNode(5, [], [argchars("cat")], []))], + ], + [], + ), + CommandNode( + 6, + [], + [ + argchars("echo"), + argchars("@(a)"), + argchars("?(b)"), + argchars("*(c)"), + argchars("+(d)"), + argchars("!(e)"), + ], + [], + ), + CommandNode( + 7, + [], + [argchars("cat")], + [HeredocRedirNode("XHere", ("fixed", 0), argchars("body\n"), False, "EOF")], + ), + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_if_clause() -> None: + script = "if true; then echo hi; fi" + expected = [ + IfNode( + cond=CommandNode(1, [], [argchars("true")], []), + then_b=CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + else_b=None, + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_test_clause() -> None: + script = "[[ -n x ]]" + expected = [cond_unary("-n", cond_term("x", line=1), line=1)] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_test_clause_flags() -> None: + flags = [ + "-e", + "-f", + "-d", + "-c", + "-b", + "-p", + "-S", + "-L", + "-k", + "-g", + "-u", + "-G", + "-O", + "-N", + "-r", + "-w", + "-x", + "-s", + "-t", + "-z", + "-n", + "-o", + "-v", + "-R", + ] + for flag in flags: + script = f"[[ {flag} x ]]" + expected = [cond_unary(flag, cond_term("x", line=1), line=1)] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_test_clause_binary_ops() -> None: + cases = [ + ("[[ a -eq b ]]", ["[[", "a", "-eq", "b", "]]"]), + ("[[ a -ne b ]]", ["[[", "a", "-ne", "b", "]]"]), + ("[[ a -lt b ]]", ["[[", "a", "-lt", "b", "]]"]), + ("[[ a -gt b ]]", ["[[", "a", "-gt", "b", "]]"]), + ("[[ a -le b ]]", ["[[", "a", "-le", "b", "]]"]), + ("[[ a -ge b ]]", ["[[", "a", "-ge", "b", "]]"]), + ("[[ a -nt b ]]", ["[[", "a", "-nt", "b", "]]"]), + ("[[ a -ot b ]]", ["[[", "a", "-ot", "b", "]]"]), + ("[[ a -ef b ]]", ["[[", "a", "-ef", "b", "]]"]), + ("[[ a = b ]]", ["[[", "a", "=", "b", "]]"]), + ("[[ a == b ]]", ["[[", "a", "==", "b", "]]"]), + ("[[ a != b ]]", ["[[", "a", "!=", "b", "]]"]), + ("[[ a < b ]]", ["[[", "a", "<", "b", "]]"]), + ("[[ a > b ]]", ["[[", "a", ">", "b", "]]"]), + ("[[ a =~ b ]]", ["[[", "a", "=~", "b", "]]"]), + ("[[ a && b ]]", ["[[", "a", "&&", "b", "]]"]), + ("[[ a || b ]]", ["[[", "a", "||", "b", "]]"]), + ] + for script, tokens in cases: + left = cond_term(tokens[1], line=1) + right = cond_term(tokens[3], line=1) + if tokens[2] == "&&": + expected = [cond_and(left, right, line=1)] + elif tokens[2] == "||": + expected = [cond_or(left, right, line=1)] + else: + expected = [cond_binary(tokens[2], left, right, line=1)] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_binary_cmd_ops() -> None: + cases = [ + ( + "true && false", + AndNode( + left_operand=CommandNode(1, [], [argchars("true")], []), + right_operand=CommandNode(1, [], [argchars("false")], []), + no_braces=True, + ), + ), + ( + "true || false", + OrNode( + left_operand=CommandNode(1, [], [argchars("true")], []), + right_operand=CommandNode(1, [], [argchars("false")], []), + no_braces=True, + ), + ), + ( + "echo hi | cat", + PipeNode( + is_background=False, + items=[ + CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + CommandNode(1, [], [argchars("cat")], []), + ], + ), + ), + ] + for script, expected_node in cases: + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, [expected_node]) + + +def test_shfmt_script_redir_all() -> None: + script = "echo hi &> out" + expected = [ + CommandNode( + 1, + [], + [argchars("echo"), argchars("hi")], + [SingleArgRedirNode("ErrAndOut", ("var", argchars("out")))], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_redir_ops() -> None: + cases = [ + ( + "echo hi > out", + [argchars("echo"), argchars("hi")], + [FileRedirNode("To", ("fixed", 1), argchars("out"))], + ), + ( + "echo hi >> out", + [argchars("echo"), argchars("hi")], + [FileRedirNode("Append", ("fixed", 1), argchars("out"))], + ), + ( + "echo hi >| out", + [argchars("echo"), argchars("hi")], + [FileRedirNode("Clobber", ("fixed", 1), argchars("out"))], + ), + ( + "cat < in", + [argchars("cat")], + [FileRedirNode("From", ("fixed", 0), argchars("in"))], + ), + ( + "cat <> inout", + [argchars("cat")], + [FileRedirNode("FromTo", ("fixed", 0), argchars("inout"))], + ), + ( + "cat <&2", + [argchars("echo"), argchars("hi")], + [DupRedirNode("ToFD", ("fixed", 1), ("fixed", 2), move=False)], + ), + ( + "cat <&0", + [argchars("cat")], + [DupRedirNode("FromFD", ("fixed", 0), ("fixed", 0), move=False)], + ), + ( + "echo hi &>> out", + [argchars("echo"), argchars("hi")], + [SingleArgRedirNode("AppendErrAndOut", ("var", argchars("out")))], + ), + ] + for script, args, redirs in cases: + expected = [CommandNode(1, [], args, redirs)] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_param_exp_ops() -> None: + script = ( + "echo ${a-b} ${a:-b} ${a+b} ${a:+b} ${a?b} ${a:?b} ${a=b} ${a:=b} " + "${a%p} ${a%%p} ${a#p} ${a##p} ${#a}" + ) + expected = [ + CommandNode( + 1, + [], + [ + argchars("echo"), + [VArgChar("Minus", False, "a", argchars("b"))], + [VArgChar("Minus", True, "a", argchars("b"))], + [VArgChar("Plus", False, "a", argchars("b"))], + [VArgChar("Plus", True, "a", argchars("b"))], + [VArgChar("Question", False, "a", argchars("b"))], + [VArgChar("Question", True, "a", argchars("b"))], + [VArgChar("Assign", False, "a", argchars("b"))], + [VArgChar("Assign", True, "a", argchars("b"))], + [VArgChar("TrimR", False, "a", argchars("p"))], + [VArgChar("TrimRMax", False, "a", argchars("p"))], + [VArgChar("TrimL", False, "a", argchars("p"))], + [VArgChar("TrimLMax", False, "a", argchars("p"))], + [VArgChar("Length", False, "a", [])], + ], + [], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_param_exp_names() -> None: + script = "echo ${!arr*} ${!arr@}" + expected = [ + CommandNode( + 1, + [], + [argchars("echo"), argchars("${!arr*}"), argchars("${!arr@}")], + [], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_proc_subst_ops() -> None: + script = "echo <(echo hi) >(cat)" + expected = [ + CommandNode( + 1, + [], + [ + argchars("echo"), + [ + PArgChar( + "<(", CommandNode(1, [], [argchars("echo"), argchars("hi")], []) + ) + ], + [PArgChar(">(", CommandNode(1, [], [argchars("cat")], []))], + ], + [], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_extglob_ops() -> None: + script = "echo @(a) ?(b) *(c) +(d) !(e)" + expected = [ + CommandNode( + 1, + [], + [ + argchars("echo"), + argchars("@(a)"), + argchars("?(b)"), + argchars("*(c)"), + argchars("+(d)"), + argchars("!(e)"), + ], + [], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_case_ops() -> None: + script = "case x in a) echo hi ;& b) echo bye ;; esac" + expected = [ + CaseNode( + line_number=1, + argument=argchars("x"), + cases=[ + { + "cpattern": [argchars("a")], + "cbody": CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + "fallthrough": True, + }, + { + "cpattern": [argchars("b")], + "cbody": CommandNode( + 1, [], [argchars("echo"), argchars("bye")], [] + ), + "fallthrough": False, + }, + ], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_arithm_ops() -> None: + cases = [ + ("((1+2))", "1 + 2"), + ("((1-2))", "1 - 2"), + ("((1*2))", "1 * 2"), + ("((1/2))", "1 / 2"), + ("((1%2))", "1 % 2"), + ("((1**2))", "1 ** 2"), + ("((1==2))", "1 == 2"), + ("((1!=2))", "1 != 2"), + ("((1>2))", "1 > 2"), + ("((1<2))", "1 < 2"), + ("((1>=2))", "1 >= 2"), + ("((1<=2))", "1 <= 2"), + ("((1<<2))", "1 << 2"), + ("((1>>2))", "1 >> 2"), + ("((1&2))", "1 & 2"), + ("((1|2))", "1 | 2"), + ("((1^2))", "1 ^ 2"), + ("((1&&2))", "1 && 2"), + ("((1||2))", "1 || 2"), + ("((1,2))", "1 , 2"), + ("((1?2:3))", "1 ? 2 : 3"), + ("((i=1))", "i = 1"), + ("((i+=1))", "i += 1"), + ("((i-=1))", "i -= 1"), + ("((i*=1))", "i *= 1"), + ("((i/=1))", "i /= 1"), + ("((i%=1))", "i %= 1"), + ("((i&=1))", "i &= 1"), + ("((i|=1))", "i |= 1"), + ("((i^=1))", "i ^= 1"), + ("((i<<=1))", "i <<= 1"), + ("((i>>=1))", "i >>= 1"), + ("((!i))", "!i"), + ("((~i))", "~i"), + ("((++i))", "++i"), + ("((--i))", "--i"), + ("((i++))", "i++"), + ("((i--))", "i--"), + ] + for script, expr in cases: + expected = [ArithNode(line_number=1, body=[argchars(expr)])] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def test_shfmt_script_case_clause() -> None: + script = "case x in a) echo hi ;; esac" + expected = [ + CaseNode( + line_number=1, + argument=argchars("x"), + cases=[ + { + "cpattern": [argchars("a")], + "cbody": CommandNode(1, [], [argchars("echo"), argchars("hi")], []), + "fallthrough": False, + } + ], + ) + ] + + actual = shfmt_to_shasta_nodes(script) + + assert_shasta_equal(actual, expected) + + +def run_tests() -> int: + tests = [ + test_call_expr_with_assign, + test_binary_cmd_ops, + test_pipe_all_injects_dup, + test_if_and_while_until, + test_for_clause_variants, + test_case_clause_fallthrough, + test_redirection_variants, + test_word_parts_and_param_exp, + test_param_exp_names_fallback, + test_test_clause_binary, + test_file_to_nodes, + test_shfmt_script_simple_call, + test_shfmt_script_escape_param, + test_shfmt_script_escape_glob_literal, + test_shfmt_script_pipe_all, + test_shfmt_script_background, + test_shfmt_script_background_pipe, + test_shfmt_script_negation, + test_shfmt_script_time_clause, + test_shfmt_script_time_clause_posix, + test_shfmt_script_function_decl, + test_shfmt_script_coproc, + test_shfmt_script_subshell, + test_shfmt_script_group, + test_shfmt_script_group_redir, + test_shfmt_script_complex_one, + test_shfmt_script_complex_two, + test_shfmt_script_if_clause, + test_shfmt_script_test_clause, + test_shfmt_script_test_clause_flags, + test_shfmt_script_test_clause_binary_ops, + test_shfmt_script_binary_cmd_ops, + test_shfmt_script_redir_all, + test_shfmt_script_redir_ops, + test_shfmt_script_param_exp_ops, + test_shfmt_script_param_exp_names, + test_shfmt_script_proc_subst_ops, + test_shfmt_script_extglob_ops, + test_shfmt_script_case_ops, + test_shfmt_script_arithm_ops, + test_shfmt_script_case_clause, + ] + failures = 0 + for test in tests: + try: + test() + print(f"{test.__name__}: OK") + except AssertionError as exc: + failures += 1 + print(f"{test.__name__}: FAIL") + print(exc) + return failures + + +if __name__ == "__main__": + raise SystemExit(run_tests())