|
| 1 | +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin <bernard.ladenthin@gmail.com> |
| 2 | +# |
| 3 | +# SPDX-License-Identifier: MIT |
| 4 | + |
| 5 | +"""Drive the agent release jar as an editor would, over the Agent Client Protocol. |
| 6 | +
|
| 7 | +Starts the given command (the agent with --acp), speaks newline-delimited JSON-RPC 2.0 on its |
| 8 | +stdin/stdout the way JetBrains IDEs and Zed do, and checks what an editor would show: |
| 9 | +
|
| 10 | + 1. initialize and session/new answer, with the approval modes and the slash commands; |
| 11 | + 2. a plain prompt streams an answer containing "4" and ends with stopReason end_turn; |
| 12 | + 3. a prompt that needs read_file produces a tool_call of kind "read" and brings the marker |
| 13 | + from the session's working directory back. |
| 14 | +
|
| 15 | +Only the standard library, so it runs on a stock runner. Everything the agent sent is written |
| 16 | +to the log file given as the second argument, its stderr next to it as <name>.err.log. |
| 17 | +
|
| 18 | +Usage: agent_acp_smoke.py <workspace> <log> -- <command...> |
| 19 | +""" |
| 20 | + |
| 21 | +import json |
| 22 | +import os |
| 23 | +import subprocess |
| 24 | +import sys |
| 25 | +import threading |
| 26 | +import time |
| 27 | + |
| 28 | +TIMEOUT = float(os.environ.get("AGENT_SMOKE_TIMEOUT", "600")) |
| 29 | + |
| 30 | + |
| 31 | +def fail(message): |
| 32 | + print(f"::error::{message}", file=sys.stderr) |
| 33 | + sys.exit(1) |
| 34 | + |
| 35 | + |
| 36 | +def main(): |
| 37 | + if len(sys.argv) < 5 or sys.argv[3] != "--": |
| 38 | + fail("usage: agent_acp_smoke.py <workspace> <log> -- <command...>") |
| 39 | + workspace, log_path, command = sys.argv[1], sys.argv[2], sys.argv[4:] |
| 40 | + err_path = (log_path[:-4] if log_path.endswith(".log") else log_path) + ".err.log" |
| 41 | + marker = f"ACP_SMOKE_{int(time.time())}" |
| 42 | + with open(os.path.join(workspace, "marker.txt"), "w", encoding="utf-8") as f: |
| 43 | + f.write(marker + "\n") |
| 44 | + |
| 45 | + with open(err_path, "w", encoding="utf-8") as err_log: |
| 46 | + agent = subprocess.Popen( |
| 47 | + command, |
| 48 | + stdin=subprocess.PIPE, |
| 49 | + stdout=subprocess.PIPE, |
| 50 | + stderr=err_log, |
| 51 | + text=True, |
| 52 | + encoding="utf-8", |
| 53 | + bufsize=1, |
| 54 | + ) |
| 55 | + received = [] |
| 56 | + lock = threading.Condition() |
| 57 | + |
| 58 | + def read(): |
| 59 | + with open(log_path, "w", encoding="utf-8") as log: |
| 60 | + for line in agent.stdout: |
| 61 | + log.write(line) |
| 62 | + log.flush() |
| 63 | + line = line.strip() |
| 64 | + if not line: |
| 65 | + continue |
| 66 | + try: |
| 67 | + message = json.loads(line) |
| 68 | + except ValueError: |
| 69 | + # stdout is the protocol: anything else on it would break a real editor |
| 70 | + message = {"__not_json__": line} |
| 71 | + with lock: |
| 72 | + received.append(message) |
| 73 | + lock.notify_all() |
| 74 | + |
| 75 | + threading.Thread(target=read, daemon=True).start() |
| 76 | + ids = iter(range(1, 1000)) |
| 77 | + |
| 78 | + def send(message): |
| 79 | + agent.stdin.write(json.dumps(message) + "\n") |
| 80 | + agent.stdin.flush() |
| 81 | + |
| 82 | + def request(method, params): |
| 83 | + request_id = next(ids) |
| 84 | + send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) |
| 85 | + return request_id |
| 86 | + |
| 87 | + def response(request_id): |
| 88 | + deadline = time.time() + TIMEOUT |
| 89 | + with lock: |
| 90 | + while time.time() < deadline: |
| 91 | + for message in received: |
| 92 | + if "__not_json__" in message: |
| 93 | + fail(f"the agent wrote something other than JSON-RPC to stdout: {message['__not_json__']!r}") |
| 94 | + if message.get("id") == request_id and "method" not in message: |
| 95 | + if "error" in message: |
| 96 | + fail(f"request {request_id} failed: {message['error']}") |
| 97 | + return message["result"] |
| 98 | + if message.get("method") == "session/request_permission": |
| 99 | + # nothing this smoke asks for is gated; a question means the gate moved |
| 100 | + fail(f"unexpected permission request: {message}") |
| 101 | + if agent.poll() is not None: |
| 102 | + fail(f"the agent exited with {agent.returncode} before answering request {request_id}") |
| 103 | + lock.wait(1) |
| 104 | + fail(f"no answer to request {request_id} within {TIMEOUT:.0f}s") |
| 105 | + |
| 106 | + def updates(kind, since): |
| 107 | + with lock: |
| 108 | + return [ |
| 109 | + m["params"]["update"] |
| 110 | + for m in received[since:] |
| 111 | + if m.get("method") == "session/update" and m["params"]["update"].get("sessionUpdate") == kind |
| 112 | + ] |
| 113 | + |
| 114 | + try: |
| 115 | + init = response(request("initialize", {"protocolVersion": 1, "clientCapabilities": {}})) |
| 116 | + if init.get("protocolVersion") != 1: |
| 117 | + fail(f"unexpected protocol version: {init}") |
| 118 | + session = response(request("session/new", {"cwd": workspace, "mcpServers": []})) |
| 119 | + session_id = session["sessionId"] |
| 120 | + modes = [m["id"] for m in session["modes"]["availableModes"]] |
| 121 | + if modes != ["manual", "auto"]: |
| 122 | + fail(f"unexpected session modes: {modes}") |
| 123 | + print(f"OK: initialize + session/new (modes {modes})") |
| 124 | + |
| 125 | + def prompt(text): |
| 126 | + since = len(received) |
| 127 | + result = response( |
| 128 | + request("session/prompt", {"sessionId": session_id, "prompt": [{"type": "text", "text": text}]}) |
| 129 | + ) |
| 130 | + answer = "".join(u["content"].get("text", "") for u in updates("agent_message_chunk", since)) |
| 131 | + return result, answer, since |
| 132 | + |
| 133 | + result, answer, _ = prompt("What is 2 + 2? Answer with one short sentence.") |
| 134 | + if result.get("stopReason") != "end_turn": |
| 135 | + fail(f"plain prompt ended with {result}") |
| 136 | + if "4" not in answer: |
| 137 | + fail(f"the answer does not contain 4: {answer!r}") |
| 138 | + print(f"OK: plain answer over ACP ({answer.strip()[:80]!r})") |
| 139 | + |
| 140 | + commands = [c["name"] for c in updates("available_commands_update", 0)[-1]["availableCommands"]] |
| 141 | + if "status" not in commands: |
| 142 | + fail(f"slash commands were not announced: {commands}") |
| 143 | + print(f"OK: {len(commands)} slash commands announced") |
| 144 | + |
| 145 | + result, answer, since = prompt("Read the file marker.txt with the read_file tool and tell me its exact content.") |
| 146 | + calls = updates("tool_call", since) |
| 147 | + if not any(c.get("kind") == "read" and c.get("title", "").startswith("read_file") for c in calls): |
| 148 | + fail(f"no read_file tool call: {calls}") |
| 149 | + seen = answer + json.dumps(updates("tool_call_update", since)) |
| 150 | + if marker not in seen: |
| 151 | + fail(f"the marker never reached the editor: {answer!r}") |
| 152 | + print("OK: tool round over ACP (read_file)") |
| 153 | + finally: |
| 154 | + agent.stdin.close() |
| 155 | + try: |
| 156 | + agent.wait(timeout=30) |
| 157 | + except subprocess.TimeoutExpired: |
| 158 | + agent.kill() |
| 159 | + fail("the agent did not exit after the editor closed stdin") |
| 160 | + if agent.returncode != 0: |
| 161 | + fail(f"the agent exited with {agent.returncode} after the editor closed stdin") |
| 162 | + print("OK: exits cleanly when the editor hangs up") |
| 163 | + |
| 164 | + |
| 165 | +if __name__ == "__main__": |
| 166 | + main() |
0 commit comments