diff --git a/docs/motion-dataset.md b/docs/motion-dataset.md new file mode 100644 index 0000000..347d4d0 --- /dev/null +++ b/docs/motion-dataset.md @@ -0,0 +1,39 @@ +# 명령값 대 실제 이동 데이터셋 (#102) + +#99 실측(docs/r11-closed-loop-field-validation.md)과 Agent 로그를 묶어 움직임 보정(#106) 학습용 CSV 를 만든다. +로봇·서비스에 접근하지 않는 오프라인 변환이다. + +## 만들기 + +```bash +journalctl -u poppy-agent -o cat --since '2026-11-12 09:00' > agent.log # 운영자 +python3 scripts/motion_dataset_export.py agent.log --measurements measured.csv --output dataset.csv +``` + +stderr 에 `rows=N missing_measurement=M` 을 출력한다. 실측이 없는 행도 버리지 않고 `missing_measurement=1` 로 남긴다. + +## 실측 CSV (현장 기록) + +| 열 | 의미 | +|---|---| +| `execution_id` | 실행 ID 전체 또는 앞 8자 | +| `motion_index` | 그 실행 안의 MOVE·TURN 순번(1부터, WAIT·자세·프리셋 제외) | +| `measured` | 실측값. MOVE 는 m, TURN 은 ° (명령 단위와 같음) | +| `lateral_cm` | MOVE 옆 이탈(cm), 없으면 빈칸 | +| `surface` | 바닥 종류(예: `mat`, `wood`) | +| `note` | 비고(개인정보 금지) | + +## 출력 열 + +`timestamp`, `execution_id`, `motion_index`, `command_type`, `direction`, `commanded`, `unit`(`m`/`deg`), +`command_rate`(MOVE `vx` m/s, TURN `vyaw` rad/s), `planned_seconds`, `elapsed_seconds`, `stop_move_successes`, +`motion_status`, `achieved`(#97 폐루프의 `StopMove` 직전 도달값, 없으면 빈칸), `measured`, `error`(= measured − commanded), +`lateral_cm`, `surface`, `note`, `missing_measurement`. + +- 실행 단위 연결: 각 동작 행은 직전 `execution_started` 의 `execution_id` 에 붙는다. +- 신선도·실패: `motion_status` 가 `FAILED`/`CANCELLED` 인 행도 남긴다. 학습 때 거를지는 사용하는 쪽이 정한다. + +## 개인정보·비밀 + +Agent 로그의 allow-list 필드만 읽는다. `agent_id`, `robot_id`, `approval_ref`, IP, 토큰, 사람 이름·사진은 출력하지 않는다. +데이터셋 파일은 저장소에 커밋하지 않고 `~/poppy-ai/data` 에 둔다. diff --git a/scripts/motion_dataset_export.py b/scripts/motion_dataset_export.py new file mode 100644 index 0000000..bb80feb --- /dev/null +++ b/scripts/motion_dataset_export.py @@ -0,0 +1,167 @@ +"""Join Agent motion logs with field measurements into a CSV dataset (#102). + +Input 1: Agent structured JSON logs (``journalctl -u poppy-agent -o cat`` or +lines with a prefix; the JSON object starts at the first ``{``). Each +``execution_physical_turn_finished`` event (MOVE and TURN) becomes one row, +numbered per Execution in log order (``motion_index`` from 1). The Execution +is the most recent ``execution_started`` before it. + +Input 2 (optional): the field measurement CSV from +docs/r11-closed-loop-field-validation.md with columns ``execution_id`` +(full id or its first 8 characters), ``motion_index``, ``measured``, +``lateral_cm``, ``surface`` and ``note``. ``measured`` uses the command unit +(meters for MOVE, degrees for TURN). + +Output: CSV on stdout or ``--output``. Rows without a measurement keep empty +measurement fields and ``missing_measurement=1``; nothing is dropped silently. +Only allow-listed, secret-free log fields are read. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from collections.abc import Iterable, Iterator, Sequence +from pathlib import Path +from typing import TextIO + +MOTION_EVENT = "execution_physical_turn_finished" +STARTED_EVENT = "execution_started" +COLUMNS = ( + "timestamp", + "execution_id", + "motion_index", + "command_type", + "direction", + "commanded", + "unit", + "command_rate", + "planned_seconds", + "elapsed_seconds", + "stop_move_successes", + "motion_status", + "achieved", + "measured", + "error", + "lateral_cm", + "surface", + "note", + "missing_measurement", +) + + +def parse_log_lines(lines: Iterable[str]) -> Iterator[dict[str, object]]: + for line in lines: + start = line.find("{") + if start < 0: + continue + try: + record = json.loads(line[start:]) + except ValueError: + continue + if isinstance(record, dict) and isinstance(record.get("event"), str): + yield record + + +def motion_rows(records: Iterable[dict[str, object]]) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + execution_id = "" + index = 0 + for record in records: + context = record.get("context") + context = context if isinstance(context, dict) else {} + if record["event"] == STARTED_EVENT: + execution_id = str(context.get("execution_id", "")) + index = 0 + continue + if record["event"] != MOTION_EVENT: + continue + index += 1 + turn = context.get("command_type") == "TURN" + rows.append( + { + "timestamp": record.get("timestamp", ""), + "execution_id": execution_id, + "motion_index": index, + "command_type": context.get("command_type", ""), + "direction": context.get("direction", ""), + "commanded": context.get("angle_degrees" if turn else "distance_meters", ""), + "unit": "deg" if turn else "m", + "command_rate": context.get("vyaw" if turn else "vx", ""), + "planned_seconds": context.get("duration_seconds", ""), + "elapsed_seconds": context.get("elapsed_seconds", ""), + "stop_move_successes": context.get("stop_move_successes", ""), + "motion_status": context.get("execution_status", ""), + "achieved": context.get("achieved_magnitude", ""), + } + ) + return rows + + +def load_measurements(stream: TextIO) -> dict[tuple[str, int], dict[str, str]]: + measurements: dict[tuple[str, int], dict[str, str]] = {} + for row in csv.DictReader(stream): + key = (row.get("execution_id", "").strip()[:8], int(row.get("motion_index", "0"))) + measurements[key] = row + return measurements + + +def join( + rows: Sequence[dict[str, object]], measurements: dict[tuple[str, int], dict[str, str]] +) -> list[dict[str, object]]: + joined = [] + for row in rows: + key = (str(row["execution_id"])[:8], int(str(row["motion_index"]))) + measured = measurements.get(key) + out = dict(row) + value = (measured or {}).get("measured", "").strip() + if value: + out["measured"] = value + try: + out["error"] = round(float(value) - float(str(row["commanded"])), 4) + except ValueError: + out["error"] = "" + out["missing_measurement"] = 0 + else: + out["measured"] = out["error"] = "" + out["missing_measurement"] = 1 + for name in ("lateral_cm", "surface", "note"): + out[name] = (measured or {}).get(name, "") + joined.append(out) + return joined + + +def write_csv(rows: Sequence[dict[str, object]], stream: TextIO) -> None: + writer = csv.DictWriter(stream, fieldnames=COLUMNS, lineterminator="\n") + writer.writeheader() + for row in rows: + writer.writerow({name: row.get(name, "") for name in COLUMNS}) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("log", type=Path, help="Agent JSON log file") + parser.add_argument("--measurements", type=Path, help="field measurement CSV") + parser.add_argument("--output", type=Path, help="output CSV (default stdout)") + args = parser.parse_args(argv) + with args.log.open(encoding="utf-8") as log: + rows = motion_rows(parse_log_lines(log)) + measurements: dict[tuple[str, int], dict[str, str]] = {} + if args.measurements is not None: + with args.measurements.open(encoding="utf-8", newline="") as stream: + measurements = load_measurements(stream) + joined = join(rows, measurements) + if args.output is None: + write_csv(joined, sys.stdout) + else: + with args.output.open("w", encoding="utf-8", newline="") as stream: + write_csv(joined, stream) + missing = sum(1 for row in joined if row["missing_measurement"] == 1) + print(f"rows={len(joined)} missing_measurement={missing}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_motion_dataset_export.py b/tests/test_motion_dataset_export.py new file mode 100644 index 0000000..710bd36 --- /dev/null +++ b/tests/test_motion_dataset_export.py @@ -0,0 +1,126 @@ +"""scripts/motion_dataset_export.py with synthetic log lines only (#102).""" + +from __future__ import annotations + +import csv +import importlib.util +import io +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "motion_dataset_export", ROOT / "scripts" / "motion_dataset_export.py" +) +assert SPEC is not None and SPEC.loader is not None +export = importlib.util.module_from_spec(SPEC) +sys.modules["motion_dataset_export"] = export +SPEC.loader.exec_module(export) + +EXEC_A = "0e1f2a3b-0000-0000-0000-000000000001" +EXEC_B = "9f8e7d6c-0000-0000-0000-000000000002" + + +def line(event: str, prefix: str = "", **context: object) -> str: + payload = {"timestamp": "2026-11-12T00:00:00+00:00", "event": event, "context": context} + return prefix + json.dumps(payload) + "\n" + + +LOG = [ + "-- Boot abc --\n", + line("execution_started", execution_id=EXEC_A), + line( + "execution_physical_turn_finished", + prefix="Nov 12 09:00:01 host python[1]: ", + execution_status="COMPLETED", + command_type="MOVE", + direction="FORWARD", + distance_meters=1.0, + vx=0.5, + duration_seconds=2.0, + elapsed_seconds=2.01, + stop_move_successes=3, + ), + line( + "execution_physical_turn_finished", + execution_status="COMPLETED", + command_type="TURN", + direction="LEFT", + angle_degrees=90, + vyaw=0.785, + duration_seconds=2.0, + achieved_magnitude=88.5, + ), + "not json {broken\n", + line("execution_started", execution_id=EXEC_B), + line( + "execution_physical_turn_finished", + execution_status="FAILED", + command_type="MOVE", + direction="BACKWARD", + distance_meters=0.5, + ), + line("heartbeat_sent"), +] + +MEASUREMENTS = """execution_id,motion_index,measured,lateral_cm,surface,note +0e1f2a3b,1,1.04,2,mat, +0e1f2a3b-0000-0000-0000-000000000001,2,91.0,,mat,clockwise check +""" + + +def run_export() -> list[dict[str, str]]: + rows = export.motion_rows(export.parse_log_lines(LOG)) + joined = export.join(rows, export.load_measurements(io.StringIO(MEASUREMENTS))) + out = io.StringIO() + export.write_csv(joined, out) + out.seek(0) + return list(csv.DictReader(out)) + + +def test_rows_are_numbered_per_execution_with_units() -> None: + rows = run_export() + assert [(r["execution_id"][:8], r["motion_index"]) for r in rows] == [ + ("0e1f2a3b", "1"), + ("0e1f2a3b", "2"), + ("9f8e7d6c", "1"), + ] + assert [(r["command_type"], r["commanded"], r["unit"]) for r in rows] == [ + ("MOVE", "1.0", "m"), + ("TURN", "90", "deg"), + ("MOVE", "0.5", "m"), + ] + assert rows[0]["command_rate"] == "0.5" and rows[1]["command_rate"] == "0.785" + assert rows[1]["achieved"] == "88.5" + + +def test_measurements_join_by_short_or_full_id_and_compute_error() -> None: + rows = run_export() + assert (rows[0]["measured"], rows[0]["error"], rows[0]["lateral_cm"]) == ("1.04", "0.04", "2") + assert (rows[1]["measured"], rows[1]["error"]) == ("91.0", "1.0") + assert rows[1]["surface"] == "mat" + + +def test_missing_measurement_is_kept_and_flagged() -> None: + rows = run_export() + assert rows[2]["missing_measurement"] == "1" + assert rows[2]["measured"] == "" and rows[2]["motion_status"] == "FAILED" + assert [r["missing_measurement"] for r in rows[:2]] == ["0", "0"] + + +def test_cli_writes_csv_and_reports_counts(tmp_path: Path, capsys) -> None: + log = tmp_path / "agent.log" + log.write_text("".join(LOG), encoding="utf-8") + measured = tmp_path / "measured.csv" + measured.write_text(MEASUREMENTS, encoding="utf-8") + output = tmp_path / "dataset.csv" + + assert export.main([str(log), "--measurements", str(measured), "--output", str(output)]) == 0 + assert output.read_text(encoding="utf-8").splitlines()[0].startswith("timestamp,execution_id") + assert "rows=3 missing_measurement=1" in capsys.readouterr().err + + +def test_columns_hold_no_identity_or_secret_fields() -> None: + forbidden = {"agent_id", "robot_id", "approval_ref", "token", "ip"} + assert forbidden.isdisjoint(export.COLUMNS)