Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions docs/motion-dataset.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# 명령값 대 실제 이동 데이터셋 (#102)

#99 실측(docs/r11-closed-loop-field-validation.md)과 Agent 로그를 묶어 움직임 보정(#106) 학습용 CSV 를 만든다.
로봇·서비스에 접근하지 않는 오프라인 변환이다.

## 만들기

```bash
journalctl -u poppy-agent -o cat --since '2026-11-12 09:00' > agent.log # 운영자
python3 scripts/motion_dataset_export.py agent.log --measurements measured.csv --output dataset.csv
```

stderr 에 `rows=N missing_measurement=M` 을 출력한다. 실측이 없는 행도 버리지 않고 `missing_measurement=1` 로 남긴다.

## 실측 CSV (현장 기록)

| 열 | 의미 |
|---|---|
| `execution_id` | 실행 ID 전체 또는 앞 8자 |
| `motion_index` | 그 실행 안의 MOVE·TURN 순번(1부터, WAIT·자세·프리셋 제외) |
| `measured` | 실측값. MOVE 는 m, TURN 은 ° (명령 단위와 같음) |
| `lateral_cm` | MOVE 옆 이탈(cm), 없으면 빈칸 |
| `surface` | 바닥 종류(예: `mat`, `wood`) |
| `note` | 비고(개인정보 금지) |

## 출력 열

`timestamp`, `execution_id`, `motion_index`, `command_type`, `direction`, `commanded`, `unit`(`m`/`deg`),
`command_rate`(MOVE `vx` m/s, TURN `vyaw` rad/s), `planned_seconds`, `elapsed_seconds`, `stop_move_successes`,
`motion_status`, `achieved`(#97 폐루프의 `StopMove` 직전 도달값, 없으면 빈칸), `measured`, `error`(= measured − commanded),
`lateral_cm`, `surface`, `note`, `missing_measurement`.

- 실행 단위 연결: 각 동작 행은 직전 `execution_started` 의 `execution_id` 에 붙는다.
- 신선도·실패: `motion_status` 가 `FAILED`/`CANCELLED` 인 행도 남긴다. 학습 때 거를지는 사용하는 쪽이 정한다.

## 개인정보·비밀

Agent 로그의 allow-list 필드만 읽는다. `agent_id`, `robot_id`, `approval_ref`, IP, 토큰, 사람 이름·사진은 출력하지 않는다.
데이터셋 파일은 저장소에 커밋하지 않고 `~/poppy-ai/data` 에 둔다.
167 changes: 167 additions & 0 deletions scripts/motion_dataset_export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
"""Join Agent motion logs with field measurements into a CSV dataset (#102).

Input 1: Agent structured JSON logs (``journalctl -u poppy-agent -o cat`` or
lines with a prefix; the JSON object starts at the first ``{``). Each
``execution_physical_turn_finished`` event (MOVE and TURN) becomes one row,
numbered per Execution in log order (``motion_index`` from 1). The Execution
is the most recent ``execution_started`` before it.

Input 2 (optional): the field measurement CSV from
docs/r11-closed-loop-field-validation.md with columns ``execution_id``
(full id or its first 8 characters), ``motion_index``, ``measured``,
``lateral_cm``, ``surface`` and ``note``. ``measured`` uses the command unit
(meters for MOVE, degrees for TURN).

Output: CSV on stdout or ``--output``. Rows without a measurement keep empty
measurement fields and ``missing_measurement=1``; nothing is dropped silently.
Only allow-listed, secret-free log fields are read.
"""

from __future__ import annotations

import argparse
import csv
import json
import sys
from collections.abc import Iterable, Iterator, Sequence
from pathlib import Path
from typing import TextIO

MOTION_EVENT = "execution_physical_turn_finished"
STARTED_EVENT = "execution_started"
COLUMNS = (
"timestamp",
"execution_id",
"motion_index",
"command_type",
"direction",
"commanded",
"unit",
"command_rate",
"planned_seconds",
"elapsed_seconds",
"stop_move_successes",
"motion_status",
"achieved",
"measured",
"error",
"lateral_cm",
"surface",
"note",
"missing_measurement",
)


def parse_log_lines(lines: Iterable[str]) -> Iterator[dict[str, object]]:
for line in lines:
start = line.find("{")
if start < 0:
continue
try:
record = json.loads(line[start:])
except ValueError:
continue
if isinstance(record, dict) and isinstance(record.get("event"), str):
yield record


def motion_rows(records: Iterable[dict[str, object]]) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
execution_id = ""
index = 0
for record in records:
context = record.get("context")
context = context if isinstance(context, dict) else {}
if record["event"] == STARTED_EVENT:
execution_id = str(context.get("execution_id", ""))
index = 0
continue
if record["event"] != MOTION_EVENT:
continue
index += 1
turn = context.get("command_type") == "TURN"
rows.append(
{
"timestamp": record.get("timestamp", ""),
"execution_id": execution_id,
"motion_index": index,
"command_type": context.get("command_type", ""),
"direction": context.get("direction", ""),
"commanded": context.get("angle_degrees" if turn else "distance_meters", ""),
"unit": "deg" if turn else "m",
"command_rate": context.get("vyaw" if turn else "vx", ""),
"planned_seconds": context.get("duration_seconds", ""),
"elapsed_seconds": context.get("elapsed_seconds", ""),
"stop_move_successes": context.get("stop_move_successes", ""),
"motion_status": context.get("execution_status", ""),
"achieved": context.get("achieved_magnitude", ""),
}
)
return rows

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 필수 수정 · HIGH

전체 실행 ID도 8자로 잘라 서로 다른 실행을 합칩니다

측정 CSV가 전체 UUID를 제공해도 키를 앞 8자로 축약합니다. 같은 접두사를 가진 두 실행의 동일한 motion_index가 있으면 뒤에서 읽은 측정값이 앞선 값을 덮어쓰고, 조인 시 두 실행 모두 같은 측정값을 받습니다. 전체 ID는 그대로 매칭하고, 짧은 ID는 유일성이 확인될 때만 해석해야 합니다.

이유: UUID 접두사 충돌은 데이터가 누적될수록 현실적인 위험이며, 결과 CSV만 보고는 잘못된 연결을 판별하기 어렵습니다.

제안: 전체 ID 보존과 짧은 ID의 유일성 검증을 구현하고, 접두사 충돌 테스트를 추가하세요.



def load_measurements(stream: TextIO) -> dict[tuple[str, int], dict[str, str]]:
measurements: dict[tuple[str, int], dict[str, str]] = {}
for row in csv.DictReader(stream):
key = (row.get("execution_id", "").strip()[:8], int(row.get("motion_index", "0")))
measurements[key] = row
return measurements


def join(
rows: Sequence[dict[str, object]], measurements: dict[tuple[str, int], dict[str, str]]
) -> list[dict[str, object]]:
joined = []
for row in rows:
key = (str(row["execution_id"])[:8], int(str(row["motion_index"])))
measured = measurements.get(key)
out = dict(row)
value = (measured or {}).get("measured", "").strip()
if value:
out["measured"] = value
try:
out["error"] = round(float(value) - float(str(row["commanded"])), 4)
except ValueError:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔷 수정 권장 · MEDIUM

잘못된 측정값을 정상 측정으로 표시한 채 통과시킵니다

measured가 비어 있지 않지만 숫자가 아니면 변환 오류를 삼키고 error만 빈칸으로 둔 뒤 missing_measurement=0으로 기록합니다. 그 결과 손상된 실측 문자열이 정상 라벨처럼 데이터셋에 남습니다. 비유한값(NaN, Infinity)도 현재 검사를 통과할 수 있습니다.

이유: 학습용 CSV에서는 입력 오류를 조용히 정상 데이터로 분류하는 것이 단순한 표시 문제보다 큰 데이터 품질 위험입니다.

제안: 유한 숫자 검증과 실패 동작을 명확히 하고, 비숫자 및 NaN/Infinity 입력 테스트를 추가하세요.

out["error"] = ""
out["missing_measurement"] = 0
else:
out["measured"] = out["error"] = ""
out["missing_measurement"] = 1
for name in ("lateral_cm", "surface", "note"):
out[name] = (measured or {}).get(name, "")
joined.append(out)
return joined


def write_csv(rows: Sequence[dict[str, object]], stream: TextIO) -> None:
writer = csv.DictWriter(stream, fieldnames=COLUMNS, lineterminator="\n")
writer.writeheader()
for row in rows:
writer.writerow({name: row.get(name, "") for name in COLUMNS})


def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("log", type=Path, help="Agent JSON log file")
parser.add_argument("--measurements", type=Path, help="field measurement CSV")
parser.add_argument("--output", type=Path, help="output CSV (default stdout)")
args = parser.parse_args(argv)
with args.log.open(encoding="utf-8") as log:
rows = motion_rows(parse_log_lines(log))
measurements: dict[tuple[str, int], dict[str, str]] = {}
if args.measurements is not None:
with args.measurements.open(encoding="utf-8", newline="") as stream:
measurements = load_measurements(stream)
joined = join(rows, measurements)
if args.output is None:
write_csv(joined, sys.stdout)
else:
with args.output.open("w", encoding="utf-8", newline="") as stream:
write_csv(joined, stream)
missing = sum(1 for row in joined if row["missing_measurement"] == 1)
print(f"rows={len(joined)} missing_measurement={missing}", file=sys.stderr)
return 0


if __name__ == "__main__":
sys.exit(main())
126 changes: 126 additions & 0 deletions tests/test_motion_dataset_export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
"""scripts/motion_dataset_export.py with synthetic log lines only (#102)."""

from __future__ import annotations

import csv
import importlib.util
import io
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location(
"motion_dataset_export", ROOT / "scripts" / "motion_dataset_export.py"
)
assert SPEC is not None and SPEC.loader is not None
export = importlib.util.module_from_spec(SPEC)
sys.modules["motion_dataset_export"] = export
SPEC.loader.exec_module(export)

EXEC_A = "0e1f2a3b-0000-0000-0000-000000000001"
EXEC_B = "9f8e7d6c-0000-0000-0000-000000000002"


def line(event: str, prefix: str = "", **context: object) -> str:
payload = {"timestamp": "2026-11-12T00:00:00+00:00", "event": event, "context": context}
return prefix + json.dumps(payload) + "\n"


LOG = [
"-- Boot abc --\n",
line("execution_started", execution_id=EXEC_A),
line(
"execution_physical_turn_finished",
prefix="Nov 12 09:00:01 host python[1]: ",
execution_status="COMPLETED",
command_type="MOVE",
direction="FORWARD",
distance_meters=1.0,
vx=0.5,
duration_seconds=2.0,
elapsed_seconds=2.01,
stop_move_successes=3,
),
line(
"execution_physical_turn_finished",
execution_status="COMPLETED",
command_type="TURN",
direction="LEFT",
angle_degrees=90,
vyaw=0.785,
duration_seconds=2.0,
achieved_magnitude=88.5,
),
"not json {broken\n",
line("execution_started", execution_id=EXEC_B),
line(
"execution_physical_turn_finished",
execution_status="FAILED",
command_type="MOVE",
direction="BACKWARD",
distance_meters=0.5,
),
line("heartbeat_sent"),
]

MEASUREMENTS = """execution_id,motion_index,measured,lateral_cm,surface,note
0e1f2a3b,1,1.04,2,mat,
0e1f2a3b-0000-0000-0000-000000000001,2,91.0,,mat,clockwise check
"""


def run_export() -> list[dict[str, str]]:
rows = export.motion_rows(export.parse_log_lines(LOG))
joined = export.join(rows, export.load_measurements(io.StringIO(MEASUREMENTS)))
out = io.StringIO()
export.write_csv(joined, out)
out.seek(0)
return list(csv.DictReader(out))


def test_rows_are_numbered_per_execution_with_units() -> None:
rows = run_export()
assert [(r["execution_id"][:8], r["motion_index"]) for r in rows] == [
("0e1f2a3b", "1"),
("0e1f2a3b", "2"),
("9f8e7d6c", "1"),
]
assert [(r["command_type"], r["commanded"], r["unit"]) for r in rows] == [
("MOVE", "1.0", "m"),
("TURN", "90", "deg"),
("MOVE", "0.5", "m"),
]
assert rows[0]["command_rate"] == "0.5" and rows[1]["command_rate"] == "0.785"
assert rows[1]["achieved"] == "88.5"


def test_measurements_join_by_short_or_full_id_and_compute_error() -> None:
rows = run_export()
assert (rows[0]["measured"], rows[0]["error"], rows[0]["lateral_cm"]) == ("1.04", "0.04", "2")
assert (rows[1]["measured"], rows[1]["error"]) == ("91.0", "1.0")
assert rows[1]["surface"] == "mat"


def test_missing_measurement_is_kept_and_flagged() -> None:
rows = run_export()
assert rows[2]["missing_measurement"] == "1"
assert rows[2]["measured"] == "" and rows[2]["motion_status"] == "FAILED"
assert [r["missing_measurement"] for r in rows[:2]] == ["0", "0"]


def test_cli_writes_csv_and_reports_counts(tmp_path: Path, capsys) -> None:
log = tmp_path / "agent.log"
log.write_text("".join(LOG), encoding="utf-8")
measured = tmp_path / "measured.csv"
measured.write_text(MEASUREMENTS, encoding="utf-8")
output = tmp_path / "dataset.csv"

assert export.main([str(log), "--measurements", str(measured), "--output", str(output)]) == 0
assert output.read_text(encoding="utf-8").splitlines()[0].startswith("timestamp,execution_id")
assert "rows=3 missing_measurement=1" in capsys.readouterr().err


def test_columns_hold_no_identity_or_secret_fields() -> None:
forbidden = {"agent_id", "robot_id", "approval_ref", "token", "ip"}
assert forbidden.isdisjoint(export.COLUMNS)
Loading