Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion research/experiments/mechanism_replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,11 @@ def verify_replay_submission(runtime,episode,history,submission):
(runtime.campaign_id,work_id)).fetchall()
recorded=[json.loads(row[0]) for row in rows]
completed={row['action_work_id']:row for row in recorded}
submitted_actions=[action for entry in history for action in entry.get('actions',[]) if isinstance(action,Mapping)]
submitted_actions=[action for entry in history for action in entry.get('actions',[])
if isinstance(action,Mapping) and not (
action.get('action_name')=='read_context_segment' or
isinstance(action.get('result'),Mapping) and
action['result'].get('operation')=='read_context_segment')]
forged=[action.get('action_work_id') for action in submitted_actions if action.get('state')=='completed' and
(action.get('kind')!='replay_transition' or completed.get(action.get('action_work_id'))!=action)]
performed={row['result']['record_id'] for row in recorded if row['result']['record_id'] is not None}
Expand Down
13 changes: 13 additions & 0 deletions tests/test_mechanism_replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,19 @@ def test_unexecuted_and_forged_operations_cannot_certify_repair(tmp_path):
assert verify_replay_submission(runtime,episode,history,submission)['valid'] is False


def test_observational_context_reads_do_not_count_as_forged_scientific_actions(tmp_path):
runtime,episode,_=runtime_fixture(tmp_path)
history,submission=repair(runtime,episode)
history.append({'actions':[{'kind':'analysis','action_work_id':'episode--replay-fixture--read-context--page-1',
'state':'completed','result':{'operation':'read_context_segment','segment_id':'public-context',
'pointer':'','offset':0,'next_offset':32,'content':'observational context only'}}]})
verified=verify_replay_submission(runtime,episode,history,submission)
assert verified['forged_history_actions']==[]
assert verified['valid'] is True
assert verified['scientific_metric_values_generated'] is False
assert verified['gpu_actual_seconds']==0


def test_source_artifacts_are_reconciled_after_edit_not_implicitly_reused(tmp_path):
runtime,episode,prepared=runtime_fixture(tmp_path)
binding=prepared['action_bindings'][0]
Expand Down
34 changes: 34 additions & 0 deletions tests/test_study_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -1553,6 +1553,40 @@ def test_large_model_context_interning_preserves_exact_json_and_tail_text():
assert stats[0]["output_characters"] < stats[0]["input_characters"]


def test_aide_artifact_observation_payload_bloat_is_losslessly_shared_before_codex_transport():
"""Keep full evidence while sharing duplicated AIDE fields in the actual wire request."""
from research.experiments.study_context import restore_messages
from research.experiments.study_runtime import _intern_delivered_messages
from research.agents.codex_transport import request_payload

manifest = {"train_ids": [f"sample-{index:06d}" for index in range(10_000)],
"validation_ids": [f"validation-{index:06d}" for index in range(2_000)],
"dataset_contract": "registered-dataset-manifest," * 4_300}
training_config = {"optimizer": "AdamW", "learning_rate": 0.0003,
"batch_size": 64, "updates": 1200, "normalization": {"mean": [0.1] * 64,
"scale": [0.9] * 64}}
observations = [{"artifact_id": f"run-{index}", "dataset_manifest": manifest,
"training_configuration": training_config,
"validation_log": (f"validation-run-{index}:" + ("observed-value," * 6_500))}
for index in range(6)]
aide_context = {"view": "native_agent_public_artifact_index",
"artifacts": observations, "observations": observations}
original = [{"role": "user", "content": "AIDE public evidence:\n" +
json.dumps(aide_context, ensure_ascii=False, separators=(",", ":"))}]

delivered, stats = _intern_delivered_messages(original)
wire = request_payload(delivered, json_mode=True)
wire_characters = len(json.dumps(wire, ensure_ascii=False, separators=(",", ":")))

assert stats and stats[0]["input_characters"] > 5_000_000
assert stats[0]["output_characters"] < stats[0]["input_characters"]
# This is the observed Codex CLI input-envelope capacity, not an Agent
# reasoning quota. The repeated immutable evidence is still losslessly
# represented in the request, below that transport's real limit.
assert wire_characters < 1_050_000
assert restore_messages(delivered) == original


def test_downstream_executor_error_preserves_actual_model_response(tmp_path, monkeypatch):
campaign = tmp_path / "campaign"
campaign.mkdir()
Expand Down
Loading