评测编码Agent前，先用同一组任务跑两遍

dev.to · AI#AI评测#Coding Agent#基准测试

Editor brief · 编辑速览

slice24/ TASKS.md controls.yaml tasks/ T01_discount_rule/ prompt.md repo/ oracle/ test_discount.py sha256.txt T02_date_parse/ ... runs/ replay_a.jsonl replay_b.jsonl

# Proposal: example controls for a 24-task replay slice. # Label this as a template until you fill every field on your machine. slice_id: slice24-checkout-2026-09-13 unit_of_analysis: one_attempt_per_task max_attempts: 1 wall_clock_sec: 180 cpus: 2 memory_mb: 2048 network: denied writable_paths: ["repo/"] oracle_cmd: "python -m pytest oracle/ -q" oracle_sha256_file: oracle/sha256.txt seed_policy: fixed language_toolchain: "python3.11" forbidden_edits: ["oracle/", "controls.yaml"] replay_required: 2 max_replay_drift: 0

# proposal_harness.py — replay gate for a 24-task slice from __future__ import annotations import hashlib, json, subprocess, time from pathlib import Path SLICE = Path("slice24") WALL_CLOCK = 180 def sha256_tree(root: Path) -> str: h = hashlib.sha256() for p in sorted(root.rglob("*")): if p.is_file(): h.update(p.relative_to(root).as_posix().encode()) h.update(p.read_bytes()) return h.hexdigest() def oracle_intact(task: Path) -> bool: expected = (task / "oracle" / "sha256.txt").read_text().strip() return sha256_tree(task / "oracle") == expected def run_oracle(task: Path) -> bool: proc = subprocess.run( ["python", "-m", "pytest", str(task / "oracle"), "-q"], cwd=task / "repo", capture_output=True, timeout=WALL_CLOCK, ) return proc.returncode == 0 def run_agent(agent_id: str, task: Path) -> None: prompt = (task / "prompt.md").read_text() # Replace this stub with your agent CLI. Keep the timeout. subprocess.run( ["your-agent", "--id", agent_id, "--prompt", prompt, "--cwd", str(task / "repo")], timeout=WALL_CLOCK, check=False, ) def one_attempt(agent_id: str, task: Path) -> dict: if not oracle_intact(task): return {"task": task.name, "void": "oracle_before"} t0 = time.monotonic() run_agent(agent_id, task) elapsed = round(time.monotonic() - t0, 3) if not oracle_intact(task): return {"task": task.name, "agent": agent_id, "void": "oracle_mutated", "sec": elapsed} passed = run_oracle(task) return {"task": task.name, "agent": agent_id, "pass": int(passed), "sec": elapsed, "void": None} def replay_agent(agent_id: str) -> list[dict]: rows = [] for task in sorted((SLICE / "tasks").iterdir()): first = one_attempt(agent_id, task) second = one_attempt(agent_id, task) if first.get("pass") != second.get("pass") or first.get("void") or second.get("void"): rows.append({"task": task.name, "agent": agent_id, "void": "replay_drift"}) else: rows.append(first) return rows def paired_report(a: list[dict], b: list[dict]) -> dict: by_a = {r["task"]: r for r in a} wins = {"A": 0, "B": 0, "tie": 0, "void": 0} for r in b: t = r["task"] left, right = by_a[t], r if left.get("void") or right.get("void"): wins["void"] += 1 elif left.get("pass") and not right.get("pass"): wins["A"] += 1 elif right.get("pass") and not left.get("pass"): wins["B"] += 1 else: wins["tie"] += 1 return wins if __name__ == "__main__": a_rows = replay_agent("A") b_rows = replay_agent("B") Path("slice24/runs/replay_a.jsonl").write_text( "\n".join(json.dumps(r) for r in a_rows) + "\n" ) Path("slice24/runs/replay_b.jsonl").write_text( "\n".join(json.dumps(r) for r in b_rows) + "n" if False else "\n".join(json.dumps(r) for r in b_rows) + "\n" ) print(json.dumps(paired_report(a_rows, b_rows), indent=2))

find slice24/tasks -name sha256.txt -print python -m pytest slice24/tasks/T01_discount_rule/oracle -q sha256sum slice24/controls.yaml python proposal_harness.py diff -u slice24/runs/replay_a.jsonl slice24/runs/replay_a.jsonl.bak

完整中文译文