basic BMAD setup
This commit is contained in:
Ebbe Baß
@@ -0,0 +1,363 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# ///
|
||||
"""Run eval cases through the recorded harness.
|
||||
|
||||
A case is `input + rubric + optional state_prefix + optional files`. For each case and
|
||||
config the runner makes a clean room outside the project, copies the skill under test into
|
||||
the folder the harness reads skills from, stages the fixtures, runs the harness command once
|
||||
from the workspace, and records what it printed with timing and token usage. Grading happens
|
||||
elsewhere, from the transcript and artifacts left behind.
|
||||
|
||||
Modes (--mode) decide which configs each case runs under:
|
||||
|
||||
quality : one config, "skill", the skill staged in the workspace.
|
||||
baseline : "skill" and "bare" (nothing staged), same input, so the bare-model floor is
|
||||
measured under identical conditions.
|
||||
variant : "skill" (--skill-path) and "variant" (--variant-path).
|
||||
|
||||
Run layout: <run-dir>/<config>/<case-id>/ (plus /run-N/ when --runs > 1), each holding
|
||||
prompt.txt, transcript.jsonl, stderr.txt, timing.json and cwd/ (the workspace after the run),
|
||||
so `aggregate_benchmark.py --baseline <run-dir>/bare --variant <run-dir>/skill` compares
|
||||
configs from the timing.json files.
|
||||
|
||||
Harness: `[workflow.harness]` in this skill's customization, read through the project's
|
||||
resolver; `--harness <json>` with the same keys for a project without BMad. The keys and the
|
||||
clean room are described in eval_common.py. Without a harness the runner stages every case,
|
||||
records each as skipped and exits 3.
|
||||
|
||||
Usage:
|
||||
uv run run_evals.py --cases CASES.json --skill-path SKILL_DIR --output-dir DIR
|
||||
[--mode quality|baseline|variant] [--variant-path SKILL_DIR] [--project-root DIR]
|
||||
[--harness HARNESS.json] [--case-ids A1,B3] [--runs N] [--timeout SECS]
|
||||
[--workers N] [--label NAME] [--quiet]
|
||||
|
||||
CASES.json is a list of cases or {"cases": [...]}. Each case:
|
||||
{"id": "...", "input": "...", "rubric": [...], "state_prefix": "..."?, "files": ["..."]?}
|
||||
|
||||
Exit 0 when every run completed, 1 when any run errored or timed out or the command was
|
||||
not found, 2 on a usage error, 3 when no harness is recorded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
from eval_common import (
|
||||
DEFAULT_SKILL_DIR,
|
||||
account_transcript,
|
||||
contained,
|
||||
find_project_root,
|
||||
make_run_dir,
|
||||
read_json,
|
||||
resolve_harness,
|
||||
run_in_clean_room,
|
||||
safe_name,
|
||||
utc_now_iso,
|
||||
write_json,
|
||||
)
|
||||
|
||||
FAILURE_STATUSES = ("error", "timeout", "exception", "harness-missing")
|
||||
|
||||
# --- staging: skill under test + fixtures ------------------------------------
|
||||
|
||||
|
||||
def stage_skill(skill_path: Path, cwd: Path, skills_subdir: str) -> Path:
|
||||
"""Copy the skill where the harness discovers skills inside the workspace.
|
||||
|
||||
A copy, not a symlink: a run that edits its staged skill must not touch the source or
|
||||
the other cases.
|
||||
"""
|
||||
dest = cwd / skills_subdir / skill_path.name
|
||||
shutil.copytree(skill_path, dest, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", ".git"))
|
||||
return dest
|
||||
|
||||
|
||||
def resolve_fixtures(files: list, project_root: Path, cases_dir: Path) -> list[tuple[Path, str]]:
|
||||
"""Map each `files` entry to (source, dest-relative-path).
|
||||
|
||||
The entry's own relative path is preserved inside the workspace, so a bare filename lands
|
||||
at its root and a nested path keeps its directory structure, matching the path the case
|
||||
input references.
|
||||
"""
|
||||
out: list[tuple[Path, str]] = []
|
||||
for entry in files or []:
|
||||
entry = str(entry)
|
||||
for candidate in (
|
||||
(project_root / entry).resolve(),
|
||||
(cases_dir / entry).resolve(),
|
||||
Path(entry).resolve(),
|
||||
):
|
||||
if candidate.is_file():
|
||||
out.append((candidate, entry))
|
||||
break
|
||||
else:
|
||||
print(f"Warning: fixture not found: {entry}", file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def stage_fixtures(fixtures: list[tuple[Path, str]], cwd: Path) -> None:
|
||||
for src, dest_rel in fixtures:
|
||||
dest = contained(cwd, dest_rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dest)
|
||||
|
||||
|
||||
def compose_prompt(case: dict) -> str:
|
||||
"""Prepend the state_prefix, a bracketed prime that places the skill mid-workflow."""
|
||||
input_text = str(case.get("input", ""))
|
||||
prefix = case.get("state_prefix")
|
||||
if prefix:
|
||||
return f"{str(prefix).rstrip()}\n\n{input_text}"
|
||||
return input_text
|
||||
|
||||
|
||||
# --- per-case execution -----------------------------------------------------
|
||||
|
||||
|
||||
def run_case(
|
||||
case: dict,
|
||||
case_dir: Path,
|
||||
run_dir: Path,
|
||||
harness: dict | None,
|
||||
timeout: int,
|
||||
config: str,
|
||||
skill_path: Path | None,
|
||||
fixtures: list[tuple[Path, str]],
|
||||
) -> dict:
|
||||
case_id = str(case.get("id", "unnamed"))
|
||||
case_dir.mkdir(parents=True, exist_ok=True)
|
||||
prompt = compose_prompt(case)
|
||||
write_json(case_dir / "case.json", case)
|
||||
|
||||
def timing(status: str, **extra: object) -> None:
|
||||
write_json(case_dir / "timing.json", {"case_id": case_id, "config": config, "status": status, **extra})
|
||||
|
||||
def result(status: str, **extra: object) -> dict:
|
||||
return {
|
||||
"case_id": case_id,
|
||||
"config": config,
|
||||
"status": status,
|
||||
"case_dir": str(case_dir.relative_to(run_dir)),
|
||||
**extra,
|
||||
}
|
||||
|
||||
def stage(cwd: Path) -> None:
|
||||
stage_fixtures(fixtures, cwd)
|
||||
if skill_path is not None:
|
||||
stage_skill(skill_path, cwd, (harness or {}).get("skill_dir", DEFAULT_SKILL_DIR))
|
||||
|
||||
if harness is None:
|
||||
(case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
|
||||
try:
|
||||
stage(case_dir / "cwd")
|
||||
except ValueError as e:
|
||||
timing("error", reason=str(e), captured_at=utc_now_iso())
|
||||
return result("error", reason=str(e))
|
||||
timing("skipped", captured_at=utc_now_iso())
|
||||
return result("skipped", reason="no harness recorded", prompt_chars=len(prompt))
|
||||
|
||||
try:
|
||||
run = run_in_clean_room(harness, case_dir, prompt, timeout, stage)
|
||||
except ValueError as e:
|
||||
timing("error", reason=str(e), captured_at=utc_now_iso())
|
||||
return result("error", reason=str(e))
|
||||
|
||||
accounting = account_transcript(run["stdout"])
|
||||
timing(
|
||||
run["status"],
|
||||
elapsed_s=run["elapsed_s"],
|
||||
return_code=run["return_code"],
|
||||
input_tokens=accounting["input_tokens"],
|
||||
output_tokens=accounting["output_tokens"],
|
||||
total_tokens=accounting["total_tokens"],
|
||||
tokens_reported=accounting["tokens_reported"],
|
||||
total_steps=accounting["total_steps"],
|
||||
total_tool_calls=accounting["total_tool_calls"],
|
||||
captured_at=utc_now_iso(),
|
||||
)
|
||||
return result(
|
||||
run["status"],
|
||||
elapsed_s=run["elapsed_s"],
|
||||
return_code=run["return_code"],
|
||||
tokens=accounting["total_tokens"],
|
||||
tool_calls=accounting["tool_calls"],
|
||||
error_tail=run["stderr"][-2000:],
|
||||
)
|
||||
|
||||
|
||||
# --- main -------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_cases(cases_file: Path) -> list[dict]:
|
||||
data = read_json(cases_file)
|
||||
if isinstance(data, dict) and "cases" in data:
|
||||
cases = data["cases"]
|
||||
elif isinstance(data, list):
|
||||
cases = data
|
||||
else:
|
||||
raise ValueError("cases file must be a list or {'cases': [...]}")
|
||||
if not isinstance(cases, list):
|
||||
raise ValueError("'cases' must be a list")
|
||||
return cases
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--cases", required=True, type=Path)
|
||||
p.add_argument("--skill-path", required=True, type=Path, help="the skill under test (holds SKILL.md)")
|
||||
p.add_argument("--output-dir", required=True, type=Path)
|
||||
p.add_argument("--mode", choices=("quality", "baseline", "variant"), default="quality")
|
||||
p.add_argument("--variant-path", type=Path, default=None, help="variant mode: the stripped or prior skill")
|
||||
p.add_argument(
|
||||
"--project-root",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="holds _bmad/ and anchors fixture paths; found from the skill path when omitted",
|
||||
)
|
||||
p.add_argument("--harness", type=Path, default=None, help="harness JSON for a project without BMad")
|
||||
p.add_argument("--case-ids", default=None, help="comma-separated subset of case ids to run")
|
||||
p.add_argument("--runs", type=int, default=1, help="repeats per case per config for the variance benchmark")
|
||||
p.add_argument("--timeout", type=int, default=600)
|
||||
p.add_argument("--workers", type=int, default=4)
|
||||
p.add_argument("--label", default="evals", help="label for the run id")
|
||||
p.add_argument("--quiet", action="store_true")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
cases_file = args.cases.resolve()
|
||||
if not cases_file.is_file():
|
||||
print(f"cases file not found: {cases_file}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
skill_path = args.skill_path.resolve()
|
||||
if not (skill_path / "SKILL.md").is_file():
|
||||
print(f"skill path has no SKILL.md: {skill_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if args.mode == "variant":
|
||||
if args.variant_path is None:
|
||||
print("--mode variant requires --variant-path", file=sys.stderr)
|
||||
return 2
|
||||
variant_path = args.variant_path.resolve()
|
||||
if not (variant_path / "SKILL.md").is_file():
|
||||
print(f"variant path has no SKILL.md: {variant_path}", file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
variant_path = None
|
||||
|
||||
project_root = args.project_root.resolve() if args.project_root else find_project_root(skill_path)
|
||||
fixture_root = project_root or cases_file.parent
|
||||
|
||||
if args.mode == "baseline":
|
||||
configs: list[tuple[str, Path | None]] = [("skill", skill_path), ("bare", None)]
|
||||
elif args.mode == "variant":
|
||||
configs = [("skill", skill_path), ("variant", variant_path)]
|
||||
else:
|
||||
configs = [("skill", skill_path)]
|
||||
|
||||
cases = load_cases(cases_file)
|
||||
if args.case_ids:
|
||||
wanted = {x.strip() for x in args.case_ids.split(",") if x.strip()}
|
||||
cases = [c for c in cases if str(c.get("id")) in wanted]
|
||||
|
||||
try:
|
||||
harness, harness_note = resolve_harness(project_root, args.harness)
|
||||
except (ValueError, json.JSONDecodeError) as e:
|
||||
print(f"harness invalid: {e}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
run_id, run_dir = make_run_dir(args.output_dir, safe_name(args.label))
|
||||
write_json(
|
||||
run_dir / "run.json",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"cases_file": str(cases_file),
|
||||
"skill_path": str(skill_path),
|
||||
"variant_path": str(variant_path) if variant_path else None,
|
||||
"mode": args.mode,
|
||||
"configs": [name for name, _ in configs],
|
||||
"runs_per_case": args.runs,
|
||||
"harness": harness_note,
|
||||
"command": (harness or {}).get("command"),
|
||||
"started_at": utc_now_iso(),
|
||||
"case_count": len(cases),
|
||||
},
|
||||
)
|
||||
|
||||
if not args.quiet:
|
||||
if harness is None:
|
||||
print(f"[run_evals] no harness ({harness_note}); staging cases only", file=sys.stderr)
|
||||
else:
|
||||
print(f"[run_evals] harness: {' '.join(harness['command'])}", file=sys.stderr)
|
||||
print(
|
||||
f"[run_evals] {len(cases)} cases x {len(configs)} configs x {args.runs} runs, "
|
||||
f"mode={args.mode}, run_dir={run_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
jobs: list[tuple[str, dict, Path, Path | None]] = []
|
||||
for config_name, config_skill in configs:
|
||||
for c in cases:
|
||||
base = run_dir / config_name / safe_name(c.get("id", "unnamed"))
|
||||
for i in range(max(1, args.runs)):
|
||||
case_dir = base / f"run-{i + 1}" if args.runs > 1 else base
|
||||
jobs.append((config_name, c, case_dir, config_skill))
|
||||
|
||||
results: list[dict] = []
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
|
||||
fut_to_case = {
|
||||
pool.submit(
|
||||
run_case,
|
||||
c,
|
||||
case_dir,
|
||||
run_dir,
|
||||
harness,
|
||||
int(c.get("timeout", args.timeout)),
|
||||
config_name,
|
||||
config_skill,
|
||||
resolve_fixtures(c.get("files", []), fixture_root, cases_file.parent),
|
||||
): c
|
||||
for config_name, c, case_dir, config_skill in jobs
|
||||
}
|
||||
for fut in as_completed(fut_to_case):
|
||||
c = fut_to_case[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception as e:
|
||||
res = {"case_id": str(c.get("id")), "status": "exception", "reason": str(e)}
|
||||
results.append(res)
|
||||
if not args.quiet:
|
||||
print(
|
||||
f" [{res.get('status')}] {res.get('config', '?')}/{res.get('case_id')} "
|
||||
f"({res.get('elapsed_s', 0)}s)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
failures = sum(1 for r in results if r.get("status") in FAILURE_STATUSES)
|
||||
summary = {
|
||||
"run_id": run_id,
|
||||
"completed_at": utc_now_iso(),
|
||||
"mode": args.mode,
|
||||
"harness": harness_note,
|
||||
"total": len(jobs),
|
||||
"executed": sum(1 for r in results if r.get("status") == "ok"),
|
||||
"skipped": sum(1 for r in results if r.get("status") == "skipped"),
|
||||
"failures": failures,
|
||||
"run_dir": str(run_dir),
|
||||
"results": results,
|
||||
}
|
||||
write_json(run_dir / "execution-summary.json", summary)
|
||||
print(json.dumps(summary, indent=2))
|
||||
if harness is None:
|
||||
return 3
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user