basic BMAD setup

.agents/skills/bmad-eval/scripts/aggregate_benchmark.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Variance benchmark: summarize a metric across N runs, and compare two configs.
+
+A single skill run is noisy. Running the same case N times and summarizing the
+spread tells you whether a difference between two versions is real or just noise.
+This script computes, per numeric metric, the mean, the sample standard deviation
+(n-1, the unbiased estimator for a sample), the min, and the max across N runs.
+Given two such config summaries it reports the delta on each shared metric so a
+"did the change help" question gets a number instead of a guess.
+
+Input shapes accepted for a single config:
+  - a list of run records, each a flat dict of metric -> number
+      [{"elapsed_s": 12.1, "total_tokens": 800}, {"elapsed_s": 11.4, ...}]
+  - {"runs": [ ...records... ]}
+  - a directory of run folders, each holding timing.json files written by
+    run_evals.py (the script reads every timing.json under the directory and
+    treats each as one run record)
+
+Usage:
+  Summarize one config across its runs:
+    uv run aggregate_benchmark.py --runs CONFIG_A.json
+    uv run aggregate_benchmark.py --runs RUN_DIR/        (reads timing.json files)
+
+  Compare two configs (each summarized, then delta = B - A):
+    uv run aggregate_benchmark.py --baseline A.json --variant B.json
+
+  Self-test on a known fixture (no external input needed):
+    uv run aggregate_benchmark.py --self-test
+
+Output is one JSON object on stdout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+NUMERIC = (int, float)
+
+
+# --- statistics -------------------------------------------------------------
+
+
+def sample_stddev(values: list[float]) -> float:
+    """Sample standard deviation using n-1 (Bessel's correction).
+
+    Returns 0.0 for fewer than two values, where the sample variance is
+    undefined and reporting zero spread is the least surprising choice.
+    """
+    n = len(values)
+    if n < 2:
+        return 0.0
+    mean = sum(values) / n
+    var = sum((x - mean) ** 2 for x in values) / (n - 1)
+    return math.sqrt(var)
+
+
+def summarize_metric(values: list[float]) -> dict:
+    return {
+        "n": len(values),
+        "mean": (sum(values) / len(values)) if values else 0.0,
+        "stddev": sample_stddev(values),
+        "min": min(values) if values else 0.0,
+        "max": max(values) if values else 0.0,
+    }
+
+
+def collect_numeric_metrics(records: list[dict]) -> dict[str, list[float]]:
+    """Group every numeric field across records by metric name."""
+    by_metric: dict[str, list[float]] = {}
+    for rec in records:
+        if not isinstance(rec, dict):
+            continue
+        for key, val in rec.items():
+            if isinstance(val, bool):
+                continue  # bools are ints in Python; not a metric
+            if isinstance(val, NUMERIC):
+                by_metric.setdefault(key, []).append(float(val))
+    return by_metric
+
+
+def summarize_config(records: list[dict]) -> dict:
+    by_metric = collect_numeric_metrics(records)
+    return {
+        "runs": len(records),
+        "metrics": {name: summarize_metric(vals) for name, vals in sorted(by_metric.items())},
+    }
+
+
+def delta_configs(baseline: dict, variant: dict) -> dict:
+    """Per shared metric, delta = variant.mean - baseline.mean, plus context."""
+    b_metrics = baseline.get("metrics", {})
+    v_metrics = variant.get("metrics", {})
+    shared = sorted(set(b_metrics) & set(v_metrics))
+    out: dict[str, dict] = {}
+    for name in shared:
+        b = b_metrics[name]
+        v = v_metrics[name]
+        diff = v["mean"] - b["mean"]
+        pct = (diff / b["mean"] * 100.0) if b["mean"] != 0 else None
+        out[name] = {
+            "baseline_mean": b["mean"],
+            "variant_mean": v["mean"],
+            "delta": diff,
+            "delta_pct": pct,
+            "baseline_stddev": b["stddev"],
+            "variant_stddev": v["stddev"],
+        }
+    return out
+
+
+# --- input loading ----------------------------------------------------------
+
+
+def load_records(path: Path) -> tuple[list[dict], int]:
+    """Run records from a JSON file, a {'runs': [...]} file, or a dir of timing.json files.
+
+    Returns (completed records, number excluded). A record whose `status` is anything but
+    "ok" did not finish, so its elapsed time would skew the mean; it is left out and counted.
+    """
+    if path.is_dir():
+        data: list = []
+        for f in sorted(path.rglob("timing.json")):
+            try:
+                data.append(json.loads(f.read_text(encoding="utf-8")))
+            except (OSError, json.JSONDecodeError):
+                continue
+    else:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        if isinstance(data, dict) and "runs" in data:
+            data = data["runs"]
+        if not isinstance(data, list):
+            raise ValueError(f"expected a list of run records in {path}")
+    records = [r for r in data if isinstance(r, dict)]
+    completed = [r for r in records if r.get("status", "ok") == "ok"]
+    return completed, len(records) - len(completed)
+
+
+# --- self-test --------------------------------------------------------------
+
+
+def run_self_test() -> int:
+    """Verify mean/stddev/min/max/delta on a known fixture."""
+    config_a = [
+        {"elapsed_s": 10.0, "total_tokens": 100},
+        {"elapsed_s": 12.0, "total_tokens": 200},
+        {"elapsed_s": 14.0, "total_tokens": 300},
+    ]
+    summary_a = summarize_config(config_a)
+    el = summary_a["metrics"]["elapsed_s"]
+    # mean of 10,12,14 = 12; n-1 stddev = sqrt(((-2)^2+0+2^2)/2)=sqrt(4)=2
+    assert el["n"] == 3, el
+    assert abs(el["mean"] - 12.0) < 1e-9, el
+    assert abs(el["stddev"] - 2.0) < 1e-9, el
+    assert el["min"] == 10.0 and el["max"] == 14.0, el
+    tok = summary_a["metrics"]["total_tokens"]
+    # mean of 100,200,300 = 200; n-1 stddev = sqrt((10000+0+10000)/2)=100
+    assert abs(tok["mean"] - 200.0) < 1e-9, tok
+    assert abs(tok["stddev"] - 100.0) < 1e-9, tok
+
+    # single value -> stddev 0
+    one = summarize_config([{"x": 5}])
+    assert one["metrics"]["x"]["stddev"] == 0.0, one
+
+    # bools are not treated as metrics
+    with_bool = summarize_config([{"ok": True, "x": 1}, {"ok": False, "x": 3}])
+    assert "ok" not in with_bool["metrics"], with_bool
+    assert abs(with_bool["metrics"]["x"]["mean"] - 2.0) < 1e-9, with_bool
+
+    # delta: variant slower by 3s on mean, faster question answered by sign
+    config_b = [
+        {"elapsed_s": 13.0, "total_tokens": 90},
+        {"elapsed_s": 15.0, "total_tokens": 110},
+        {"elapsed_s": 17.0, "total_tokens": 100},
+    ]
+    summary_b = summarize_config(config_b)
+    d = delta_configs(summary_a, summary_b)
+    # elapsed mean: A=12, B=15 -> delta +3, pct +25%
+    assert abs(d["elapsed_s"]["delta"] - 3.0) < 1e-9, d
+    assert abs(d["elapsed_s"]["delta_pct"] - 25.0) < 1e-9, d
+    # tokens mean: A=200, B=100 -> delta -100, pct -50%
+    assert abs(d["total_tokens"]["delta"] + 100.0) < 1e-9, d
+    assert abs(d["total_tokens"]["delta_pct"] + 50.0) < 1e-9, d
+
+    print(
+        json.dumps(
+            {
+                "self_test": "passed",
+                "checked": [
+                    "mean",
+                    "stddev_n_minus_1",
+                    "min",
+                    "max",
+                    "single_value_stddev",
+                    "bool_excluded",
+                    "delta",
+                    "delta_pct",
+                ],
+            }
+        )
+    )
+    return 0
+
+
+# --- main -------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--runs", type=Path, help="summarize one config (JSON file or dir of timing.json)")
+    p.add_argument("--baseline", type=Path, help="baseline config for a two-config comparison")
+    p.add_argument("--variant", type=Path, help="variant config for a two-config comparison")
+    p.add_argument("--self-test", action="store_true", help="run the built-in fixture self-test and exit")
+    args = p.parse_args(argv)
+
+    if args.self_test:
+        return run_self_test()
+
+    if args.baseline and args.variant:
+        b_records, b_excluded = load_records(args.baseline)
+        v_records, v_excluded = load_records(args.variant)
+        b = summarize_config(b_records)
+        v = summarize_config(v_records)
+        out = {
+            "baseline": b,
+            "variant": v,
+            "delta": delta_configs(b, v),
+            "excluded_incomplete": {"baseline": b_excluded, "variant": v_excluded},
+        }
+        print(json.dumps(out, indent=2))
+        return 0
+
+    if args.runs:
+        records, excluded = load_records(args.runs)
+        out = summarize_config(records)
+        out["excluded_incomplete"] = excluded
+        print(json.dumps(out, indent=2))
+        return 0
+
+    p.error("provide --runs, or both --baseline and --variant, or --self-test")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval/scripts/convert_cases.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Convert eval cases between the skill-creator and the runner formats.
+
+skill-creator's evals.json:
+  {"skill_name": "...", "evals": [{"id": 1, "prompt": "...", "expected_output": "...",
+                                   "files": [...], "expectations": [...]}]}
+
+The runner's cases.json:
+  [{"id": "1", "input": "...", "rubric": [...], "state_prefix": null, "files": [...]}]
+
+Toward the runner, `expected_output` lands at the end of the rubric as
+"output matches: <text>". Toward skill-creator, that rubric line becomes
+`expected_output` again and a numeric id becomes an int.
+
+Usage:
+  uv run convert_cases.py EVALS.json [--to runner|skill-creator] [--output OUT.json]
+         [--skill-name NAME]
+
+Without --output the result goes to stdout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+OUTPUT_PREFIX = "output matches: "
+
+
+def to_runner(data: dict) -> list[dict]:
+    cases = []
+    for ev in data.get("evals", []):
+        rubric = [str(e) for e in ev.get("expectations", [])]
+        expected = ev.get("expected_output")
+        if expected:
+            rubric.append(f"{OUTPUT_PREFIX}{expected}")
+        cases.append(
+            {
+                "id": str(ev["id"]),
+                "input": ev.get("prompt", ""),
+                "rubric": rubric,
+                "state_prefix": None,
+                "files": list(ev.get("files", [])),
+            }
+        )
+    return cases
+
+
+def to_skill_creator(cases: list[dict], skill_name: str) -> dict:
+    evals = []
+    for case in cases:
+        raw_id = str(case["id"])
+        expectations = []
+        expected_output = ""
+        for item in case.get("rubric", []):
+            text = str(item)
+            if text.startswith(OUTPUT_PREFIX) and not expected_output:
+                expected_output = text[len(OUTPUT_PREFIX) :]
+            else:
+                expectations.append(text)
+        evals.append(
+            {
+                "id": int(raw_id) if raw_id.isdigit() else raw_id,
+                "prompt": case.get("input", ""),
+                "expected_output": expected_output,
+                "files": list(case.get("files", [])),
+                "expectations": expectations,
+            }
+        )
+    return {"skill_name": skill_name, "evals": evals}
+
+
+def load_cases(data: object) -> list[dict]:
+    if isinstance(data, dict) and "cases" in data:
+        data = data["cases"]
+    if not isinstance(data, list):
+        raise ValueError("runner cases must be a list or {'cases': [...]}")
+    return data
+
+
+def default_skill_name(path: Path) -> str:
+    # <skill>/evals/cases.json names the skill two levels up.
+    parent = path.resolve().parent
+    return parent.parent.name if parent.name == "evals" else parent.name
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("input", type=Path, help="evals.json (skill-creator) or cases.json (runner)")
+    p.add_argument("--to", choices=("runner", "skill-creator"), default="runner")
+    p.add_argument("--output", type=Path, default=None, help="write here instead of stdout")
+    p.add_argument("--skill-name", default=None, help="skill_name for --to skill-creator")
+    args = p.parse_args(argv)
+
+    data = json.loads(args.input.read_text(encoding="utf-8"))
+    if args.to == "runner":
+        if not isinstance(data, dict) or "evals" not in data:
+            print("error: skill-creator input must be an object with an 'evals' list", file=sys.stderr)
+            return 2
+        result: object = to_runner(data)
+    else:
+        result = to_skill_creator(load_cases(data), args.skill_name or default_skill_name(args.input))
+
+    text = json.dumps(result, indent=2, ensure_ascii=False) + "\n"
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        sys.stdout.write(text)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval/scripts/eval_common.py

@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""What run_evals.py and run_triggers.py share: the harness, the clean room, containment, run folders.
+
+The harness is the agent CLI the evals run through, described by four keys the model that first
+runs an eval records in this skill's customization (`[workflow.harness]`):
+
+  command     argv for one non-interactive run; "{prompt}" is the input, "{cwd}" the workspace.
+  skill_dir   the folder under the workspace the CLI reads skills from.
+  env         env vars for the run: a value is set as given, with "~" meaning the fresh HOME;
+              an empty value forwards the host's value when it has one.
+  home_files  files or folders under the host home brought into the fresh HOME at the same
+              path, so a CLI logged in through a file or keychain stays logged in.
+
+The runners read them through the project's customization resolver, or from `--harness <json>`
+for a project without BMad. A run happens in a clean room: a temporary folder outside the
+project holding the workspace and the fresh HOME, so the project's instruction files and
+installed skills are not ancestors of the run. The workspace is copied back afterwards.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from collections.abc import Callable, Mapping
+from datetime import UTC, datetime
+from pathlib import Path
+
+SKILL_ROOT = Path(__file__).resolve().parents[1]
+HARNESS_KEY = "workflow.harness"
+DEFAULT_SKILL_DIR = ".agents/skills"
+SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
+# A subprocess on Windows cannot start without these; nothing else crosses uninvited.
+PLATFORM_ENV = ("SYSTEMROOT", "COMSPEC", "PATHEXT", "TEMP", "TMP") if sys.platform == "win32" else ()
+
+
+def utc_now_iso() -> str:
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def write_json(path: Path, data: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def read_json(path: Path) -> object:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def find_project_root(start: Path) -> Path | None:
+    """Nearest ancestor holding `_bmad/`, else the nearest holding `.git`."""
+    git_root = None
+    current = start.resolve()
+    while True:
+        if (current / "_bmad").is_dir():
+            return current
+        if git_root is None and (current / ".git").exists():
+            git_root = current
+        if current.parent == current:
+            return git_root
+        current = current.parent
+
+
+# --- harness -----------------------------------------------------------------
+
+
+def validate_harness(harness: object) -> dict:
+    if not isinstance(harness, dict):
+        raise ValueError("harness must be a table")
+    command = harness.get("command")
+    if not isinstance(command, list) or not command or not all(isinstance(t, str) for t in command):
+        raise ValueError("harness.command must be a non-empty list of strings")
+    if not any("{prompt}" in t for t in command):
+        raise ValueError("harness.command needs a {prompt} token")
+    if not isinstance(harness.get("skill_dir", DEFAULT_SKILL_DIR), str):
+        raise ValueError("harness.skill_dir must be a string")
+    env = harness.get("env", {})
+    if not isinstance(env, dict) or not all(isinstance(v, str) for v in env.values()):
+        raise ValueError('harness.env is a table of var = "value" ("" forwards the host value, "~" is the fresh HOME)')
+    if not isinstance(harness.get("home_files", []), list):
+        raise ValueError("harness.home_files must be a list")
+    return harness
+
+
+def resolve_harness(project_root: Path | None, explicit: Path | None) -> tuple[dict | None, str]:
+    """The harness table and where it came from; (None, why) when there is none.
+
+    The runner never reads the override files itself: the project's resolver merges the layers.
+    """
+    if explicit is not None:
+        if not explicit.is_file():
+            return None, f"harness file not found: {explicit}"
+        return validate_harness(read_json(explicit)), str(explicit)
+    if project_root is None:
+        return None, "no project root"
+    resolver = project_root / "_bmad" / "scripts" / "resolve_customization.py"
+    if not resolver.is_file():
+        return None, "BMad is not set up in this project; pass --harness"
+    argv = [sys.executable, str(resolver), "--skill", str(SKILL_ROOT), "--project-root", str(project_root)]
+    proc = subprocess.run([*argv, "--key", HARNESS_KEY], capture_output=True, text=True)
+    if proc.returncode != 0:
+        return None, f"customization resolver failed: {proc.stderr.strip()[-300:]}"
+    harness = json.loads(proc.stdout or "{}").get(HARNESS_KEY)
+    if not isinstance(harness, dict) or not harness.get("command"):
+        return None, "no harness recorded in bmad-eval's customization"
+    return validate_harness(harness), f"customization {HARNESS_KEY}"
+
+
+def build_argv(harness: Mapping, prompt: str, cwd: str) -> list[str]:
+    """The command with its placeholders filled."""
+    return [
+        str(t).replace("{prompt}", prompt).replace("{query}", prompt).replace("{cwd}", cwd) for t in harness["command"]
+    ]
+
+
+def build_case_env(harness: Mapping | None, home_dir: Path, host_env: Mapping[str, str]) -> dict[str, str]:
+    """The subprocess environment, built from scratch, never from os.environ.
+
+    PATH, the fresh HOME (and USERPROFILE on Windows), then the harness's `env`: a value is set
+    as given with "~" expanded to the fresh HOME; an empty value forwards the host's value when
+    the host has one set non-empty, and is left out otherwise, since an empty credential would
+    override the CLI's own login.
+    """
+    env = {"PATH": host_env.get("PATH", ""), "HOME": str(home_dir)}
+    if sys.platform == "win32":
+        env["USERPROFILE"] = str(home_dir)
+    for name in PLATFORM_ENV:
+        if name in host_env:
+            env[name] = host_env[name]
+    for name, value in ((harness or {}).get("env") or {}).items():
+        if value == "":
+            if host_env.get(str(name)):
+                env[str(name)] = host_env[str(name)]
+        else:
+            env[str(name)] = expand_home(str(value), home_dir)
+    return env
+
+
+def expand_home(value: str, home_dir: Path) -> str:
+    if value == "~":
+        return str(home_dir)
+    if value.startswith("~/"):
+        return str(home_dir / value[2:])
+    return value
+
+
+def make_home(harness: Mapping | None, room: Path) -> Path:
+    """The fresh HOME for one run: folders named by `env` values made, `home_files` brought over.
+
+    `home_files` are paths relative to the host home (`.codex/auth.json`, `Library/Keychains`),
+    placed at the same path inside the fresh HOME: a file is copied, a folder is linked.
+    """
+    harness = harness or {}
+    home = room / ".home"
+    home.mkdir(parents=True, exist_ok=True)
+    for value in (harness.get("env") or {}).values():
+        if str(value).startswith("~/"):
+            contained(home, str(value)[2:]).mkdir(parents=True, exist_ok=True)
+    host_home = Path.home()
+    for entry in harness.get("home_files") or []:
+        rel = str(entry).removeprefix("~/").removeprefix("~")
+        src = host_home / rel
+        dest = contained(home, rel)
+        if dest.exists() or dest.is_symlink():
+            continue
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        if src.is_dir():
+            dest.symlink_to(src, target_is_directory=True)
+        elif src.is_file():
+            shutil.copy2(src, dest)
+        else:
+            print(f"Warning: home_files entry not found on this host: {src}", file=sys.stderr)
+    return home
+
+
+# --- the clean room ----------------------------------------------------------
+
+
+def run_in_clean_room(
+    harness: Mapping, case_dir: Path, prompt: str, timeout: int, stage: Callable[[Path], None]
+) -> dict:
+    """One harness run in a temporary room outside the project; the workspace comes back to case_dir/cwd.
+
+    `stage(cwd)` fills the workspace before the run. The room holds the workspace, made a git
+    root so harnesses that look for a project boundary stop there, and the fresh HOME beside it.
+    Leaves prompt.txt, transcript.jsonl (stdout), stderr.txt and cwd/ in case_dir. Returns
+    status ("ok", "error", "timeout", "harness-missing"), return_code, elapsed_s, stdout, stderr.
+    """
+    case_dir.mkdir(parents=True, exist_ok=True)
+    (case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
+    with tempfile.TemporaryDirectory(prefix="bmad-eval-", ignore_cleanup_errors=True) as tmp:
+        room = Path(tmp)
+        cwd = room / "cwd"
+        cwd.mkdir()
+        stage(cwd)
+        subprocess.run(["git", "init", "-q", str(cwd)], capture_output=True)
+        env = build_case_env(harness, make_home(harness, room), os.environ)
+        argv = build_argv(harness, prompt, str(cwd))
+        status, return_code, stdout, stderr = "ok", 0, b"", b""
+        start = time.time()
+        try:
+            proc = subprocess.run(
+                argv, capture_output=True, stdin=subprocess.DEVNULL, cwd=str(cwd), env=env, timeout=timeout
+            )
+            stdout, stderr, return_code = proc.stdout or b"", proc.stderr or b"", proc.returncode
+            if return_code != 0:
+                status = "error"
+        except FileNotFoundError as e:
+            status, return_code, stderr = "harness-missing", -1, f"command not found: {e}".encode()
+        except subprocess.TimeoutExpired as e:
+            status, return_code = "timeout", -1
+            stdout, stderr = e.stdout or b"", (e.stderr or b"") + f"\nTIMEOUT after {timeout}s".encode()
+        elapsed = time.time() - start
+        shutil.copytree(cwd, case_dir / "cwd", symlinks=True, ignore=shutil.ignore_patterns(".git"), dirs_exist_ok=True)
+    stdout_text = stdout.decode("utf-8", errors="replace")
+    stderr_text = stderr.decode("utf-8", errors="replace")
+    (case_dir / "transcript.jsonl").write_text(stdout_text, encoding="utf-8")
+    (case_dir / "stderr.txt").write_text(stderr_text, encoding="utf-8")
+    return {
+        "status": status,
+        "return_code": return_code,
+        "elapsed_s": round(elapsed, 3),
+        "stdout": stdout_text,
+        "stderr": stderr_text,
+    }
+
+
+def account_transcript(transcript_text: str) -> dict:
+    """Best-effort usage from what the harness printed.
+
+    When the output is line-delimited JSON, per-message usage on assistant events is summed,
+    a turn-level usage block on any event overrides the sum, and tool calls are counted;
+    anything else degrades to zero counts with `tokens_reported` false, and elapsed time is
+    the metric.
+    """
+    input_tokens = 0
+    output_tokens = 0
+    total_steps = 0
+    tool_calls: dict[str, int] = {}
+    found_usage = False
+    for raw in transcript_text.splitlines():
+        raw = raw.strip()
+        if not raw:
+            continue
+        try:
+            evt = json.loads(raw)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(evt, dict):
+            continue
+        if evt.get("type") == "assistant":
+            total_steps += 1
+            msg = evt.get("message", {})
+            usage = msg.get("usage") if isinstance(msg, dict) else None
+            if isinstance(usage, dict):
+                found_usage = True
+                input_tokens += int(usage.get("input_tokens", 0) or 0)
+                output_tokens += int(usage.get("output_tokens", 0) or 0)
+            for item in msg.get("content", []) if isinstance(msg, dict) else []:
+                if isinstance(item, dict) and item.get("type") == "tool_use":
+                    name = item.get("name", "?")
+                    tool_calls[name] = tool_calls.get(name, 0) + 1
+        elif isinstance(evt.get("usage"), dict):
+            # A turn-level usage block is authoritative over the running sum.
+            usage = evt["usage"]
+            found_usage = True
+            input_tokens = int(usage.get("input_tokens", input_tokens) or input_tokens)
+            output_tokens = int(usage.get("output_tokens", output_tokens) or output_tokens)
+    return {
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "total_tokens": input_tokens + output_tokens,
+        "tokens_reported": found_usage,
+        "total_steps": total_steps,
+        "tool_calls": tool_calls,
+        "total_tool_calls": sum(tool_calls.values()),
+    }
+
+
+# --- containment and run folders ----------------------------------------------
+
+
+def safe_name(value: object) -> str:
+    """A folder name from a case id: no separators, no `..`, never empty."""
+    name = SAFE_NAME_RE.sub("_", str(value)).strip(".")
+    return name or "unnamed"
+
+
+def contained(root: Path, rel: str) -> Path:
+    """`root / rel` when it stays inside root; ValueError when it would escape."""
+    root = root.resolve()
+    target = (root / rel).resolve()
+    if target != root and root not in target.parents:
+        raise ValueError(f"path escapes the workspace: {rel}")
+    return target
+
+
+def make_run_dir(output_dir: Path, label: str) -> tuple[str, Path]:
+    """A new run folder. A second run in the same second gets a suffix, never the same folder."""
+    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    for n in range(1, 1000):
+        run_id = f"{stamp}-{label}" if n == 1 else f"{stamp}-{label}-{n}"
+        run_dir = (output_dir / run_id).resolve()
+        try:
+            run_dir.mkdir(parents=True, exist_ok=False)
+        except FileExistsError:
+            continue
+        return run_id, run_dir
+    raise RuntimeError(f"could not create a run folder under {output_dir}")

.agents/skills/bmad-eval/scripts/run_evals.py

@@ -0,0 +1,363 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Run eval cases through the recorded harness.
+
+A case is `input + rubric + optional state_prefix + optional files`. For each case and
+config the runner makes a clean room outside the project, copies the skill under test into
+the folder the harness reads skills from, stages the fixtures, runs the harness command once
+from the workspace, and records what it printed with timing and token usage. Grading happens
+elsewhere, from the transcript and artifacts left behind.
+
+Modes (--mode) decide which configs each case runs under:
+
+  quality  : one config, "skill", the skill staged in the workspace.
+  baseline : "skill" and "bare" (nothing staged), same input, so the bare-model floor is
+             measured under identical conditions.
+  variant  : "skill" (--skill-path) and "variant" (--variant-path).
+
+Run layout: <run-dir>/<config>/<case-id>/ (plus /run-N/ when --runs > 1), each holding
+prompt.txt, transcript.jsonl, stderr.txt, timing.json and cwd/ (the workspace after the run),
+so `aggregate_benchmark.py --baseline <run-dir>/bare --variant <run-dir>/skill` compares
+configs from the timing.json files.
+
+Harness: `[workflow.harness]` in this skill's customization, read through the project's
+resolver; `--harness <json>` with the same keys for a project without BMad. The keys and the
+clean room are described in eval_common.py. Without a harness the runner stages every case,
+records each as skipped and exits 3.
+
+Usage:
+  uv run run_evals.py --cases CASES.json --skill-path SKILL_DIR --output-dir DIR
+    [--mode quality|baseline|variant] [--variant-path SKILL_DIR] [--project-root DIR]
+    [--harness HARNESS.json] [--case-ids A1,B3] [--runs N] [--timeout SECS]
+    [--workers N] [--label NAME] [--quiet]
+
+CASES.json is a list of cases or {"cases": [...]}. Each case:
+  {"id": "...", "input": "...", "rubric": [...], "state_prefix": "..."?, "files": ["..."]?}
+
+Exit 0 when every run completed, 1 when any run errored or timed out or the command was
+not found, 2 on a usage error, 3 when no harness is recorded.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from eval_common import (
+    DEFAULT_SKILL_DIR,
+    account_transcript,
+    contained,
+    find_project_root,
+    make_run_dir,
+    read_json,
+    resolve_harness,
+    run_in_clean_room,
+    safe_name,
+    utc_now_iso,
+    write_json,
+)
+
+FAILURE_STATUSES = ("error", "timeout", "exception", "harness-missing")
+
+# --- staging: skill under test + fixtures ------------------------------------
+
+
+def stage_skill(skill_path: Path, cwd: Path, skills_subdir: str) -> Path:
+    """Copy the skill where the harness discovers skills inside the workspace.
+
+    A copy, not a symlink: a run that edits its staged skill must not touch the source or
+    the other cases.
+    """
+    dest = cwd / skills_subdir / skill_path.name
+    shutil.copytree(skill_path, dest, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", ".git"))
+    return dest
+
+
+def resolve_fixtures(files: list, project_root: Path, cases_dir: Path) -> list[tuple[Path, str]]:
+    """Map each `files` entry to (source, dest-relative-path).
+
+    The entry's own relative path is preserved inside the workspace, so a bare filename lands
+    at its root and a nested path keeps its directory structure, matching the path the case
+    input references.
+    """
+    out: list[tuple[Path, str]] = []
+    for entry in files or []:
+        entry = str(entry)
+        for candidate in (
+            (project_root / entry).resolve(),
+            (cases_dir / entry).resolve(),
+            Path(entry).resolve(),
+        ):
+            if candidate.is_file():
+                out.append((candidate, entry))
+                break
+        else:
+            print(f"Warning: fixture not found: {entry}", file=sys.stderr)
+    return out
+
+
+def stage_fixtures(fixtures: list[tuple[Path, str]], cwd: Path) -> None:
+    for src, dest_rel in fixtures:
+        dest = contained(cwd, dest_rel)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+
+
+def compose_prompt(case: dict) -> str:
+    """Prepend the state_prefix, a bracketed prime that places the skill mid-workflow."""
+    input_text = str(case.get("input", ""))
+    prefix = case.get("state_prefix")
+    if prefix:
+        return f"{str(prefix).rstrip()}\n\n{input_text}"
+    return input_text
+
+
+# --- per-case execution -----------------------------------------------------
+
+
+def run_case(
+    case: dict,
+    case_dir: Path,
+    run_dir: Path,
+    harness: dict | None,
+    timeout: int,
+    config: str,
+    skill_path: Path | None,
+    fixtures: list[tuple[Path, str]],
+) -> dict:
+    case_id = str(case.get("id", "unnamed"))
+    case_dir.mkdir(parents=True, exist_ok=True)
+    prompt = compose_prompt(case)
+    write_json(case_dir / "case.json", case)
+
+    def timing(status: str, **extra: object) -> None:
+        write_json(case_dir / "timing.json", {"case_id": case_id, "config": config, "status": status, **extra})
+
+    def result(status: str, **extra: object) -> dict:
+        return {
+            "case_id": case_id,
+            "config": config,
+            "status": status,
+            "case_dir": str(case_dir.relative_to(run_dir)),
+            **extra,
+        }
+
+    def stage(cwd: Path) -> None:
+        stage_fixtures(fixtures, cwd)
+        if skill_path is not None:
+            stage_skill(skill_path, cwd, (harness or {}).get("skill_dir", DEFAULT_SKILL_DIR))
+
+    if harness is None:
+        (case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
+        try:
+            stage(case_dir / "cwd")
+        except ValueError as e:
+            timing("error", reason=str(e), captured_at=utc_now_iso())
+            return result("error", reason=str(e))
+        timing("skipped", captured_at=utc_now_iso())
+        return result("skipped", reason="no harness recorded", prompt_chars=len(prompt))
+
+    try:
+        run = run_in_clean_room(harness, case_dir, prompt, timeout, stage)
+    except ValueError as e:
+        timing("error", reason=str(e), captured_at=utc_now_iso())
+        return result("error", reason=str(e))
+
+    accounting = account_transcript(run["stdout"])
+    timing(
+        run["status"],
+        elapsed_s=run["elapsed_s"],
+        return_code=run["return_code"],
+        input_tokens=accounting["input_tokens"],
+        output_tokens=accounting["output_tokens"],
+        total_tokens=accounting["total_tokens"],
+        tokens_reported=accounting["tokens_reported"],
+        total_steps=accounting["total_steps"],
+        total_tool_calls=accounting["total_tool_calls"],
+        captured_at=utc_now_iso(),
+    )
+    return result(
+        run["status"],
+        elapsed_s=run["elapsed_s"],
+        return_code=run["return_code"],
+        tokens=accounting["total_tokens"],
+        tool_calls=accounting["tool_calls"],
+        error_tail=run["stderr"][-2000:],
+    )
+
+
+# --- main -------------------------------------------------------------------
+
+
+def load_cases(cases_file: Path) -> list[dict]:
+    data = read_json(cases_file)
+    if isinstance(data, dict) and "cases" in data:
+        cases = data["cases"]
+    elif isinstance(data, list):
+        cases = data
+    else:
+        raise ValueError("cases file must be a list or {'cases': [...]}")
+    if not isinstance(cases, list):
+        raise ValueError("'cases' must be a list")
+    return cases
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--cases", required=True, type=Path)
+    p.add_argument("--skill-path", required=True, type=Path, help="the skill under test (holds SKILL.md)")
+    p.add_argument("--output-dir", required=True, type=Path)
+    p.add_argument("--mode", choices=("quality", "baseline", "variant"), default="quality")
+    p.add_argument("--variant-path", type=Path, default=None, help="variant mode: the stripped or prior skill")
+    p.add_argument(
+        "--project-root",
+        type=Path,
+        default=None,
+        help="holds _bmad/ and anchors fixture paths; found from the skill path when omitted",
+    )
+    p.add_argument("--harness", type=Path, default=None, help="harness JSON for a project without BMad")
+    p.add_argument("--case-ids", default=None, help="comma-separated subset of case ids to run")
+    p.add_argument("--runs", type=int, default=1, help="repeats per case per config for the variance benchmark")
+    p.add_argument("--timeout", type=int, default=600)
+    p.add_argument("--workers", type=int, default=4)
+    p.add_argument("--label", default="evals", help="label for the run id")
+    p.add_argument("--quiet", action="store_true")
+    args = p.parse_args(argv)
+
+    cases_file = args.cases.resolve()
+    if not cases_file.is_file():
+        print(f"cases file not found: {cases_file}", file=sys.stderr)
+        return 2
+
+    skill_path = args.skill_path.resolve()
+    if not (skill_path / "SKILL.md").is_file():
+        print(f"skill path has no SKILL.md: {skill_path}", file=sys.stderr)
+        return 2
+
+    if args.mode == "variant":
+        if args.variant_path is None:
+            print("--mode variant requires --variant-path", file=sys.stderr)
+            return 2
+        variant_path = args.variant_path.resolve()
+        if not (variant_path / "SKILL.md").is_file():
+            print(f"variant path has no SKILL.md: {variant_path}", file=sys.stderr)
+            return 2
+    else:
+        variant_path = None
+
+    project_root = args.project_root.resolve() if args.project_root else find_project_root(skill_path)
+    fixture_root = project_root or cases_file.parent
+
+    if args.mode == "baseline":
+        configs: list[tuple[str, Path | None]] = [("skill", skill_path), ("bare", None)]
+    elif args.mode == "variant":
+        configs = [("skill", skill_path), ("variant", variant_path)]
+    else:
+        configs = [("skill", skill_path)]
+
+    cases = load_cases(cases_file)
+    if args.case_ids:
+        wanted = {x.strip() for x in args.case_ids.split(",") if x.strip()}
+        cases = [c for c in cases if str(c.get("id")) in wanted]
+
+    try:
+        harness, harness_note = resolve_harness(project_root, args.harness)
+    except (ValueError, json.JSONDecodeError) as e:
+        print(f"harness invalid: {e}", file=sys.stderr)
+        return 2
+
+    run_id, run_dir = make_run_dir(args.output_dir, safe_name(args.label))
+    write_json(
+        run_dir / "run.json",
+        {
+            "run_id": run_id,
+            "cases_file": str(cases_file),
+            "skill_path": str(skill_path),
+            "variant_path": str(variant_path) if variant_path else None,
+            "mode": args.mode,
+            "configs": [name for name, _ in configs],
+            "runs_per_case": args.runs,
+            "harness": harness_note,
+            "command": (harness or {}).get("command"),
+            "started_at": utc_now_iso(),
+            "case_count": len(cases),
+        },
+    )
+
+    if not args.quiet:
+        if harness is None:
+            print(f"[run_evals] no harness ({harness_note}); staging cases only", file=sys.stderr)
+        else:
+            print(f"[run_evals] harness: {' '.join(harness['command'])}", file=sys.stderr)
+        print(
+            f"[run_evals] {len(cases)} cases x {len(configs)} configs x {args.runs} runs, "
+            f"mode={args.mode}, run_dir={run_dir}",
+            file=sys.stderr,
+        )
+
+    jobs: list[tuple[str, dict, Path, Path | None]] = []
+    for config_name, config_skill in configs:
+        for c in cases:
+            base = run_dir / config_name / safe_name(c.get("id", "unnamed"))
+            for i in range(max(1, args.runs)):
+                case_dir = base / f"run-{i + 1}" if args.runs > 1 else base
+                jobs.append((config_name, c, case_dir, config_skill))
+
+    results: list[dict] = []
+    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
+        fut_to_case = {
+            pool.submit(
+                run_case,
+                c,
+                case_dir,
+                run_dir,
+                harness,
+                int(c.get("timeout", args.timeout)),
+                config_name,
+                config_skill,
+                resolve_fixtures(c.get("files", []), fixture_root, cases_file.parent),
+            ): c
+            for config_name, c, case_dir, config_skill in jobs
+        }
+        for fut in as_completed(fut_to_case):
+            c = fut_to_case[fut]
+            try:
+                res = fut.result()
+            except Exception as e:
+                res = {"case_id": str(c.get("id")), "status": "exception", "reason": str(e)}
+            results.append(res)
+            if not args.quiet:
+                print(
+                    f"  [{res.get('status')}] {res.get('config', '?')}/{res.get('case_id')} "
+                    f"({res.get('elapsed_s', 0)}s)",
+                    file=sys.stderr,
+                )
+
+    failures = sum(1 for r in results if r.get("status") in FAILURE_STATUSES)
+    summary = {
+        "run_id": run_id,
+        "completed_at": utc_now_iso(),
+        "mode": args.mode,
+        "harness": harness_note,
+        "total": len(jobs),
+        "executed": sum(1 for r in results if r.get("status") == "ok"),
+        "skipped": sum(1 for r in results if r.get("status") == "skipped"),
+        "failures": failures,
+        "run_dir": str(run_dir),
+        "results": results,
+    }
+    write_json(run_dir / "execution-summary.json", summary)
+    print(json.dumps(summary, indent=2))
+    if harness is None:
+        return 3
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval/scripts/run_triggers.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Trigger evals: does a skill's description fire on each near-miss query?
+
+A trigger query is a should/should-not user message that shares keywords with the skill, so
+the description has to discriminate. For each query the runner stages a synthetic skill with
+the description under test where the harness reads skills, sends the query through the
+harness in a clean room, and checks whether the skill loaded. Each query runs several times
+(--runs-per-query) so the rate is stable, not a coin flip.
+
+Detection is a canary, the same on every harness. The synthetic skill's body asks the model
+to begin its reply with a token unique to that attempt; the token appearing in what the
+harness printed is the load. The skill's name and description never contain the token, so a
+harness that lists its discovered skills at startup cannot fake a hit.
+
+A query whose attempts did not all complete (command failed, timed out) is unmeasured, never
+passed: a should-not query with no completed attempts would otherwise pass at a rate of zero.
+
+Each attempt keeps prompt.txt, transcript.jsonl (what the harness printed), stderr.txt,
+timing.json and cwd/ (the workspace after the run, staged skill included) under
+<run-dir>/queries/qNNN-rN/. Harness and clean room are as in run_evals.py.
+
+Usage:
+  uv run run_triggers.py --skill-path SKILL_DIR --queries QUERIES.json --output-dir DIR
+    [--project-root DIR] [--harness HARNESS.json] [--runs-per-query N] [--threshold 0.5]
+    [--timeout SECS] [--workers N] [--quiet]
+
+QUERIES.json is a list of {"query": "...", "should_trigger": true|false}.
+
+Exit 0 when every query was measured, 1 when any attempt failed or the command was not
+found, 2 on a usage error, 3 when no harness is recorded. A query failing its threshold is a
+result, not an error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from eval_common import (
+    DEFAULT_SKILL_DIR,
+    account_transcript,
+    find_project_root,
+    make_run_dir,
+    read_json,
+    resolve_harness,
+    run_in_clean_room,
+    utc_now_iso,
+    write_json,
+)
+
+CANARY_PREFIX = "TRIGGER-LOADED-"
+
+
+def unquote_yaml(value: str) -> str:
+    """A YAML scalar's text: single quotes doubled inside, double quotes with backslash escapes."""
+    value = value.strip()
+    if len(value) >= 2 and value[0] == value[-1] == "'":
+        return value[1:-1].replace("''", "'")
+    if len(value) >= 2 and value[0] == value[-1] == '"':
+        return value[1:-1].replace('\\"', '"').replace("\\n", "\n").replace("\\\\", "\\")
+    return value
+
+
+def parse_skill_md(skill_path: Path) -> tuple[str, str]:
+    """Return (name, description) from SKILL.md frontmatter, quotes and block indicators removed."""
+    text = (skill_path / "SKILL.md").read_text(encoding="utf-8")
+    m = re.match(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL)
+    if not m:
+        raise ValueError(f"SKILL.md at {skill_path} is missing frontmatter")
+    fields: dict[str, list[str]] = {}
+    current = None
+    for line in m.group(1).splitlines():
+        if line[:1] not in (" ", "\t") and ":" in line:
+            key, _, value = line.partition(":")
+            current = key.strip()
+            fields[current] = [] if value.strip() in ("|", ">", "|-", ">-") else [value.strip()]
+        elif current and line.strip():
+            fields[current].append(line.strip())
+    name = unquote_yaml(" ".join(fields.get("name", [])))
+    if not name:
+        raise ValueError(f"SKILL.md at {skill_path} has no name")
+    return name, unquote_yaml(" ".join(fields.get("description", [])))
+
+
+# --- synthetic skill and detection -------------------------------------------
+
+
+def write_synthetic_skill(skills_dir: Path, skill_name: str, description: str, token: str) -> str:
+    """Write a skill the harness can discover, carrying the description and the canary. Returns its name."""
+    clean_name = f"{skill_name}-trig-{token[len(CANARY_PREFIX) :]}"
+    root = skills_dir / clean_name
+    root.mkdir(parents=True, exist_ok=True)
+    indented = "\n  ".join(description.split("\n"))
+    (root / "SKILL.md").write_text(
+        f"---\nname: {clean_name}\ndescription: |\n  {indented}\n---\n\n"
+        f"# {skill_name}\n\nThis skill handles: {description}\n\n"
+        f"Begin your reply with the exact token `{token}`, then continue.\n",
+        encoding="utf-8",
+    )
+    return clean_name
+
+
+def detect_load(output: str, token: str) -> bool:
+    """Did the synthetic skill load? Its canary token in the output says so."""
+    return token in output
+
+
+# --- per-attempt execution ----------------------------------------------------
+
+
+def run_query_once(
+    query: str, skill_name: str, description: str, harness: dict, attempt_dir: Path, timeout: int
+) -> tuple[bool | None, str]:
+    """One attempt: (loaded, "") when it completed, (None, why) when it did not."""
+    token = CANARY_PREFIX + uuid.uuid4().hex[:8]
+
+    def stage(cwd: Path) -> None:
+        write_synthetic_skill(cwd / harness.get("skill_dir", DEFAULT_SKILL_DIR), skill_name, description, token)
+
+    run = run_in_clean_room(harness, attempt_dir, query, timeout, stage)
+    accounting = account_transcript(run["stdout"])
+    loaded = detect_load(run["stdout"], token) if run["status"] == "ok" else None
+    write_json(
+        attempt_dir / "timing.json",
+        {
+            "status": run["status"],
+            "elapsed_s": run["elapsed_s"],
+            "return_code": run["return_code"],
+            "loaded": loaded,
+            "total_tokens": accounting["total_tokens"],
+            "tokens_reported": accounting["tokens_reported"],
+            "captured_at": utc_now_iso(),
+        },
+    )
+    if run["status"] != "ok":
+        return None, f"{run['status']}: {run['stderr'][-500:].strip()}"
+    return loaded, ""
+
+
+# --- main -------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--skill-path", required=True, type=Path)
+    p.add_argument("--queries", required=True, type=Path)
+    p.add_argument("--output-dir", required=True, type=Path)
+    p.add_argument(
+        "--project-root", type=Path, default=None, help="holds _bmad/; found from the skill path when omitted"
+    )
+    p.add_argument("--harness", type=Path, default=None, help="harness JSON for a project without BMad")
+    p.add_argument("--runs-per-query", type=int, default=3)
+    p.add_argument("--threshold", type=float, default=0.5)
+    p.add_argument("--timeout", type=int, default=180)
+    p.add_argument("--workers", type=int, default=4)
+    p.add_argument("--quiet", action="store_true")
+    args = p.parse_args(argv)
+
+    skill_path = args.skill_path.resolve()
+    queries_file = args.queries.resolve()
+    if not queries_file.is_file():
+        print(f"queries file not found: {queries_file}", file=sys.stderr)
+        return 2
+
+    skill_name, description = parse_skill_md(skill_path)
+    queries = read_json(queries_file)
+    if not isinstance(queries, list):
+        print("queries file must be a JSON list", file=sys.stderr)
+        return 2
+
+    project_root = args.project_root.resolve() if args.project_root else find_project_root(skill_path)
+    try:
+        harness, harness_note = resolve_harness(project_root, args.harness)
+    except (ValueError, json.JSONDecodeError) as e:
+        print(f"harness invalid: {e}", file=sys.stderr)
+        return 2
+
+    run_id, run_dir = make_run_dir(args.output_dir, f"{skill_name}-triggers")
+    (run_dir / "queries").mkdir(parents=True, exist_ok=True)
+    write_json(
+        run_dir / "run.json",
+        {
+            "run_id": run_id,
+            "skill_name": skill_name,
+            "description": description,
+            "harness": harness_note,
+            "command": (harness or {}).get("command"),
+            "started_at": utc_now_iso(),
+            "query_count": len(queries),
+            "runs_per_query": args.runs_per_query,
+            "threshold": args.threshold,
+        },
+    )
+
+    if harness is None:
+        if not args.quiet:
+            print(f"[run_triggers] no harness ({harness_note}); staging only", file=sys.stderr)
+        output = {
+            "run_id": run_id,
+            "completed_at": utc_now_iso(),
+            "skill_name": skill_name,
+            "description": description,
+            "status": "skipped",
+            "reason": "no harness recorded",
+            "results": [],
+            "summary": {"total": len(queries), "passed": 0, "failed": 0, "unmeasured": len(queries)},
+        }
+        write_json(run_dir / "triggers-result.json", output)
+        print(json.dumps(output, indent=2))
+        return 3
+
+    def run_one(idx: int, q: dict, run_idx: int) -> tuple[int, bool | None, str]:
+        attempt_dir = run_dir / "queries" / f"q{idx:03d}-r{run_idx}"
+        try:
+            loaded, error = run_query_once(q["query"], skill_name, description, harness, attempt_dir, args.timeout)
+        except (OSError, ValueError) as e:
+            loaded, error = None, str(e)
+        return idx, loaded, error
+
+    attempts: dict[int, list[bool]] = {idx: [] for idx in range(len(queries))}
+    errors: dict[int, list[str]] = {idx: [] for idx in range(len(queries))}
+    if not args.quiet:
+        print(f"[run_triggers] harness: {' '.join(harness['command'])}", file=sys.stderr)
+        print(f"[run_triggers] {len(queries)} queries x {args.runs_per_query} runs", file=sys.stderr)
+
+    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
+        futures = [
+            pool.submit(run_one, idx, q, run_idx)
+            for idx, q in enumerate(queries)
+            for run_idx in range(args.runs_per_query)
+        ]
+        for fut in as_completed(futures):
+            idx, loaded, error = fut.result()
+            if loaded is None:
+                errors[idx].append(error)
+                if not args.quiet:
+                    print(f"  attempt failed for query {idx}: {error}", file=sys.stderr)
+            else:
+                attempts[idx].append(loaded)
+
+    results = []
+    for idx, q in enumerate(queries):
+        runs = attempts[idx]
+        measured = len(runs) == args.runs_per_query
+        rate = (sum(runs) / len(runs)) if runs else 0.0
+        should = bool(q.get("should_trigger", True))
+        passed = ((rate >= args.threshold) if should else (rate < args.threshold)) if measured else None
+        results.append(
+            {
+                "query": q["query"],
+                "should_trigger": should,
+                "trigger_rate": round(rate, 3),
+                "triggers": int(sum(runs)),
+                "runs": len(runs),
+                "errors": errors[idx],
+                "pass": passed,
+            }
+        )
+
+    unmeasured = sum(1 for r in results if r["pass"] is None)
+    output = {
+        "run_id": run_id,
+        "completed_at": utc_now_iso(),
+        "skill_name": skill_name,
+        "description": description,
+        "harness": harness_note,
+        "results": results,
+        "summary": {
+            "total": len(results),
+            "passed": sum(1 for r in results if r["pass"] is True),
+            "failed": sum(1 for r in results if r["pass"] is False),
+            "unmeasured": unmeasured,
+        },
+    }
+    write_json(run_dir / "triggers-result.json", output)
+    print(json.dumps(output, indent=2))
+    return 1 if unmeasured else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval/scripts/tests/fake_harness.py

@@ -0,0 +1,52 @@
+"""A stand-in agent CLI for the runner tests: `fake_harness.py <prompt> <cwd>`.
+
+Behaves like a harness enough to exercise the runners: reads the skills staged under
+`.agents/skills` in the cwd, obeys a canary line when the prompt says "yes", writes an
+artifact, prints a usage line, and exits non-zero when the prompt says "crash". It also
+checks the clean room: HOME must sit inside the stage and the host secret must be absent.
+"""
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+CANARY_RE = re.compile(r"token `([^`]+)`")
+
+
+def main() -> int:
+    prompt, cwd = sys.argv[1], Path(sys.argv[2])
+    home = Path(os.environ.get("HOME", ""))
+    # run_evals puts HOME beside the cwd; run_triggers puts it inside the stage it runs from.
+    if home.resolve().parent not in (cwd.resolve().parent, cwd.resolve()):
+        print(f"HOME not inside the stage: {home}", file=sys.stderr)
+        return 4
+    if "HOST_SECRET" in os.environ:
+        print("host secret leaked", file=sys.stderr)
+        return 5
+    if "crash" in prompt:
+        print("boom", file=sys.stderr)
+        return 1
+    tokens = []
+    for skill_md in sorted(cwd.glob(".agents/skills/*/SKILL.md")):
+        m = CANARY_RE.search(skill_md.read_text(encoding="utf-8"))
+        if m:
+            tokens.append(m.group(1))
+        if "edit-me" in prompt:
+            skill_md.write_text("edited by the run\n", encoding="utf-8")
+    (cwd / "made.txt").write_text(prompt, encoding="utf-8")
+    (cwd / "where.txt").write_text(str(cwd.resolve()), encoding="utf-8")
+    (cwd / "env.json").write_text(
+        json.dumps({k: v for k, v in os.environ.items() if k in ("FORWARDED", "ABSENT", "FAKE_CONFIG_DIR")}),
+        encoding="utf-8",
+    )
+    (cwd / "login.txt").write_text(str((home / ".fake" / "auth.json").is_file()), encoding="utf-8")
+    reply = ("yes" in prompt and tokens and tokens[0] + " ") or ""
+    print(json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": reply + "done"}]}}))
+    print(json.dumps({"type": "result", "usage": {"input_tokens": 10, "output_tokens": 5}}))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval/scripts/tests/test_aggregate_benchmark.py

@@ -0,0 +1,41 @@
+"""aggregate_benchmark.py leaves incomplete runs out of the numbers."""
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from aggregate_benchmark import load_records, summarize_config  # noqa: E402
+
+
+class AggregateTest(unittest.TestCase):
+    def test_incomplete_runs_are_excluded_and_counted(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            rows = [
+                ("a", {"status": "ok", "elapsed_s": 10.0}),
+                ("b", {"status": "ok", "elapsed_s": 12.0}),
+                ("c", {"status": "timeout", "elapsed_s": 600.0}),
+                ("d", {"status": "harness-missing", "elapsed_s": 0.01}),
+            ]
+            for name, data in rows:
+                (root / name).mkdir()
+                (root / name / "timing.json").write_text(json.dumps(data))
+            records, excluded = load_records(root)
+            self.assertEqual(excluded, 2)
+            self.assertEqual(summarize_config(records)["metrics"]["elapsed_s"]["mean"], 11.0)
+
+    def test_records_without_status_count_as_complete(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            f = Path(tmp) / "runs.json"
+            f.write_text(json.dumps({"runs": [{"elapsed_s": 1.0}, {"elapsed_s": 3.0}]}))
+            records, excluded = load_records(f)
+            self.assertEqual((len(records), excluded), (2, 0))
+
+
+if __name__ == "__main__":
+    unittest.main()

.agents/skills/bmad-eval/scripts/tests/test_convert_cases.py

@@ -0,0 +1,99 @@
+"""convert_cases.py maps skill-creator evals.json to runner cases.json and back."""
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).resolve().parents[1] / "convert_cases.py"
+
+SKILL_CREATOR = {
+    "skill_name": "brief-writer",
+    "evals": [
+        {
+            "id": 1,
+            "prompt": "write a brief for InsuLens from memo.md",
+            "expected_output": "a brief.md naming InsuLens",
+            "files": ["evals/files/memo.md"],
+            "expectations": ["brief.md exists and names InsuLens", "no invented claims"],
+        },
+        {"id": 2, "prompt": "just say hi", "expected_output": "", "files": [], "expectations": []},
+    ],
+}
+
+
+def run(*args: str) -> subprocess.CompletedProcess:
+    return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True)
+
+
+class ConvertCasesTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp())
+        self.evals = self.tmp / "my-skill" / "evals" / "evals.json"
+        self.evals.parent.mkdir(parents=True)
+        self.evals.write_text(json.dumps(SKILL_CREATOR), encoding="utf-8")
+
+    def test_to_runner(self):
+        proc = run(str(self.evals))
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        cases = json.loads(proc.stdout)
+        self.assertEqual(
+            cases[0],
+            {
+                "id": "1",
+                "input": "write a brief for InsuLens from memo.md",
+                "rubric": [
+                    "brief.md exists and names InsuLens",
+                    "no invented claims",
+                    "output matches: a brief.md naming InsuLens",
+                ],
+                "state_prefix": None,
+                "files": ["evals/files/memo.md"],
+            },
+        )
+        self.assertEqual(cases[1]["rubric"], [], "empty expected_output adds no rubric line")
+        self.assertEqual(cases[1]["id"], "2")
+
+    def test_output_flag_writes_file(self):
+        out = self.tmp / "out" / "cases.json"
+        proc = run(str(self.evals), "--output", str(out))
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertEqual(proc.stdout, "")
+        self.assertEqual(len(json.loads(out.read_text(encoding="utf-8"))), 2)
+
+    def test_round_trip(self):
+        cases_path = self.tmp / "my-skill" / "evals" / "cases.json"
+        run(str(self.evals), "--output", str(cases_path))
+        proc = run(str(cases_path), "--to", "skill-creator")
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        back = json.loads(proc.stdout)
+        self.assertEqual(back["skill_name"], "my-skill", "skill_name defaults to the folder above evals/")
+        self.assertEqual(back["evals"], SKILL_CREATOR["evals"])
+
+    def test_skill_name_flag_and_non_numeric_id(self):
+        cases_path = self.tmp / "cases.json"
+        cases_path.write_text(
+            json.dumps(
+                {"cases": [{"id": "create-1", "input": "x", "rubric": ["a"], "state_prefix": None, "files": []}]}
+            ),
+            encoding="utf-8",
+        )
+        proc = run(str(cases_path), "--to", "skill-creator", "--skill-name", "named")
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        back = json.loads(proc.stdout)
+        self.assertEqual(back["skill_name"], "named")
+        self.assertEqual(back["evals"][0]["id"], "create-1")
+        self.assertEqual(back["evals"][0]["expected_output"], "")
+
+    def test_rejects_runner_input_toward_runner(self):
+        bad = self.tmp / "cases.json"
+        bad.write_text("[]", encoding="utf-8")
+        proc = run(str(bad))
+        self.assertEqual(proc.returncode, 2)
+        self.assertIn("evals", proc.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

.agents/skills/bmad-eval/scripts/tests/test_eval_common.py

@@ -0,0 +1,68 @@
+"""The clean-room contract in eval_common.build_case_env, shared by both runners.
+
+The eval result is only honest if nothing from the host shell leaks into the subprocess: exactly
+PATH, a fresh HOME, and the harness's `env` table, where a value is set as given with "~" meaning
+the fresh HOME and an empty value forwards the host's only when the host has it. Nothing else.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from eval_common import PLATFORM_ENV, build_case_env, validate_harness  # noqa: E402
+
+HOST_ENV = {
+    "PATH": "/usr/bin:/bin",
+    "HOME": "/Users/host",
+    "MY_API_KEY": "sk-test-123",
+    "AWS_SECRET_ACCESS_KEY": "host-secret-must-not-leak",
+    "AGENT_CONFIG_DIR": "/Users/host/.agent",
+    "EXTRA_VAR": "extra",
+}
+
+HOME = Path("/tmp/eval-case/.home")
+
+
+def keys(env: dict) -> set[str]:
+    return set(env) - set(PLATFORM_ENV) - {"USERPROFILE"}
+
+
+class EnvIsolationTest(unittest.TestCase):
+    def test_set_forward_and_expand(self):
+        harness = {"env": {"MY_API_KEY": "", "AGENT_CONFIG_DIR": "~/.agent", "MODE": "quiet"}}
+        env = build_case_env(harness, HOME, HOST_ENV)
+        self.assertEqual(keys(env), {"PATH", "HOME", "AGENT_CONFIG_DIR", "MY_API_KEY", "MODE"})
+        self.assertEqual(env["PATH"], HOST_ENV["PATH"])
+        self.assertEqual(env["HOME"], str(HOME), "HOME must be the fresh case home")
+        self.assertEqual(env["AGENT_CONFIG_DIR"], str(HOME / ".agent"), "~ points into the fresh home, not the host's")
+        self.assertEqual(env["MY_API_KEY"], "sk-test-123", "an empty value forwards the host's")
+        self.assertEqual(env["MODE"], "quiet")
+        self.assertNotIn("AWS_SECRET_ACCESS_KEY", env, "host secrets leaked")
+
+    def test_forwarded_var_absent_when_host_lacks_it(self):
+        # An empty credential would override the CLI's own login; the key must be absent, never empty.
+        for host in ({}, {"MY_API_KEY": ""}):
+            env = build_case_env({"env": {"MY_API_KEY": ""}}, HOME, {"PATH": "/bin", **host})
+            self.assertNotIn("MY_API_KEY", env)
+
+    def test_no_harness_still_minimal(self):
+        self.assertEqual(keys(build_case_env(None, HOME, HOST_ENV)), {"PATH", "HOME"})
+
+    def test_validation(self):
+        good = {"command": ["x", "{prompt}"], "env": {"A": ""}, "home_files": []}
+        self.assertIs(validate_harness(good), good)
+        for bad in (
+            {"command": ["x"]},
+            {"command": "x {prompt}"},
+            {"command": ["x", "{prompt}"], "env": ["A"]},
+            {"command": ["x", "{prompt}"], "home_files": "a"},
+        ):
+            with self.assertRaises(ValueError, msg=bad):
+                validate_harness(bad)
+
+
+if __name__ == "__main__":
+    unittest.main()

.agents/skills/bmad-eval/scripts/tests/test_run_evals.py

@@ -0,0 +1,213 @@
+"""run_evals.py against the fake harness: staging, containment, exit codes, run folders."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+import eval_common  # noqa: E402
+
+RUNNER = SCRIPTS_DIR / "run_evals.py"
+FAKE = Path(__file__).resolve().parent / "fake_harness.py"
+
+
+def write(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def make_project(root: Path, command: list[str] | None = None) -> tuple[Path, Path, Path]:
+    """A project with a skill, a cases file, a fixture and a harness file. Returns (skill, cases, harness)."""
+    skill = root / "skills" / "greet"
+    write(skill / "SKILL.md", "---\nname: greet\ndescription: Greets. Use when greeting.\n---\nSay hi.\n")
+    write(skill / "scripts" / "__pycache__" / "x.pyc", "")
+    write(root / "fixtures" / "notes.md", "fixture\n")
+    cases = root / "evals" / "cases.json"
+    write(
+        cases,
+        json.dumps(
+            [
+                {"id": "c1", "input": "yes hello", "rubric": ["says hi"], "files": ["fixtures/notes.md"]},
+                {"id": "../escape", "input": "hello", "rubric": ["x"]},
+            ]
+        ),
+    )
+    harness = root / "harness.json"
+    write(
+        harness,
+        json.dumps(
+            {
+                "command": command or [sys.executable, str(FAKE), "{prompt}", "{cwd}"],
+                "skill_dir": ".agents/skills",
+                "env": {"FAKE_CONFIG_DIR": "~/.fake", "FORWARDED": "", "ABSENT": ""},
+                "home_files": [".fake/auth.json"],
+            }
+        ),
+    )
+    return skill, cases, harness
+
+
+def run(*args: str, host_home: Path | None = None) -> subprocess.CompletedProcess:
+    """Run the runner with a host secret set and, when given, a fake host home holding a login file."""
+    env = {**os.environ, "HOST_SECRET": "must-not-leak", "FORWARDED": "yes"}
+    env.pop("ABSENT", None)
+    if host_home is not None:
+        (host_home / ".fake").mkdir(parents=True, exist_ok=True)
+        (host_home / ".fake" / "auth.json").write_text("{}", encoding="utf-8")
+        env.update(HOME=str(host_home), USERPROFILE=str(host_home))
+    return subprocess.run([sys.executable, str(RUNNER), *args], capture_output=True, text=True, env=env)
+
+
+class RunEvalsTest(unittest.TestCase):
+    def test_baseline_stages_a_copy_and_contains_the_run(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root)
+            out = root / "out"
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(out),
+                "--mode", "baseline", "--project-root", str(root), "--harness", str(harness), "--quiet",
+                host_home=root / "host-home",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 0, res.stderr)
+            summary = json.loads(res.stdout)
+            self.assertEqual(summary["executed"], 4)
+            run_dir = Path(summary["run_dir"])
+            staged = run_dir / "skill" / "c1" / "cwd" / ".agents" / "skills" / "greet"
+            self.assertTrue(staged.is_dir() and not staged.is_symlink(), "the skill is copied, not linked")
+            self.assertFalse((staged / "scripts" / "__pycache__").exists())
+            self.assertFalse((run_dir / "bare" / "c1" / "cwd" / ".agents").exists(), "bare config stages nothing")
+            self.assertEqual((run_dir / "skill" / "c1" / "cwd" / "fixtures" / "notes.md").read_text(), "fixture\n")
+            case = run_dir / "skill" / "c1"
+            self.assertTrue((case / "cwd" / "made.txt").is_file(), "the workspace comes back as cwd/")
+            self.assertFalse((case / "cwd" / ".git").exists())
+            where = Path((case / "cwd" / "where.txt").read_text())
+            self.assertNotIn(run_dir.resolve(), where.parents, "the run happened outside the project")
+            self.assertEqual((case / "cwd" / "login.txt").read_text(), "True", "home_files brought the login over")
+            seen = json.loads((case / "cwd" / "env.json").read_text())
+            self.assertEqual(seen["FORWARDED"], "yes", "an empty value forwards the host's")
+            self.assertNotIn("ABSENT", seen, "a var the host lacks is left out")
+            self.assertTrue(
+                seen["FAKE_CONFIG_DIR"].endswith(".fake") and Path(seen["FAKE_CONFIG_DIR"]).is_absolute(), "~ expands"
+            )
+            for name in ("prompt.txt", "transcript.jsonl", "stderr.txt", "timing.json", "case.json"):
+                self.assertTrue((case / name).is_file(), name)
+            # the id with `..` stays inside the run folder
+            self.assertTrue((run_dir / "skill" / "_escape").is_dir())
+            self.assertFalse((out / "escape").exists())
+            timing = json.loads((run_dir / "skill" / "c1" / "timing.json").read_text())
+            self.assertEqual(timing["status"], "ok")
+            self.assertEqual(timing["total_tokens"], 15)
+            self.assertTrue(timing["tokens_reported"])
+            run_json = json.loads((run_dir / "run.json").read_text())
+            self.assertEqual(run_json["command"][2], "{prompt}")
+
+    def test_edit_in_a_run_does_not_touch_the_source(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root)
+            write(cases, json.dumps([{"id": "e", "input": "edit-me", "rubric": ["x"]}]))
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
+                "--project-root", str(root), "--harness", str(harness), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 0, res.stderr)
+            self.assertIn("name: greet", (skill / "SKILL.md").read_text())
+
+    def test_fixture_escaping_the_workspace_is_an_error(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root)
+            write(root / "secret.txt", "s\n")
+            write(cases, json.dumps([{"id": "f", "input": "hello", "rubric": ["x"], "files": ["../../secret.txt"]}]))
+            out = root / "out"
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(out),
+                "--project-root", str(root / "evals" / "deep"), "--harness", str(harness), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 1)
+            summary = json.loads(res.stdout)
+            self.assertEqual(summary["results"][0]["status"], "error")
+            self.assertIn("escapes", summary["results"][0]["reason"])
+            # nothing landed above the case's cwd
+            run_dir = Path(summary["run_dir"])
+            self.assertEqual(sorted(p.name for p in run_dir.iterdir()), ["execution-summary.json", "run.json", "skill"])
+
+    def test_failures_exit_nonzero(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root)
+            write(cases, json.dumps([{"id": "k", "input": "crash", "rubric": ["x"]}]))
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
+                "--project-root", str(root), "--harness", str(harness), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 1)
+            self.assertEqual(json.loads(res.stdout)["failures"], 1)
+
+    def test_missing_command_exits_nonzero(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root, command=["no-such-harness-xyz", "{prompt}"])
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
+                "--project-root", str(root), "--harness", str(harness), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 1)
+            self.assertEqual({r["status"] for r in json.loads(res.stdout)["results"]}, {"harness-missing"})
+
+    def test_no_harness_stages_only_and_exits_3(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, _ = make_project(root)
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
+                "--project-root", str(root), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 3, res.stderr)
+            summary = json.loads(res.stdout)
+            self.assertEqual(summary["skipped"], 2)
+            self.assertIn("BMad is not set up", summary["harness"])
+
+    def test_invalid_harness_is_a_usage_error(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill, cases, harness = make_project(root)
+            write(harness, json.dumps({"command": ["claude", "-p"]}))
+            res = run(
+                "--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
+                "--harness", str(harness), "--quiet",
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 2)
+            self.assertIn("{prompt}", res.stderr)
+
+    def test_run_folders_never_collide(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp)
+            a_id, a_dir = eval_common.make_run_dir(out, "x")
+            b_id, b_dir = eval_common.make_run_dir(out, "x")
+            self.assertNotEqual(a_dir, b_dir)
+            self.assertTrue(a_dir.is_dir() and b_dir.is_dir())
+            self.assertTrue(b_id.startswith(a_id))
+
+    def test_contained_and_safe_name(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            self.assertEqual(eval_common.contained(root, "a/b.txt"), (root / "a" / "b.txt").resolve())
+            with self.assertRaises(ValueError):
+                eval_common.contained(root, "../b.txt")
+            with self.assertRaises(ValueError):
+                eval_common.contained(root, "/etc/passwd")
+        self.assertEqual(eval_common.safe_name("../escape"), "_escape")
+        self.assertEqual(eval_common.safe_name(".."), "unnamed")
+        self.assertEqual(eval_common.safe_name("Case 1/B"), "Case_1_B")
+
+
+if __name__ == "__main__":
+    unittest.main()

.agents/skills/bmad-eval/scripts/tests/test_run_triggers.py

@@ -0,0 +1,127 @@
+"""run_triggers.py against the fake harness: canary detection, unmeasured queries, exit codes."""
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from run_triggers import CANARY_PREFIX, detect_load, parse_skill_md, write_synthetic_skill  # noqa: E402
+
+RUNNER = SCRIPTS_DIR / "run_triggers.py"
+FAKE = Path(__file__).resolve().parent / "fake_harness.py"
+
+
+def write(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+class TriggerRunTest(unittest.TestCase):
+    def test_canary_scores_queries_and_leaves_failed_ones_unmeasured(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill = root / "skills" / "greet"
+            write(
+                skill / "SKILL.md",
+                "---\nname: greet\ndescription: 'Greets the user''s friends. Use when greeting.'\n---\nSay hi.\n",
+            )
+            queries = root / "triggers.json"
+            write(
+                queries,
+                json.dumps(
+                    [
+                        {"query": "yes greet me", "should_trigger": True},
+                        {"query": "no thanks", "should_trigger": False},
+                        {"query": "yes but wrong", "should_trigger": False},
+                        {"query": "crash", "should_trigger": False},
+                    ]
+                ),
+            )
+            harness = root / "harness.json"
+            write(harness, json.dumps({"command": [sys.executable, str(FAKE), "{prompt}", "{cwd}"]}))
+            res = subprocess.run(
+                [sys.executable, str(RUNNER), "--skill-path", str(skill), "--queries", str(queries),
+                 "--output-dir", str(root / "out"), "--harness", str(harness), "--runs-per-query", "2", "--quiet"],
+                capture_output=True, text=True,
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 1, res.stderr)
+            out = json.loads(res.stdout)
+            by_query = {r["query"]: r for r in out["results"]}
+            self.assertEqual(by_query["yes greet me"]["trigger_rate"], 1.0)
+            self.assertTrue(by_query["yes greet me"]["pass"])
+            self.assertEqual(by_query["no thanks"]["trigger_rate"], 0.0)
+            self.assertTrue(by_query["no thanks"]["pass"])
+            self.assertFalse(by_query["yes but wrong"]["pass"], "a should-not query that fired fails")
+            crashed = by_query["crash"]
+            self.assertIsNone(crashed["pass"], "a query with failed attempts is never a pass")
+            self.assertEqual(crashed["runs"], 0)
+            self.assertEqual(len(crashed["errors"]), 2)
+            self.assertEqual(out["summary"], {"total": 4, "passed": 2, "failed": 1, "unmeasured": 1})
+            self.assertEqual(
+                out["description"], "Greets the user's friends. Use when greeting.", "YAML quotes are removed"
+            )
+            attempt = root / "out" / out["run_id"] / "queries" / "q000-r0"
+            for name in ("prompt.txt", "transcript.jsonl", "stderr.txt", "timing.json"):
+                self.assertTrue((attempt / name).is_file(), name)
+            staged = list((attempt / "cwd" / ".agents" / "skills").glob("greet-trig-*/SKILL.md"))
+            self.assertEqual(len(staged), 1, "the workspace with the staged skill comes back")
+            self.assertIn("user's friends", staged[0].read_text())
+            self.assertEqual(json.loads((attempt / "timing.json").read_text())["loaded"], True)
+
+    def test_no_harness_exits_3(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            skill = root / "greet"
+            write(skill / "SKILL.md", "---\nname: greet\ndescription: Greets.\n---\n")
+            queries = root / "triggers.json"
+            write(queries, json.dumps([{"query": "hi", "should_trigger": True}]))
+            res = subprocess.run(
+                [sys.executable, str(RUNNER), "--skill-path", str(skill), "--queries", str(queries),
+                 "--output-dir", str(root / "out"), "--project-root", str(root), "--quiet"],
+                capture_output=True, text=True,
+            )  # fmt: skip
+            self.assertEqual(res.returncode, 3, res.stderr)
+            self.assertEqual(json.loads(res.stdout)["summary"]["unmeasured"], 1)
+
+
+class ParseSkillTest(unittest.TestCase):
+    def test_quoted_and_block_descriptions(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cases = {
+                "description: 'It''s quoted. Use when x.'\n": "It's quoted. Use when x.",
+                'description: "Say \\"hi\\". Use when x."\n': 'Say "hi". Use when x.',
+                "description: |\n  Block one.\n  Block two.\n": "Block one. Block two.",
+                "description: 'Spans\n  two lines.'\n": "Spans two lines.",
+                "description: Plain. Use when x.\n": "Plain. Use when x.",
+            }
+            for frontmatter, expected in cases.items():
+                write(Path(tmp) / "SKILL.md", f"---\nname: s\n{frontmatter}---\nBody.\n")
+                self.assertEqual(parse_skill_md(Path(tmp)), ("s", expected), frontmatter)
+
+
+class CanaryTest(unittest.TestCase):
+    def test_token_is_in_the_body_not_the_name_or_description(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            token = CANARY_PREFIX + "abcd1234"
+            name = write_synthetic_skill(Path(tmp), "greet", "Greets people.\nUse when greeting.", token)
+            text = (Path(tmp) / name / "SKILL.md").read_text()
+            frontmatter = text.split("---")[1]
+            self.assertNotIn(token, frontmatter)
+            self.assertNotIn(token, name)
+            self.assertIn(f"token `{token}`", text)
+            self.assertIn("  Use when greeting.", frontmatter)
+
+    def test_detection_needs_the_token(self):
+        token = CANARY_PREFIX + "abcd1234"
+        listing = json.dumps({"type": "system", "skills": ["greet-trig-abcd1234", "other"]})
+        self.assertFalse(detect_load(listing, token), "a startup listing of skill names is not a load")
+        self.assertTrue(detect_load(listing + "\n" + json.dumps({"text": f"{token} hello"}), token))
+
+
+if __name__ == "__main__":
+    unittest.main()