basic BMAD setup
This commit is contained in:
Ebbe Baß
@@ -0,0 +1,52 @@
|
||||
"""A stand-in agent CLI for the runner tests: `fake_harness.py <prompt> <cwd>`.
|
||||
|
||||
Behaves like a harness enough to exercise the runners: reads the skills staged under
|
||||
`.agents/skills` in the cwd, obeys a canary line when the prompt says "yes", writes an
|
||||
artifact, prints a usage line, and exits non-zero when the prompt says "crash". It also
|
||||
checks the clean room: HOME must sit inside the stage and the host secret must be absent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
CANARY_RE = re.compile(r"token `([^`]+)`")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
prompt, cwd = sys.argv[1], Path(sys.argv[2])
|
||||
home = Path(os.environ.get("HOME", ""))
|
||||
# run_evals puts HOME beside the cwd; run_triggers puts it inside the stage it runs from.
|
||||
if home.resolve().parent not in (cwd.resolve().parent, cwd.resolve()):
|
||||
print(f"HOME not inside the stage: {home}", file=sys.stderr)
|
||||
return 4
|
||||
if "HOST_SECRET" in os.environ:
|
||||
print("host secret leaked", file=sys.stderr)
|
||||
return 5
|
||||
if "crash" in prompt:
|
||||
print("boom", file=sys.stderr)
|
||||
return 1
|
||||
tokens = []
|
||||
for skill_md in sorted(cwd.glob(".agents/skills/*/SKILL.md")):
|
||||
m = CANARY_RE.search(skill_md.read_text(encoding="utf-8"))
|
||||
if m:
|
||||
tokens.append(m.group(1))
|
||||
if "edit-me" in prompt:
|
||||
skill_md.write_text("edited by the run\n", encoding="utf-8")
|
||||
(cwd / "made.txt").write_text(prompt, encoding="utf-8")
|
||||
(cwd / "where.txt").write_text(str(cwd.resolve()), encoding="utf-8")
|
||||
(cwd / "env.json").write_text(
|
||||
json.dumps({k: v for k, v in os.environ.items() if k in ("FORWARDED", "ABSENT", "FAKE_CONFIG_DIR")}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(cwd / "login.txt").write_text(str((home / ".fake" / "auth.json").is_file()), encoding="utf-8")
|
||||
reply = ("yes" in prompt and tokens and tokens[0] + " ") or ""
|
||||
print(json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": reply + "done"}]}}))
|
||||
print(json.dumps({"type": "result", "usage": {"input_tokens": 10, "output_tokens": 5}}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,41 @@
|
||||
"""aggregate_benchmark.py leaves incomplete runs out of the numbers."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from aggregate_benchmark import load_records, summarize_config # noqa: E402
|
||||
|
||||
|
||||
class AggregateTest(unittest.TestCase):
|
||||
def test_incomplete_runs_are_excluded_and_counted(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
rows = [
|
||||
("a", {"status": "ok", "elapsed_s": 10.0}),
|
||||
("b", {"status": "ok", "elapsed_s": 12.0}),
|
||||
("c", {"status": "timeout", "elapsed_s": 600.0}),
|
||||
("d", {"status": "harness-missing", "elapsed_s": 0.01}),
|
||||
]
|
||||
for name, data in rows:
|
||||
(root / name).mkdir()
|
||||
(root / name / "timing.json").write_text(json.dumps(data))
|
||||
records, excluded = load_records(root)
|
||||
self.assertEqual(excluded, 2)
|
||||
self.assertEqual(summarize_config(records)["metrics"]["elapsed_s"]["mean"], 11.0)
|
||||
|
||||
def test_records_without_status_count_as_complete(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
f = Path(tmp) / "runs.json"
|
||||
f.write_text(json.dumps({"runs": [{"elapsed_s": 1.0}, {"elapsed_s": 3.0}]}))
|
||||
records, excluded = load_records(f)
|
||||
self.assertEqual((len(records), excluded), (2, 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""convert_cases.py maps skill-creator evals.json to runner cases.json and back."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "convert_cases.py"
|
||||
|
||||
SKILL_CREATOR = {
|
||||
"skill_name": "brief-writer",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"prompt": "write a brief for InsuLens from memo.md",
|
||||
"expected_output": "a brief.md naming InsuLens",
|
||||
"files": ["evals/files/memo.md"],
|
||||
"expectations": ["brief.md exists and names InsuLens", "no invented claims"],
|
||||
},
|
||||
{"id": 2, "prompt": "just say hi", "expected_output": "", "files": [], "expectations": []},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run(*args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True)
|
||||
|
||||
|
||||
class ConvertCasesTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.evals = self.tmp / "my-skill" / "evals" / "evals.json"
|
||||
self.evals.parent.mkdir(parents=True)
|
||||
self.evals.write_text(json.dumps(SKILL_CREATOR), encoding="utf-8")
|
||||
|
||||
def test_to_runner(self):
|
||||
proc = run(str(self.evals))
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
cases = json.loads(proc.stdout)
|
||||
self.assertEqual(
|
||||
cases[0],
|
||||
{
|
||||
"id": "1",
|
||||
"input": "write a brief for InsuLens from memo.md",
|
||||
"rubric": [
|
||||
"brief.md exists and names InsuLens",
|
||||
"no invented claims",
|
||||
"output matches: a brief.md naming InsuLens",
|
||||
],
|
||||
"state_prefix": None,
|
||||
"files": ["evals/files/memo.md"],
|
||||
},
|
||||
)
|
||||
self.assertEqual(cases[1]["rubric"], [], "empty expected_output adds no rubric line")
|
||||
self.assertEqual(cases[1]["id"], "2")
|
||||
|
||||
def test_output_flag_writes_file(self):
|
||||
out = self.tmp / "out" / "cases.json"
|
||||
proc = run(str(self.evals), "--output", str(out))
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertEqual(proc.stdout, "")
|
||||
self.assertEqual(len(json.loads(out.read_text(encoding="utf-8"))), 2)
|
||||
|
||||
def test_round_trip(self):
|
||||
cases_path = self.tmp / "my-skill" / "evals" / "cases.json"
|
||||
run(str(self.evals), "--output", str(cases_path))
|
||||
proc = run(str(cases_path), "--to", "skill-creator")
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
back = json.loads(proc.stdout)
|
||||
self.assertEqual(back["skill_name"], "my-skill", "skill_name defaults to the folder above evals/")
|
||||
self.assertEqual(back["evals"], SKILL_CREATOR["evals"])
|
||||
|
||||
def test_skill_name_flag_and_non_numeric_id(self):
|
||||
cases_path = self.tmp / "cases.json"
|
||||
cases_path.write_text(
|
||||
json.dumps(
|
||||
{"cases": [{"id": "create-1", "input": "x", "rubric": ["a"], "state_prefix": None, "files": []}]}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
proc = run(str(cases_path), "--to", "skill-creator", "--skill-name", "named")
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
back = json.loads(proc.stdout)
|
||||
self.assertEqual(back["skill_name"], "named")
|
||||
self.assertEqual(back["evals"][0]["id"], "create-1")
|
||||
self.assertEqual(back["evals"][0]["expected_output"], "")
|
||||
|
||||
def test_rejects_runner_input_toward_runner(self):
|
||||
bad = self.tmp / "cases.json"
|
||||
bad.write_text("[]", encoding="utf-8")
|
||||
proc = run(str(bad))
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertIn("evals", proc.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""The clean-room contract in eval_common.build_case_env, shared by both runners.
|
||||
|
||||
The eval result is only honest if nothing from the host shell leaks into the subprocess: exactly
|
||||
PATH, a fresh HOME, and the harness's `env` table, where a value is set as given with "~" meaning
|
||||
the fresh HOME and an empty value forwards the host's only when the host has it. Nothing else.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from eval_common import PLATFORM_ENV, build_case_env, validate_harness # noqa: E402
|
||||
|
||||
HOST_ENV = {
|
||||
"PATH": "/usr/bin:/bin",
|
||||
"HOME": "/Users/host",
|
||||
"MY_API_KEY": "sk-test-123",
|
||||
"AWS_SECRET_ACCESS_KEY": "host-secret-must-not-leak",
|
||||
"AGENT_CONFIG_DIR": "/Users/host/.agent",
|
||||
"EXTRA_VAR": "extra",
|
||||
}
|
||||
|
||||
HOME = Path("/tmp/eval-case/.home")
|
||||
|
||||
|
||||
def keys(env: dict) -> set[str]:
|
||||
return set(env) - set(PLATFORM_ENV) - {"USERPROFILE"}
|
||||
|
||||
|
||||
class EnvIsolationTest(unittest.TestCase):
|
||||
def test_set_forward_and_expand(self):
|
||||
harness = {"env": {"MY_API_KEY": "", "AGENT_CONFIG_DIR": "~/.agent", "MODE": "quiet"}}
|
||||
env = build_case_env(harness, HOME, HOST_ENV)
|
||||
self.assertEqual(keys(env), {"PATH", "HOME", "AGENT_CONFIG_DIR", "MY_API_KEY", "MODE"})
|
||||
self.assertEqual(env["PATH"], HOST_ENV["PATH"])
|
||||
self.assertEqual(env["HOME"], str(HOME), "HOME must be the fresh case home")
|
||||
self.assertEqual(env["AGENT_CONFIG_DIR"], str(HOME / ".agent"), "~ points into the fresh home, not the host's")
|
||||
self.assertEqual(env["MY_API_KEY"], "sk-test-123", "an empty value forwards the host's")
|
||||
self.assertEqual(env["MODE"], "quiet")
|
||||
self.assertNotIn("AWS_SECRET_ACCESS_KEY", env, "host secrets leaked")
|
||||
|
||||
def test_forwarded_var_absent_when_host_lacks_it(self):
|
||||
# An empty credential would override the CLI's own login; the key must be absent, never empty.
|
||||
for host in ({}, {"MY_API_KEY": ""}):
|
||||
env = build_case_env({"env": {"MY_API_KEY": ""}}, HOME, {"PATH": "/bin", **host})
|
||||
self.assertNotIn("MY_API_KEY", env)
|
||||
|
||||
def test_no_harness_still_minimal(self):
|
||||
self.assertEqual(keys(build_case_env(None, HOME, HOST_ENV)), {"PATH", "HOME"})
|
||||
|
||||
def test_validation(self):
|
||||
good = {"command": ["x", "{prompt}"], "env": {"A": ""}, "home_files": []}
|
||||
self.assertIs(validate_harness(good), good)
|
||||
for bad in (
|
||||
{"command": ["x"]},
|
||||
{"command": "x {prompt}"},
|
||||
{"command": ["x", "{prompt}"], "env": ["A"]},
|
||||
{"command": ["x", "{prompt}"], "home_files": "a"},
|
||||
):
|
||||
with self.assertRaises(ValueError, msg=bad):
|
||||
validate_harness(bad)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""run_evals.py against the fake harness: staging, containment, exit codes, run folders."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import eval_common # noqa: E402
|
||||
|
||||
RUNNER = SCRIPTS_DIR / "run_evals.py"
|
||||
FAKE = Path(__file__).resolve().parent / "fake_harness.py"
|
||||
|
||||
|
||||
def write(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def make_project(root: Path, command: list[str] | None = None) -> tuple[Path, Path, Path]:
|
||||
"""A project with a skill, a cases file, a fixture and a harness file. Returns (skill, cases, harness)."""
|
||||
skill = root / "skills" / "greet"
|
||||
write(skill / "SKILL.md", "---\nname: greet\ndescription: Greets. Use when greeting.\n---\nSay hi.\n")
|
||||
write(skill / "scripts" / "__pycache__" / "x.pyc", "")
|
||||
write(root / "fixtures" / "notes.md", "fixture\n")
|
||||
cases = root / "evals" / "cases.json"
|
||||
write(
|
||||
cases,
|
||||
json.dumps(
|
||||
[
|
||||
{"id": "c1", "input": "yes hello", "rubric": ["says hi"], "files": ["fixtures/notes.md"]},
|
||||
{"id": "../escape", "input": "hello", "rubric": ["x"]},
|
||||
]
|
||||
),
|
||||
)
|
||||
harness = root / "harness.json"
|
||||
write(
|
||||
harness,
|
||||
json.dumps(
|
||||
{
|
||||
"command": command or [sys.executable, str(FAKE), "{prompt}", "{cwd}"],
|
||||
"skill_dir": ".agents/skills",
|
||||
"env": {"FAKE_CONFIG_DIR": "~/.fake", "FORWARDED": "", "ABSENT": ""},
|
||||
"home_files": [".fake/auth.json"],
|
||||
}
|
||||
),
|
||||
)
|
||||
return skill, cases, harness
|
||||
|
||||
|
||||
def run(*args: str, host_home: Path | None = None) -> subprocess.CompletedProcess:
|
||||
"""Run the runner with a host secret set and, when given, a fake host home holding a login file."""
|
||||
env = {**os.environ, "HOST_SECRET": "must-not-leak", "FORWARDED": "yes"}
|
||||
env.pop("ABSENT", None)
|
||||
if host_home is not None:
|
||||
(host_home / ".fake").mkdir(parents=True, exist_ok=True)
|
||||
(host_home / ".fake" / "auth.json").write_text("{}", encoding="utf-8")
|
||||
env.update(HOME=str(host_home), USERPROFILE=str(host_home))
|
||||
return subprocess.run([sys.executable, str(RUNNER), *args], capture_output=True, text=True, env=env)
|
||||
|
||||
|
||||
class RunEvalsTest(unittest.TestCase):
|
||||
def test_baseline_stages_a_copy_and_contains_the_run(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root)
|
||||
out = root / "out"
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(out),
|
||||
"--mode", "baseline", "--project-root", str(root), "--harness", str(harness), "--quiet",
|
||||
host_home=root / "host-home",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 0, res.stderr)
|
||||
summary = json.loads(res.stdout)
|
||||
self.assertEqual(summary["executed"], 4)
|
||||
run_dir = Path(summary["run_dir"])
|
||||
staged = run_dir / "skill" / "c1" / "cwd" / ".agents" / "skills" / "greet"
|
||||
self.assertTrue(staged.is_dir() and not staged.is_symlink(), "the skill is copied, not linked")
|
||||
self.assertFalse((staged / "scripts" / "__pycache__").exists())
|
||||
self.assertFalse((run_dir / "bare" / "c1" / "cwd" / ".agents").exists(), "bare config stages nothing")
|
||||
self.assertEqual((run_dir / "skill" / "c1" / "cwd" / "fixtures" / "notes.md").read_text(), "fixture\n")
|
||||
case = run_dir / "skill" / "c1"
|
||||
self.assertTrue((case / "cwd" / "made.txt").is_file(), "the workspace comes back as cwd/")
|
||||
self.assertFalse((case / "cwd" / ".git").exists())
|
||||
where = Path((case / "cwd" / "where.txt").read_text())
|
||||
self.assertNotIn(run_dir.resolve(), where.parents, "the run happened outside the project")
|
||||
self.assertEqual((case / "cwd" / "login.txt").read_text(), "True", "home_files brought the login over")
|
||||
seen = json.loads((case / "cwd" / "env.json").read_text())
|
||||
self.assertEqual(seen["FORWARDED"], "yes", "an empty value forwards the host's")
|
||||
self.assertNotIn("ABSENT", seen, "a var the host lacks is left out")
|
||||
self.assertTrue(
|
||||
seen["FAKE_CONFIG_DIR"].endswith(".fake") and Path(seen["FAKE_CONFIG_DIR"]).is_absolute(), "~ expands"
|
||||
)
|
||||
for name in ("prompt.txt", "transcript.jsonl", "stderr.txt", "timing.json", "case.json"):
|
||||
self.assertTrue((case / name).is_file(), name)
|
||||
# the id with `..` stays inside the run folder
|
||||
self.assertTrue((run_dir / "skill" / "_escape").is_dir())
|
||||
self.assertFalse((out / "escape").exists())
|
||||
timing = json.loads((run_dir / "skill" / "c1" / "timing.json").read_text())
|
||||
self.assertEqual(timing["status"], "ok")
|
||||
self.assertEqual(timing["total_tokens"], 15)
|
||||
self.assertTrue(timing["tokens_reported"])
|
||||
run_json = json.loads((run_dir / "run.json").read_text())
|
||||
self.assertEqual(run_json["command"][2], "{prompt}")
|
||||
|
||||
def test_edit_in_a_run_does_not_touch_the_source(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root)
|
||||
write(cases, json.dumps([{"id": "e", "input": "edit-me", "rubric": ["x"]}]))
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
|
||||
"--project-root", str(root), "--harness", str(harness), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 0, res.stderr)
|
||||
self.assertIn("name: greet", (skill / "SKILL.md").read_text())
|
||||
|
||||
def test_fixture_escaping_the_workspace_is_an_error(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root)
|
||||
write(root / "secret.txt", "s\n")
|
||||
write(cases, json.dumps([{"id": "f", "input": "hello", "rubric": ["x"], "files": ["../../secret.txt"]}]))
|
||||
out = root / "out"
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(out),
|
||||
"--project-root", str(root / "evals" / "deep"), "--harness", str(harness), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 1)
|
||||
summary = json.loads(res.stdout)
|
||||
self.assertEqual(summary["results"][0]["status"], "error")
|
||||
self.assertIn("escapes", summary["results"][0]["reason"])
|
||||
# nothing landed above the case's cwd
|
||||
run_dir = Path(summary["run_dir"])
|
||||
self.assertEqual(sorted(p.name for p in run_dir.iterdir()), ["execution-summary.json", "run.json", "skill"])
|
||||
|
||||
def test_failures_exit_nonzero(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root)
|
||||
write(cases, json.dumps([{"id": "k", "input": "crash", "rubric": ["x"]}]))
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
|
||||
"--project-root", str(root), "--harness", str(harness), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 1)
|
||||
self.assertEqual(json.loads(res.stdout)["failures"], 1)
|
||||
|
||||
def test_missing_command_exits_nonzero(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root, command=["no-such-harness-xyz", "{prompt}"])
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
|
||||
"--project-root", str(root), "--harness", str(harness), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 1)
|
||||
self.assertEqual({r["status"] for r in json.loads(res.stdout)["results"]}, {"harness-missing"})
|
||||
|
||||
def test_no_harness_stages_only_and_exits_3(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, _ = make_project(root)
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
|
||||
"--project-root", str(root), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 3, res.stderr)
|
||||
summary = json.loads(res.stdout)
|
||||
self.assertEqual(summary["skipped"], 2)
|
||||
self.assertIn("BMad is not set up", summary["harness"])
|
||||
|
||||
def test_invalid_harness_is_a_usage_error(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill, cases, harness = make_project(root)
|
||||
write(harness, json.dumps({"command": ["claude", "-p"]}))
|
||||
res = run(
|
||||
"--cases", str(cases), "--skill-path", str(skill), "--output-dir", str(root / "out"),
|
||||
"--harness", str(harness), "--quiet",
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 2)
|
||||
self.assertIn("{prompt}", res.stderr)
|
||||
|
||||
def test_run_folders_never_collide(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
out = Path(tmp)
|
||||
a_id, a_dir = eval_common.make_run_dir(out, "x")
|
||||
b_id, b_dir = eval_common.make_run_dir(out, "x")
|
||||
self.assertNotEqual(a_dir, b_dir)
|
||||
self.assertTrue(a_dir.is_dir() and b_dir.is_dir())
|
||||
self.assertTrue(b_id.startswith(a_id))
|
||||
|
||||
def test_contained_and_safe_name(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.assertEqual(eval_common.contained(root, "a/b.txt"), (root / "a" / "b.txt").resolve())
|
||||
with self.assertRaises(ValueError):
|
||||
eval_common.contained(root, "../b.txt")
|
||||
with self.assertRaises(ValueError):
|
||||
eval_common.contained(root, "/etc/passwd")
|
||||
self.assertEqual(eval_common.safe_name("../escape"), "_escape")
|
||||
self.assertEqual(eval_common.safe_name(".."), "unnamed")
|
||||
self.assertEqual(eval_common.safe_name("Case 1/B"), "Case_1_B")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""run_triggers.py against the fake harness: canary detection, unmeasured queries, exit codes."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from run_triggers import CANARY_PREFIX, detect_load, parse_skill_md, write_synthetic_skill # noqa: E402
|
||||
|
||||
RUNNER = SCRIPTS_DIR / "run_triggers.py"
|
||||
FAKE = Path(__file__).resolve().parent / "fake_harness.py"
|
||||
|
||||
|
||||
def write(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
class TriggerRunTest(unittest.TestCase):
|
||||
def test_canary_scores_queries_and_leaves_failed_ones_unmeasured(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill = root / "skills" / "greet"
|
||||
write(
|
||||
skill / "SKILL.md",
|
||||
"---\nname: greet\ndescription: 'Greets the user''s friends. Use when greeting.'\n---\nSay hi.\n",
|
||||
)
|
||||
queries = root / "triggers.json"
|
||||
write(
|
||||
queries,
|
||||
json.dumps(
|
||||
[
|
||||
{"query": "yes greet me", "should_trigger": True},
|
||||
{"query": "no thanks", "should_trigger": False},
|
||||
{"query": "yes but wrong", "should_trigger": False},
|
||||
{"query": "crash", "should_trigger": False},
|
||||
]
|
||||
),
|
||||
)
|
||||
harness = root / "harness.json"
|
||||
write(harness, json.dumps({"command": [sys.executable, str(FAKE), "{prompt}", "{cwd}"]}))
|
||||
res = subprocess.run(
|
||||
[sys.executable, str(RUNNER), "--skill-path", str(skill), "--queries", str(queries),
|
||||
"--output-dir", str(root / "out"), "--harness", str(harness), "--runs-per-query", "2", "--quiet"],
|
||||
capture_output=True, text=True,
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 1, res.stderr)
|
||||
out = json.loads(res.stdout)
|
||||
by_query = {r["query"]: r for r in out["results"]}
|
||||
self.assertEqual(by_query["yes greet me"]["trigger_rate"], 1.0)
|
||||
self.assertTrue(by_query["yes greet me"]["pass"])
|
||||
self.assertEqual(by_query["no thanks"]["trigger_rate"], 0.0)
|
||||
self.assertTrue(by_query["no thanks"]["pass"])
|
||||
self.assertFalse(by_query["yes but wrong"]["pass"], "a should-not query that fired fails")
|
||||
crashed = by_query["crash"]
|
||||
self.assertIsNone(crashed["pass"], "a query with failed attempts is never a pass")
|
||||
self.assertEqual(crashed["runs"], 0)
|
||||
self.assertEqual(len(crashed["errors"]), 2)
|
||||
self.assertEqual(out["summary"], {"total": 4, "passed": 2, "failed": 1, "unmeasured": 1})
|
||||
self.assertEqual(
|
||||
out["description"], "Greets the user's friends. Use when greeting.", "YAML quotes are removed"
|
||||
)
|
||||
attempt = root / "out" / out["run_id"] / "queries" / "q000-r0"
|
||||
for name in ("prompt.txt", "transcript.jsonl", "stderr.txt", "timing.json"):
|
||||
self.assertTrue((attempt / name).is_file(), name)
|
||||
staged = list((attempt / "cwd" / ".agents" / "skills").glob("greet-trig-*/SKILL.md"))
|
||||
self.assertEqual(len(staged), 1, "the workspace with the staged skill comes back")
|
||||
self.assertIn("user's friends", staged[0].read_text())
|
||||
self.assertEqual(json.loads((attempt / "timing.json").read_text())["loaded"], True)
|
||||
|
||||
def test_no_harness_exits_3(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill = root / "greet"
|
||||
write(skill / "SKILL.md", "---\nname: greet\ndescription: Greets.\n---\n")
|
||||
queries = root / "triggers.json"
|
||||
write(queries, json.dumps([{"query": "hi", "should_trigger": True}]))
|
||||
res = subprocess.run(
|
||||
[sys.executable, str(RUNNER), "--skill-path", str(skill), "--queries", str(queries),
|
||||
"--output-dir", str(root / "out"), "--project-root", str(root), "--quiet"],
|
||||
capture_output=True, text=True,
|
||||
) # fmt: skip
|
||||
self.assertEqual(res.returncode, 3, res.stderr)
|
||||
self.assertEqual(json.loads(res.stdout)["summary"]["unmeasured"], 1)
|
||||
|
||||
|
||||
class ParseSkillTest(unittest.TestCase):
|
||||
def test_quoted_and_block_descriptions(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cases = {
|
||||
"description: 'It''s quoted. Use when x.'\n": "It's quoted. Use when x.",
|
||||
'description: "Say \\"hi\\". Use when x."\n': 'Say "hi". Use when x.',
|
||||
"description: |\n Block one.\n Block two.\n": "Block one. Block two.",
|
||||
"description: 'Spans\n two lines.'\n": "Spans two lines.",
|
||||
"description: Plain. Use when x.\n": "Plain. Use when x.",
|
||||
}
|
||||
for frontmatter, expected in cases.items():
|
||||
write(Path(tmp) / "SKILL.md", f"---\nname: s\n{frontmatter}---\nBody.\n")
|
||||
self.assertEqual(parse_skill_md(Path(tmp)), ("s", expected), frontmatter)
|
||||
|
||||
|
||||
class CanaryTest(unittest.TestCase):
|
||||
def test_token_is_in_the_body_not_the_name_or_description(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
token = CANARY_PREFIX + "abcd1234"
|
||||
name = write_synthetic_skill(Path(tmp), "greet", "Greets people.\nUse when greeting.", token)
|
||||
text = (Path(tmp) / name / "SKILL.md").read_text()
|
||||
frontmatter = text.split("---")[1]
|
||||
self.assertNotIn(token, frontmatter)
|
||||
self.assertNotIn(token, name)
|
||||
self.assertIn(f"token `{token}`", text)
|
||||
self.assertIn(" Use when greeting.", frontmatter)
|
||||
|
||||
def test_detection_needs_the_token(self):
|
||||
token = CANARY_PREFIX + "abcd1234"
|
||||
listing = json.dumps({"type": "system", "skills": ["greet-trig-abcd1234", "other"]})
|
||||
self.assertFalse(detect_load(listing, token), "a startup listing of skill names is not a load")
|
||||
self.assertTrue(detect_load(listing + "\n" + json.dumps({"text": f"{token} hello"}), token))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user