100 lines
3.7 KiB
Python
100 lines
3.7 KiB
Python
"""convert_cases.py maps skill-creator evals.json to runner cases.json and back."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
SCRIPT = Path(__file__).resolve().parents[1] / "convert_cases.py"
|
|
|
|
SKILL_CREATOR = {
|
|
"skill_name": "brief-writer",
|
|
"evals": [
|
|
{
|
|
"id": 1,
|
|
"prompt": "write a brief for InsuLens from memo.md",
|
|
"expected_output": "a brief.md naming InsuLens",
|
|
"files": ["evals/files/memo.md"],
|
|
"expectations": ["brief.md exists and names InsuLens", "no invented claims"],
|
|
},
|
|
{"id": 2, "prompt": "just say hi", "expected_output": "", "files": [], "expectations": []},
|
|
],
|
|
}
|
|
|
|
|
|
def run(*args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True)
|
|
|
|
|
|
class ConvertCasesTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.evals = self.tmp / "my-skill" / "evals" / "evals.json"
|
|
self.evals.parent.mkdir(parents=True)
|
|
self.evals.write_text(json.dumps(SKILL_CREATOR), encoding="utf-8")
|
|
|
|
def test_to_runner(self):
|
|
proc = run(str(self.evals))
|
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
|
cases = json.loads(proc.stdout)
|
|
self.assertEqual(
|
|
cases[0],
|
|
{
|
|
"id": "1",
|
|
"input": "write a brief for InsuLens from memo.md",
|
|
"rubric": [
|
|
"brief.md exists and names InsuLens",
|
|
"no invented claims",
|
|
"output matches: a brief.md naming InsuLens",
|
|
],
|
|
"state_prefix": None,
|
|
"files": ["evals/files/memo.md"],
|
|
},
|
|
)
|
|
self.assertEqual(cases[1]["rubric"], [], "empty expected_output adds no rubric line")
|
|
self.assertEqual(cases[1]["id"], "2")
|
|
|
|
def test_output_flag_writes_file(self):
|
|
out = self.tmp / "out" / "cases.json"
|
|
proc = run(str(self.evals), "--output", str(out))
|
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
|
self.assertEqual(proc.stdout, "")
|
|
self.assertEqual(len(json.loads(out.read_text(encoding="utf-8"))), 2)
|
|
|
|
def test_round_trip(self):
|
|
cases_path = self.tmp / "my-skill" / "evals" / "cases.json"
|
|
run(str(self.evals), "--output", str(cases_path))
|
|
proc = run(str(cases_path), "--to", "skill-creator")
|
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
|
back = json.loads(proc.stdout)
|
|
self.assertEqual(back["skill_name"], "my-skill", "skill_name defaults to the folder above evals/")
|
|
self.assertEqual(back["evals"], SKILL_CREATOR["evals"])
|
|
|
|
def test_skill_name_flag_and_non_numeric_id(self):
|
|
cases_path = self.tmp / "cases.json"
|
|
cases_path.write_text(
|
|
json.dumps(
|
|
{"cases": [{"id": "create-1", "input": "x", "rubric": ["a"], "state_prefix": None, "files": []}]}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
proc = run(str(cases_path), "--to", "skill-creator", "--skill-name", "named")
|
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
|
back = json.loads(proc.stdout)
|
|
self.assertEqual(back["skill_name"], "named")
|
|
self.assertEqual(back["evals"][0]["id"], "create-1")
|
|
self.assertEqual(back["evals"][0]["expected_output"], "")
|
|
|
|
def test_rejects_runner_input_toward_runner(self):
|
|
bad = self.tmp / "cases.json"
|
|
bad.write_text("[]", encoding="utf-8")
|
|
proc = run(str(bad))
|
|
self.assertEqual(proc.returncode, 2)
|
|
self.assertIn("evals", proc.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|