69 lines
2.7 KiB
Python
69 lines
2.7 KiB
Python
"""The clean-room contract in eval_common.build_case_env, shared by both runners.
|
|
|
|
The eval result is only honest if nothing from the host shell leaks into the subprocess: exactly
|
|
PATH, a fresh HOME, and the harness's `env` table, where a value is set as given with "~" meaning
|
|
the fresh HOME and an empty value forwards the host's only when the host has it. Nothing else.
|
|
"""
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(SCRIPTS_DIR))
|
|
|
|
from eval_common import PLATFORM_ENV, build_case_env, validate_harness # noqa: E402
|
|
|
|
HOST_ENV = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"HOME": "/Users/host",
|
|
"MY_API_KEY": "sk-test-123",
|
|
"AWS_SECRET_ACCESS_KEY": "host-secret-must-not-leak",
|
|
"AGENT_CONFIG_DIR": "/Users/host/.agent",
|
|
"EXTRA_VAR": "extra",
|
|
}
|
|
|
|
HOME = Path("/tmp/eval-case/.home")
|
|
|
|
|
|
def keys(env: dict) -> set[str]:
|
|
return set(env) - set(PLATFORM_ENV) - {"USERPROFILE"}
|
|
|
|
|
|
class EnvIsolationTest(unittest.TestCase):
|
|
def test_set_forward_and_expand(self):
|
|
harness = {"env": {"MY_API_KEY": "", "AGENT_CONFIG_DIR": "~/.agent", "MODE": "quiet"}}
|
|
env = build_case_env(harness, HOME, HOST_ENV)
|
|
self.assertEqual(keys(env), {"PATH", "HOME", "AGENT_CONFIG_DIR", "MY_API_KEY", "MODE"})
|
|
self.assertEqual(env["PATH"], HOST_ENV["PATH"])
|
|
self.assertEqual(env["HOME"], str(HOME), "HOME must be the fresh case home")
|
|
self.assertEqual(env["AGENT_CONFIG_DIR"], str(HOME / ".agent"), "~ points into the fresh home, not the host's")
|
|
self.assertEqual(env["MY_API_KEY"], "sk-test-123", "an empty value forwards the host's")
|
|
self.assertEqual(env["MODE"], "quiet")
|
|
self.assertNotIn("AWS_SECRET_ACCESS_KEY", env, "host secrets leaked")
|
|
|
|
def test_forwarded_var_absent_when_host_lacks_it(self):
|
|
# An empty credential would override the CLI's own login; the key must be absent, never empty.
|
|
for host in ({}, {"MY_API_KEY": ""}):
|
|
env = build_case_env({"env": {"MY_API_KEY": ""}}, HOME, {"PATH": "/bin", **host})
|
|
self.assertNotIn("MY_API_KEY", env)
|
|
|
|
def test_no_harness_still_minimal(self):
|
|
self.assertEqual(keys(build_case_env(None, HOME, HOST_ENV)), {"PATH", "HOME"})
|
|
|
|
def test_validation(self):
|
|
good = {"command": ["x", "{prompt}"], "env": {"A": ""}, "home_files": []}
|
|
self.assertIs(validate_harness(good), good)
|
|
for bad in (
|
|
{"command": ["x"]},
|
|
{"command": "x {prompt}"},
|
|
{"command": ["x", "{prompt}"], "env": ["A"]},
|
|
{"command": ["x", "{prompt}"], "home_files": "a"},
|
|
):
|
|
with self.assertRaises(ValueError, msg=bad):
|
|
validate_harness(bad)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|