"""The evaluation harness itself, run on the scripted runtime: no model is called.""" import asyncio import json from execution_engine.adk.fake_runtime import FakeAgentRuntime from execution_engine.config import Settings from eval import operating_model_eval as harness from eval import scoring from eval.scoring import CaseResult, Check from tests.support import GRAPH, REQUIREMENTS, SPEC from tests.test_execution import turns def settings() -> Settings: return Settings( # type: ignore[call-arg] _env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False ) CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5} async def run(tmp_path, scripted, monkeypatch): monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC}) return await harness.run_case( CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted) ) async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch): result = await run(tmp_path, turns(), monkeypatch) assert result.status == "completed" failed = [check for check in result.checks if not check.passed] # The scripted packager writes one word; a real one must write a summary. assert [check.name for check in failed] == ["the packager wrote a summary"] assert result.figures["cost_per_month"] == 43410 assert result.figures["scenarios"] == 10 assert (tmp_path / "deliverables").is_dir() assert json.dumps(result.as_dict()) # the whole result can be written down async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch): scripted = turns() scripted["validator"] = [] result = await run(tmp_path, scripted, monkeypatch) assert not result.passed assert result.status != "completed" def test_every_case_names_something_to_start_from(): for case in harness.load_cases(): given = harness.inputs(case) assert "scope" in given or "sources" in given, case["id"] def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask(): from execution_engine.scope import ESSENTIAL, read_scope from tests.support import BUNDLE text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text() scope = read_scope( text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values ) assert scope.gaps(ESSENTIAL) == [] assert scope.customer_case == "bench-landing-zone-shared" def test_a_sound_execution_verifies_every_requirement_and_is_kept_relatively( tmp_path, monkeypatch ): result = asyncio.run(run(tmp_path, turns(), monkeypatch)) coverage = next(c for c in result.checks if c.name.startswith("every Operational")) assert coverage.passed, coverage.detail assert result.figures["requirements_unverified"] == 0 # Results are published: no path of the machine they ran on. assert ( result.folder == "deliverables/cases/northwind-landing-zone/executions/scripted" ) assert str(tmp_path) not in json.dumps(result.as_dict()) def test_a_requirement_no_check_verifies_is_named(): model = {"checks": [c for c in GRAPH["checks"] if c["id"] != "OC-5"]} assert scoring.unverified(REQUIREMENTS, model) == ["OR-4"] assert scoring.unverified(REQUIREMENTS, GRAPH) == [] def result(case: str, people: float, cost: int, passed: bool = True) -> CaseResult: return CaseResult( case=case, status="completed", checks=[Check("completed", passed)], figures={ "people": people, "cost_per_month": cost, "roles": 5, "checks": 9, "capabilities": 17, "currency": "GBP", }, ) def test_the_spread_of_repeated_runs_is_reported(): spread = harness.spread( [result("a", 10.0, 50000), result("a", 12.0, 70000), result("b", 5.0, 1)] ) assert set(spread) == {"a"} # one run has no spread assert spread["a"]["runs"] == 2 assert spread["a"]["cost_per_month"] == { "min": 50000, "max": 70000, "mean": 60000.0, "range_pct": 33.3, } assert spread["a"]["people"]["range_pct"] == 18.2 def test_the_report_can_be_published(): results = [result("a", 10.0, 50000), result("a", 12.0, 70000, passed=False)] text = harness.markdown(results, harness.spread(results), "20261021T120000Z") assert text.startswith("# Target operating model agent: evaluation") assert "| a | 1 | PASS |" in text and "| a | 2 | FAIL |" in text assert "Spread across repeated runs" in text assert "/home/" not in text def test_runs_are_repeated_under_their_own_ids(): assert harness.parse_arguments(["--repeat", "3", "case-x"]) == (["case-x"], 3) assert harness.parse_arguments([]) == ([], 1) assert harness.execution_ids({"id": "case-x"}, 1) == ["case-x"] assert harness.execution_ids({"id": "case-x"}, 2) == ["case-x-r1", "case-x-r2"]