"""The evaluation harness itself, run on the scripted runtime: no model is called.""" import json from execution_engine.adk.fake_runtime import FakeAgentRuntime from execution_engine.config import Settings from eval import operating_model_eval as harness from tests.support import SPEC from tests.test_execution import turns def settings() -> Settings: return Settings( # type: ignore[call-arg] _env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False ) CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5} async def run(tmp_path, scripted, monkeypatch): monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC}) return await harness.run_case( CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted) ) async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch): result = await run(tmp_path, turns(), monkeypatch) assert result.status == "completed" failed = [check for check in result.checks if not check.passed] # The scripted packager writes one word; a real one must write a summary. assert [check.name for check in failed] == ["the packager wrote a summary"] assert result.figures["cost_per_month"] == 43410 assert result.figures["scenarios"] == 10 assert (tmp_path / "deliverables").is_dir() assert json.dumps(result.as_dict()) # the whole result can be written down async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch): scripted = turns() scripted["validator"] = [] result = await run(tmp_path, scripted, monkeypatch) assert not result.passed assert result.status != "completed" def test_every_case_names_something_to_start_from(): for case in harness.load_cases(): given = harness.inputs(case) assert "scope" in given or "sources" in given, case["id"] def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask(): from execution_engine.scope import ESSENTIAL, read_scope from tests.support import BUNDLE text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text() scope = read_scope( text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values ) assert scope.gaps(ESSENTIAL) == [] assert scope.customer_case == "bench-landing-zone-shared"