Files
tom_analysis_agent/tests/test_eval.py

70 lines
2.3 KiB
Python

"""The evaluation harness itself, run on the scripted runtime: no model is called."""
import json
from execution_engine.adk.fake_runtime import FakeAgentRuntime
from execution_engine.config import Settings
from eval import operating_model_eval as harness
from tests.support import SPEC
from tests.test_execution import turns
def settings() -> Settings:
return Settings( # type: ignore[call-arg]
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
)
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
async def run(tmp_path, scripted, monkeypatch):
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
return await harness.run_case(
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
)
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
result = await run(tmp_path, turns(), monkeypatch)
assert result.status == "completed"
failed = [check for check in result.checks if not check.passed]
# The scripted packager writes one word; a real one must write a summary.
assert [check.name for check in failed] == ["the packager wrote a summary"]
assert result.figures["cost_per_month"] == 43410
assert result.figures["scenarios"] == 10
assert (tmp_path / "deliverables").is_dir()
assert json.dumps(result.as_dict()) # the whole result can be written down
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
scripted = turns()
scripted["validator"] = []
result = await run(tmp_path, scripted, monkeypatch)
assert not result.passed
assert result.status != "completed"
def test_every_case_names_something_to_start_from():
for case in harness.load_cases():
given = harness.inputs(case)
assert "scope" in given or "sources" in given, case["id"]
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
from execution_engine.scope import ESSENTIAL, read_scope
from tests.support import BUNDLE
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
scope = read_scope(
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
)
assert scope.gaps(ESSENTIAL) == []
assert scope.customer_case == "bench-landing-zone-shared"