feat: working TOM agent
This commit is contained in:
69
tests/test_eval.py
Normal file
69
tests/test_eval.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
||||
from execution_engine.config import Settings
|
||||
|
||||
from eval import operating_model_eval as harness
|
||||
from tests.support import SPEC
|
||||
from tests.test_execution import turns
|
||||
|
||||
|
||||
def settings() -> Settings:
|
||||
return Settings( # type: ignore[call-arg]
|
||||
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
|
||||
)
|
||||
|
||||
|
||||
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
|
||||
|
||||
|
||||
async def run(tmp_path, scripted, monkeypatch):
|
||||
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
|
||||
return await harness.run_case(
|
||||
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
|
||||
)
|
||||
|
||||
|
||||
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
|
||||
result = await run(tmp_path, turns(), monkeypatch)
|
||||
|
||||
assert result.status == "completed"
|
||||
failed = [check for check in result.checks if not check.passed]
|
||||
# The scripted packager writes one word; a real one must write a summary.
|
||||
assert [check.name for check in failed] == ["the packager wrote a summary"]
|
||||
assert result.figures["cost_per_month"] == 43410
|
||||
assert result.figures["scenarios"] == 10
|
||||
assert (tmp_path / "deliverables").is_dir()
|
||||
assert json.dumps(result.as_dict()) # the whole result can be written down
|
||||
|
||||
|
||||
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
|
||||
scripted = turns()
|
||||
scripted["validator"] = []
|
||||
|
||||
result = await run(tmp_path, scripted, monkeypatch)
|
||||
|
||||
assert not result.passed
|
||||
assert result.status != "completed"
|
||||
|
||||
|
||||
def test_every_case_names_something_to_start_from():
|
||||
for case in harness.load_cases():
|
||||
given = harness.inputs(case)
|
||||
assert "scope" in given or "sources" in given, case["id"]
|
||||
|
||||
|
||||
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
|
||||
from execution_engine.scope import ESSENTIAL, read_scope
|
||||
|
||||
from tests.support import BUNDLE
|
||||
|
||||
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
|
||||
scope = read_scope(
|
||||
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
||||
)
|
||||
|
||||
assert scope.gaps(ESSENTIAL) == []
|
||||
assert scope.customer_case == "bench-landing-zone-shared"
|
||||
Reference in New Issue
Block a user