70 lines
2.3 KiB
Python
70 lines
2.3 KiB
Python
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
|
|
|
|
import json
|
|
|
|
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
|
from execution_engine.config import Settings
|
|
|
|
from eval import operating_model_eval as harness
|
|
from tests.support import SPEC
|
|
from tests.test_execution import turns
|
|
|
|
|
|
def settings() -> Settings:
|
|
return Settings( # type: ignore[call-arg]
|
|
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
|
|
)
|
|
|
|
|
|
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
|
|
|
|
|
|
async def run(tmp_path, scripted, monkeypatch):
|
|
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
|
|
return await harness.run_case(
|
|
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
|
|
)
|
|
|
|
|
|
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
|
|
result = await run(tmp_path, turns(), monkeypatch)
|
|
|
|
assert result.status == "completed"
|
|
failed = [check for check in result.checks if not check.passed]
|
|
# The scripted packager writes one word; a real one must write a summary.
|
|
assert [check.name for check in failed] == ["the packager wrote a summary"]
|
|
assert result.figures["cost_per_month"] == 43410
|
|
assert result.figures["scenarios"] == 10
|
|
assert (tmp_path / "deliverables").is_dir()
|
|
assert json.dumps(result.as_dict()) # the whole result can be written down
|
|
|
|
|
|
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
|
|
scripted = turns()
|
|
scripted["validator"] = []
|
|
|
|
result = await run(tmp_path, scripted, monkeypatch)
|
|
|
|
assert not result.passed
|
|
assert result.status != "completed"
|
|
|
|
|
|
def test_every_case_names_something_to_start_from():
|
|
for case in harness.load_cases():
|
|
given = harness.inputs(case)
|
|
assert "scope" in given or "sources" in given, case["id"]
|
|
|
|
|
|
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
|
|
from execution_engine.scope import ESSENTIAL, read_scope
|
|
|
|
from tests.support import BUNDLE
|
|
|
|
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
|
|
scope = read_scope(
|
|
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
|
)
|
|
|
|
assert scope.gaps(ESSENTIAL) == []
|
|
assert scope.customer_case == "bench-landing-zone-shared"
|