The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
145 lines
4.9 KiB
Python
145 lines
4.9 KiB
Python
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
|
from execution_engine.config import Settings
|
|
|
|
from eval import operating_model_eval as harness
|
|
from eval import scoring
|
|
from eval.scoring import CaseResult, Check
|
|
from tests.support import GRAPH, REQUIREMENTS, SPEC
|
|
from tests.test_execution import turns
|
|
|
|
|
|
def settings() -> Settings:
|
|
return Settings( # type: ignore[call-arg]
|
|
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
|
|
)
|
|
|
|
|
|
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
|
|
|
|
|
|
async def run(tmp_path, scripted, monkeypatch):
|
|
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
|
|
return await harness.run_case(
|
|
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
|
|
)
|
|
|
|
|
|
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
|
|
result = await run(tmp_path, turns(), monkeypatch)
|
|
|
|
assert result.status == "completed"
|
|
failed = [check for check in result.checks if not check.passed]
|
|
# The scripted packager writes one word; a real one must write a summary.
|
|
assert [check.name for check in failed] == ["the packager wrote a summary"]
|
|
assert result.figures["cost_per_month"] == 43410
|
|
assert result.figures["scenarios"] == 10
|
|
assert (tmp_path / "deliverables").is_dir()
|
|
assert json.dumps(result.as_dict()) # the whole result can be written down
|
|
|
|
|
|
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
|
|
scripted = turns()
|
|
scripted["validator"] = []
|
|
|
|
result = await run(tmp_path, scripted, monkeypatch)
|
|
|
|
assert not result.passed
|
|
assert result.status != "completed"
|
|
|
|
|
|
def test_every_case_names_something_to_start_from():
|
|
for case in harness.load_cases():
|
|
given = harness.inputs(case)
|
|
assert "scope" in given or "sources" in given, case["id"]
|
|
|
|
|
|
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
|
|
from execution_engine.scope import ESSENTIAL, read_scope
|
|
|
|
from tests.support import BUNDLE
|
|
|
|
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
|
|
scope = read_scope(
|
|
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
|
)
|
|
|
|
assert scope.gaps(ESSENTIAL) == []
|
|
assert scope.customer_case == "bench-landing-zone-shared"
|
|
|
|
|
|
def test_a_sound_execution_verifies_every_requirement_and_is_kept_relatively(
|
|
tmp_path, monkeypatch
|
|
):
|
|
result = asyncio.run(run(tmp_path, turns(), monkeypatch))
|
|
|
|
coverage = next(c for c in result.checks if c.name.startswith("every Operational"))
|
|
assert coverage.passed, coverage.detail
|
|
assert result.figures["requirements_unverified"] == 0
|
|
# Results are published: no path of the machine they ran on.
|
|
assert (
|
|
result.folder == "deliverables/cases/northwind-landing-zone/executions/scripted"
|
|
)
|
|
assert str(tmp_path) not in json.dumps(result.as_dict())
|
|
|
|
|
|
def test_a_requirement_no_check_verifies_is_named():
|
|
model = {"checks": [c for c in GRAPH["checks"] if c["id"] != "OC-5"]}
|
|
|
|
assert scoring.unverified(REQUIREMENTS, model) == ["OR-4"]
|
|
assert scoring.unverified(REQUIREMENTS, GRAPH) == []
|
|
|
|
|
|
def result(case: str, people: float, cost: int, passed: bool = True) -> CaseResult:
|
|
return CaseResult(
|
|
case=case,
|
|
status="completed",
|
|
checks=[Check("completed", passed)],
|
|
figures={
|
|
"people": people,
|
|
"cost_per_month": cost,
|
|
"roles": 5,
|
|
"checks": 9,
|
|
"capabilities": 17,
|
|
"currency": "GBP",
|
|
},
|
|
)
|
|
|
|
|
|
def test_the_spread_of_repeated_runs_is_reported():
|
|
spread = harness.spread(
|
|
[result("a", 10.0, 50000), result("a", 12.0, 70000), result("b", 5.0, 1)]
|
|
)
|
|
|
|
assert set(spread) == {"a"} # one run has no spread
|
|
assert spread["a"]["runs"] == 2
|
|
assert spread["a"]["cost_per_month"] == {
|
|
"min": 50000,
|
|
"max": 70000,
|
|
"mean": 60000.0,
|
|
"range_pct": 33.3,
|
|
}
|
|
assert spread["a"]["people"]["range_pct"] == 18.2
|
|
|
|
|
|
def test_the_report_can_be_published():
|
|
results = [result("a", 10.0, 50000), result("a", 12.0, 70000, passed=False)]
|
|
|
|
text = harness.markdown(results, harness.spread(results), "20261021T120000Z")
|
|
|
|
assert text.startswith("# Target operating model agent: evaluation")
|
|
assert "| a | 1 | PASS |" in text and "| a | 2 | FAIL |" in text
|
|
assert "Spread across repeated runs" in text
|
|
assert "/home/" not in text
|
|
|
|
|
|
def test_runs_are_repeated_under_their_own_ids():
|
|
assert harness.parse_arguments(["--repeat", "3", "case-x"]) == (["case-x"], 3)
|
|
assert harness.parse_arguments([]) == ([], 1)
|
|
assert harness.execution_ids({"id": "case-x"}, 1) == ["case-x"]
|
|
assert harness.execution_ids({"id": "case-x"}, 2) == ["case-x-r1", "case-x-r2"]
|