feat: every Operational requirement verified by a Check; eval spread and report

The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-21 21:32:05 +01:00
parent efe543e6e2
commit 6fdac22876
12 changed files with 356 additions and 23 deletions

View File

@@ -1,12 +1,15 @@
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
import asyncio
import json
from execution_engine.adk.fake_runtime import FakeAgentRuntime
from execution_engine.config import Settings
from eval import operating_model_eval as harness
from tests.support import SPEC
from eval import scoring
from eval.scoring import CaseResult, Check
from tests.support import GRAPH, REQUIREMENTS, SPEC
from tests.test_execution import turns
@@ -67,3 +70,75 @@ def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
assert scope.gaps(ESSENTIAL) == []
assert scope.customer_case == "bench-landing-zone-shared"
def test_a_sound_execution_verifies_every_requirement_and_is_kept_relatively(
tmp_path, monkeypatch
):
result = asyncio.run(run(tmp_path, turns(), monkeypatch))
coverage = next(c for c in result.checks if c.name.startswith("every Operational"))
assert coverage.passed, coverage.detail
assert result.figures["requirements_unverified"] == 0
# Results are published: no path of the machine they ran on.
assert (
result.folder == "deliverables/cases/northwind-landing-zone/executions/scripted"
)
assert str(tmp_path) not in json.dumps(result.as_dict())
def test_a_requirement_no_check_verifies_is_named():
model = {"checks": [c for c in GRAPH["checks"] if c["id"] != "OC-5"]}
assert scoring.unverified(REQUIREMENTS, model) == ["OR-4"]
assert scoring.unverified(REQUIREMENTS, GRAPH) == []
def result(case: str, people: float, cost: int, passed: bool = True) -> CaseResult:
return CaseResult(
case=case,
status="completed",
checks=[Check("completed", passed)],
figures={
"people": people,
"cost_per_month": cost,
"roles": 5,
"checks": 9,
"capabilities": 17,
"currency": "GBP",
},
)
def test_the_spread_of_repeated_runs_is_reported():
spread = harness.spread(
[result("a", 10.0, 50000), result("a", 12.0, 70000), result("b", 5.0, 1)]
)
assert set(spread) == {"a"} # one run has no spread
assert spread["a"]["runs"] == 2
assert spread["a"]["cost_per_month"] == {
"min": 50000,
"max": 70000,
"mean": 60000.0,
"range_pct": 33.3,
}
assert spread["a"]["people"]["range_pct"] == 18.2
def test_the_report_can_be_published():
results = [result("a", 10.0, 50000), result("a", 12.0, 70000, passed=False)]
text = harness.markdown(results, harness.spread(results), "20261021T120000Z")
assert text.startswith("# Target operating model agent: evaluation")
assert "| a | 1 | PASS |" in text and "| a | 2 | FAIL |" in text
assert "Spread across repeated runs" in text
assert "/home/" not in text
def test_runs_are_repeated_under_their_own_ids():
assert harness.parse_arguments(["--repeat", "3", "case-x"]) == (["case-x"], 3)
assert harness.parse_arguments([]) == ([], 1)
assert harness.execution_ids({"id": "case-x"}, 1) == ["case-x"]
assert harness.execution_ids({"id": "case-x"}, 2) == ["case-x-r1", "case-x-r2"]