feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -232,6 +232,18 @@ GRAPH: dict[str, list[dict[str, Any]]] = {
|
||||
"source": "OR-2",
|
||||
"condition": "No change to the network is applied without the platform owner's approval.",
|
||||
},
|
||||
{
|
||||
"id": "OC-5",
|
||||
"on": "C-1",
|
||||
"source": "OR-4",
|
||||
"condition": "A recovery of the hub network is rehearsed each quarter within 4 hours.",
|
||||
},
|
||||
{
|
||||
"id": "OC-6",
|
||||
"on": "C-3",
|
||||
"source": "OR-5",
|
||||
"condition": "Every Role is placed in a country the customer allows.",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
||||
from execution_engine.config import Settings
|
||||
|
||||
from eval import operating_model_eval as harness
|
||||
from tests.support import SPEC
|
||||
from eval import scoring
|
||||
from eval.scoring import CaseResult, Check
|
||||
from tests.support import GRAPH, REQUIREMENTS, SPEC
|
||||
from tests.test_execution import turns
|
||||
|
||||
|
||||
@@ -67,3 +70,75 @@ def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
|
||||
|
||||
assert scope.gaps(ESSENTIAL) == []
|
||||
assert scope.customer_case == "bench-landing-zone-shared"
|
||||
|
||||
|
||||
def test_a_sound_execution_verifies_every_requirement_and_is_kept_relatively(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
result = asyncio.run(run(tmp_path, turns(), monkeypatch))
|
||||
|
||||
coverage = next(c for c in result.checks if c.name.startswith("every Operational"))
|
||||
assert coverage.passed, coverage.detail
|
||||
assert result.figures["requirements_unverified"] == 0
|
||||
# Results are published: no path of the machine they ran on.
|
||||
assert (
|
||||
result.folder == "deliverables/cases/northwind-landing-zone/executions/scripted"
|
||||
)
|
||||
assert str(tmp_path) not in json.dumps(result.as_dict())
|
||||
|
||||
|
||||
def test_a_requirement_no_check_verifies_is_named():
|
||||
model = {"checks": [c for c in GRAPH["checks"] if c["id"] != "OC-5"]}
|
||||
|
||||
assert scoring.unverified(REQUIREMENTS, model) == ["OR-4"]
|
||||
assert scoring.unverified(REQUIREMENTS, GRAPH) == []
|
||||
|
||||
|
||||
def result(case: str, people: float, cost: int, passed: bool = True) -> CaseResult:
|
||||
return CaseResult(
|
||||
case=case,
|
||||
status="completed",
|
||||
checks=[Check("completed", passed)],
|
||||
figures={
|
||||
"people": people,
|
||||
"cost_per_month": cost,
|
||||
"roles": 5,
|
||||
"checks": 9,
|
||||
"capabilities": 17,
|
||||
"currency": "GBP",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_the_spread_of_repeated_runs_is_reported():
|
||||
spread = harness.spread(
|
||||
[result("a", 10.0, 50000), result("a", 12.0, 70000), result("b", 5.0, 1)]
|
||||
)
|
||||
|
||||
assert set(spread) == {"a"} # one run has no spread
|
||||
assert spread["a"]["runs"] == 2
|
||||
assert spread["a"]["cost_per_month"] == {
|
||||
"min": 50000,
|
||||
"max": 70000,
|
||||
"mean": 60000.0,
|
||||
"range_pct": 33.3,
|
||||
}
|
||||
assert spread["a"]["people"]["range_pct"] == 18.2
|
||||
|
||||
|
||||
def test_the_report_can_be_published():
|
||||
results = [result("a", 10.0, 50000), result("a", 12.0, 70000, passed=False)]
|
||||
|
||||
text = harness.markdown(results, harness.spread(results), "20261021T120000Z")
|
||||
|
||||
assert text.startswith("# Target operating model agent: evaluation")
|
||||
assert "| a | 1 | PASS |" in text and "| a | 2 | FAIL |" in text
|
||||
assert "Spread across repeated runs" in text
|
||||
assert "/home/" not in text
|
||||
|
||||
|
||||
def test_runs_are_repeated_under_their_own_ids():
|
||||
assert harness.parse_arguments(["--repeat", "3", "case-x"]) == (["case-x"], 3)
|
||||
assert harness.parse_arguments([]) == ([], 1)
|
||||
assert harness.execution_ids({"id": "case-x"}, 1) == ["case-x"]
|
||||
assert harness.execution_ids({"id": "case-x"}, 2) == ["case-x-r1", "case-x-r2"]
|
||||
|
||||
@@ -92,6 +92,10 @@ def test_a_graph_that_holds_together_is_accepted():
|
||||
"Capability C-1 is needed 24x7, and no Role responsible for it works those hours",
|
||||
),
|
||||
(lambda g: g["checks"].pop(1), "Condition K-1 is carried by no Check"),
|
||||
(
|
||||
lambda g: g["checks"].pop(0),
|
||||
"Operational requirement OR-1 is verified by no Check",
|
||||
),
|
||||
(
|
||||
lambda g: g["checks"][0].update(on="C-9"),
|
||||
"Check OC-1 is on C-9, which is not",
|
||||
@@ -119,6 +123,14 @@ def test_a_graph_that_does_not_hold_together_is_refused(change, problem):
|
||||
assert any(problem in fault for fault in found), found
|
||||
|
||||
|
||||
def test_a_requirement_is_verified_through_the_conditions_it_comes_from():
|
||||
# OR-3 comes from K-1 and K-2: the Checks carrying them verify it.
|
||||
changed = graph()
|
||||
assert not any(c["source"] == "OR-3" for c in changed["checks"])
|
||||
|
||||
assert not any("OR-3" in fault for fault in faults(changed))
|
||||
|
||||
|
||||
def test_a_file_that_is_not_a_graph_is_named():
|
||||
assert "not valid JSON" in check({GRAPH_FILE: "{"}, upstream())[0]
|
||||
assert "must be an object with" in check({GRAPH_FILE: "[]"}, upstream())[0]
|
||||
|
||||
@@ -133,7 +133,7 @@ def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code():
|
||||
"OC-3",
|
||||
"OC-4",
|
||||
]
|
||||
assert len(merged["checks"]) == 14
|
||||
assert len(merged["checks"]) == 16
|
||||
# Merging again replaces the scenarios' verdicts; it never adds them twice.
|
||||
again = json.loads(scenarios.merge(json.dumps(merged), found))
|
||||
assert again == merged
|
||||
@@ -165,7 +165,7 @@ def test_a_verdict_on_every_check_is_accepted():
|
||||
def test_a_check_without_a_verdict_is_refused():
|
||||
missing = {"checks": VERDICTS["checks"][:-1]}
|
||||
|
||||
assert verdict_faults(missing) == ["verdicts.json: Check OC-4 has no verdict"]
|
||||
assert verdict_faults(missing) == ["verdicts.json: Check OC-6 has no verdict"]
|
||||
|
||||
|
||||
def test_a_check_that_waits_on_a_finding_cannot_pass():
|
||||
|
||||
Reference in New Issue
Block a user