feat: every Operational requirement verified by a Check; eval spread and report

The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-21 21:32:05 +01:00
parent efe543e6e2
commit 6fdac22876
12 changed files with 356 additions and 23 deletions

View File

@@ -6,12 +6,14 @@ It costs model usage: run it deliberately.
Usage:
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
.venv/bin/python -m eval.operating_model_eval [case-id ...]
.venv/bin/python -m eval.operating_model_eval [--repeat N] [case-id ...]
The model settings are read from the environment or from `.env`. With no case named,
every case in eval/datasets/cases.json runs, one after another. Results are written
under eval/results/<time>/: each Execution's files, and results.json. The exit code
is non-zero when a case fails a check or runs over its time.
every case in eval/datasets/cases.json runs, one after another; `--repeat N` runs each
N times, so the spread of what is staffed and priced can be seen. Results are written
under eval/results/<time>/: each Execution's files, results.json, spread.json, and
report.md, which holds no path of the machine it ran on and can be published. The
exit code is non-zero when a case fails a check or runs over its time.
"""
from __future__ import annotations
@@ -19,6 +21,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import statistics
import sys
import time
from collections.abc import Mapping, Sequence
@@ -78,15 +81,24 @@ def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
return by_role
def execution_ids(case: Mapping[str, Any], repeat: int) -> list[str]:
"""The Execution of each run of a case: the case's id, or one per repeat."""
if repeat <= 1:
return [str(case["id"])]
return [f"{case['id']}-r{n}" for n in range(1, repeat + 1)]
async def run_case(
case: Mapping[str, Any],
output_root: Path,
*,
settings: Settings | None = None,
runtime: AgentRuntime | None = None,
execution_id: str | None = None,
) -> CaseResult:
"""Run one case to the end and score it. `runtime` replaces the real one, for a
scripted run of the harness itself."""
execution_id = execution_id or str(case["id"])
settings = settings or get_settings()
workflow = build(ROOT)
started = time.monotonic()
@@ -105,7 +117,7 @@ async def run_case(
try:
interaction = await asyncio.wait_for(
assembly.engine.start(
"", pre_approved=True, execution_id=case["id"], **inputs(case)
"", pre_approved=True, execution_id=execution_id, **inputs(case)
),
timeout=limit * 60,
)
@@ -150,17 +162,140 @@ async def run_case(
tokens=_tokens(used),
figures=figures,
warnings=list(state.get("warnings") or []),
folder=str(
output_root
/ "deliverables"
# Relative to the run's folder: results are published, and the machine
# they ran on is no part of them.
folder=(
Path("deliverables")
/ "cases"
/ str(state.get("customer_case"))
/ "executions"
/ case["id"]
),
/ execution_id
).as_posix(),
)
# The figures whose spread across repeated runs is reported.
SPREAD = ("people", "cost_per_month", "roles", "checks", "capabilities")
def spread(results: Sequence[CaseResult]) -> dict[str, dict[str, Any]]:
"""For each case run more than once, how far its figures moved: least, most,
mean, and the range as a share of the mean."""
by_case: dict[str, list[CaseResult]] = {}
for result in results:
by_case.setdefault(result.case, []).append(result)
found: dict[str, dict[str, Any]] = {}
for case, runs in by_case.items():
if len(runs) < 2:
continue
figures: dict[str, Any] = {"runs": len(runs)}
for name in SPREAD:
values = [
r.figures[name]
for r in runs
if isinstance(r.figures.get(name), (int, float))
]
if len(values) < 2:
continue
mean = statistics.fmean(values)
figures[name] = {
"min": min(values),
"max": max(values),
"mean": round(mean, 1),
"range_pct": round((max(values) - min(values)) / mean * 100, 1)
if mean
else 0.0,
}
found[case] = figures
return found
def markdown(
results: Sequence[CaseResult], spreads: Mapping[str, Mapping[str, Any]], stamp: str
) -> str:
"""The run, as a report that can be published."""
lines = [
"# Target operating model agent: evaluation",
"",
(
f"Run `{stamp}`. Each run is one pre-approved Execution with real models "
"on a benchmark case. Every case is fictional; no customer's data is in it."
),
"",
"## Results",
"",
(
"| Case | Run | Result | Status | Minutes | Revisions "
"| Requirements verified | Roles | People | Cost a month |"
),
"|---|---|---|---|---|---|---|---|---|---|",
]
seen: dict[str, int] = {}
for result in results:
seen[result.case] = seen.get(result.case, 0) + 1
figures = result.figures
coverage = next(
(
c.detail.split(";")[0]
for c in result.checks
if c.name.startswith("every Operational")
),
"",
)
cost = figures.get("cost_per_month")
lines.append(
f"| {result.case} | {seen[result.case]} | "
f"{'PASS' if result.passed else 'FAIL'} | {result.status} | "
f"{result.minutes} | {sum(result.revisions.values())} | {coverage} | "
f"{figures.get('roles', '')} | {figures.get('people', '')} | "
+ (
f"{cost:,} {figures.get('currency', '')} |"
if isinstance(cost, (int, float))
else " |"
)
)
failed = [(r, c) for r in results for c in r.checks if not c.passed]
lines += ["", "## Checks that failed", ""]
if failed:
lines += [
f"- **{r.case}**: {c.name}" + (f" ({c.detail})" if c.detail else "")
for r, c in failed
]
else:
lines.append("None: every run passed every check.")
if spreads:
lines += [
"",
"## Spread across repeated runs",
"",
"The same case and inputs, run again: how far the operating model moved.",
"",
"| Case | Figure | Least | Most | Mean | Range of the mean |",
"|---|---|---|---|---|---|",
]
for case, measured in spreads.items():
for name in SPREAD:
if name in measured:
f = measured[name]
lines.append(
f"| {case} | {name} | {f['min']} | {f['max']} | {f['mean']} | "
f"{f['range_pct']}% |"
)
return "\n".join(lines) + "\n"
def parse_arguments(arguments: Sequence[str]) -> tuple[list[str], int]:
"""The cases named, and how many times each is to run."""
names, repeat, rest = [], 1, list(arguments)
while rest:
argument = rest.pop(0)
if argument == "--repeat":
repeat = int(rest.pop(0))
else:
names.append(argument)
return names, repeat
def report(results: Sequence[CaseResult]) -> str:
lines = []
for result in results:
@@ -183,7 +318,8 @@ def report(results: Sequence[CaseResult]) -> str:
async def main(arguments: Sequence[str]) -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
cases = load_cases(arguments)
wanted, repeat = parse_arguments(arguments)
cases = load_cases(wanted)
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
output_root = RESULTS / stamp
output_root.mkdir(parents=True, exist_ok=True)
@@ -193,12 +329,24 @@ async def main(arguments: Sequence[str]) -> int:
)
results = []
for case in cases:
logger.info("Running %s", case["id"])
results.append(await run_case(case, output_root, settings=settings))
(output_root / "results.json").write_text(
json.dumps([result.as_dict() for result in results], indent=2),
encoding="utf-8",
)
for execution_id in execution_ids(case, repeat):
logger.info("Running %s as %s", case["id"], execution_id)
results.append(
await run_case(
case, output_root, settings=settings, execution_id=execution_id
)
)
spreads = spread(results)
(output_root / "results.json").write_text(
json.dumps([result.as_dict() for result in results], indent=2),
encoding="utf-8",
)
(output_root / "spread.json").write_text(
json.dumps(spreads, indent=2), encoding="utf-8"
)
(output_root / "report.md").write_text(
markdown(results, spreads, stamp), encoding="utf-8"
)
print(report(results))
print(f"\nWritten to {output_root}")
return 0 if all(result.passed for result in results) else 1

View File

@@ -9,7 +9,7 @@ against one, and is not built.
from __future__ import annotations
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from dataclasses import asdict, dataclass, field
from typing import Any
@@ -61,6 +61,26 @@ def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
return None
def unverified(
requirements: Sequence[Mapping[str, Any]], model: Mapping[str, Any]
) -> list[str]:
"""The Operational requirements no Check verifies: none has the requirement, or
a Condition it comes from, as its source."""
carried = {str(check.get("source")) for check in model.get("checks") or []}
missing = []
for requirement in requirements:
named = requirement.get("source") or []
named = [named] if isinstance(named, str) else named
# A Check's source is a requirement or a Condition (the graph refuses
# anything else), so a source of the requirement that a Check carries is a
# Condition it comes from.
if str(requirement.get("id")) not in carried and not any(
str(source) in carried for source in named
):
missing.append(str(requirement.get("id")))
return missing
def score(
workflow: Workflow,
state: Mapping[str, Any],
@@ -155,6 +175,21 @@ def score(
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
or {}
)
requirements = (
_attachment(
deliverables, "operational_requirements", "operational-requirements.json"
)
or []
)
missing = unverified(requirements, model)
checks.append(
Check(
"every Operational requirement is verified by a Check",
bool(requirements) and not missing,
f"{len(requirements) - len(missing)} of {len(requirements)}"
+ (f"; not verified: {', '.join(missing)}" if missing else ""),
)
)
figures = {
"items": len(spec.items(scope)),
"conditions": len(spec.conditions(scope)),
@@ -178,6 +213,7 @@ def score(
"parties": len(model.get("parties") or []),
"roles": len(model.get("roles") or []),
"checks": len(own),
"requirements_unverified": len(missing),
"checks_not_verifiable": sum(
1 for v in own if v.get("verdict") == "not_verifiable"
),