Files
tom_analysis_agent/eval/operating_model_eval.py
Jonathan Boniface 6fdac22876 feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:32:05 +01:00

357 lines
13 KiB
Python

"""Evaluate the target operating model agent on benchmark cases with real models.
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
and the agent's own Skills and role files), then scores it with eval/scoring.py.
It costs model usage: run it deliberately.
Usage:
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
.venv/bin/python -m eval.operating_model_eval [--repeat N] [case-id ...]
The model settings are read from the environment or from `.env`. With no case named,
every case in eval/datasets/cases.json runs, one after another; `--repeat N` runs each
N times, so the spread of what is staffed and priced can be seen. Results are written
under eval/results/<time>/: each Execution's files, results.json, spread.json, and
report.md, which holds no path of the machine it ran on and can be published. The
exit code is non-zero when a case fails a check or runs over its time.
"""
from __future__ import annotations
import asyncio
import json
import logging
import statistics
import sys
import time
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import yaml
from execution_engine.adk.runtime import AgentRuntime
from execution_engine.agent import open_assembly
from execution_engine.config import Settings, get_settings
from execution_engine.recorder import InMemoryRecorder
from app.operating_model import digest
from app.operating_model.workflow import ROOT, build
from eval.scoring import CaseResult, Check, score
HERE = Path(__file__).resolve().parent
CASES = HERE / "datasets" / "cases.json"
RESULTS = HERE / "results"
logger = logging.getLogger(__name__)
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
unknown = sorted(set(wanted) - {case["id"] for case in cases})
if unknown:
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
return [case for case in cases if not wanted or case["id"] in wanted]
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
"""What an Execution of the case is started with."""
given: dict[str, Any] = {}
if case.get("spec"):
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
if case.get("sources"):
given["sources"] = digest.read_folder(
(CASES.parent / case["sources"]).resolve()
)
if case.get("overlay"):
given["scope"] = yaml.safe_dump(case["overlay"])
return given
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
"""Tokens by role, when the runtime counts them."""
usage = getattr(runtime, "usage", None)
if not isinstance(usage, Mapping):
return {}
by_role: dict[str, dict[str, int]] = {}
for (role, _model), used in usage.items():
totals = by_role.setdefault(role, {})
for name, amount in vars(used).items():
if isinstance(amount, int):
totals[name] = totals.get(name, 0) + amount
return by_role
def execution_ids(case: Mapping[str, Any], repeat: int) -> list[str]:
"""The Execution of each run of a case: the case's id, or one per repeat."""
if repeat <= 1:
return [str(case["id"])]
return [f"{case['id']}-r{n}" for n in range(1, repeat + 1)]
async def run_case(
case: Mapping[str, Any],
output_root: Path,
*,
settings: Settings | None = None,
runtime: AgentRuntime | None = None,
execution_id: str | None = None,
) -> CaseResult:
"""Run one case to the end and score it. `runtime` replaces the real one, for a
scripted run of the harness itself."""
execution_id = execution_id or str(case["id"])
settings = settings or get_settings()
workflow = build(ROOT)
started = time.monotonic()
async with open_assembly(
settings,
workflow,
root=ROOT,
runtime=runtime,
recorder=InMemoryRecorder(),
output_root=output_root,
) as assembly:
used = assembly.runtime
# A case is stopped at half again its time limit: a run that loops keeps
# spending until something stops it.
limit = float(case.get("max_minutes") or 60) * 1.5
try:
interaction = await asyncio.wait_for(
assembly.engine.start(
"", pre_approved=True, execution_id=execution_id, **inputs(case)
),
timeout=limit * 60,
)
except TimeoutError:
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
return CaseResult(
case=case["id"],
status="stopped",
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
minutes=round((time.monotonic() - started) / 60, 1),
tokens=_tokens(used),
)
except Exception as exc:
logger.exception("Case %s could not run", case["id"])
return CaseResult(
case=case["id"],
status="error",
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
minutes=round((time.monotonic() - started) / 60, 1),
)
minutes = round((time.monotonic() - started) / 60, 1)
state = interaction.state
checks, revisions, figures = score(
workflow,
state,
status=interaction.status,
max_revisions=int(case.get("max_revisions", 2)),
)
allowed = case.get("max_minutes")
if allowed:
checks.append(
Check(
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
)
)
return CaseResult(
case=case["id"],
status=interaction.status,
checks=checks,
minutes=minutes,
revisions=revisions,
tokens=_tokens(used),
figures=figures,
warnings=list(state.get("warnings") or []),
# Relative to the run's folder: results are published, and the machine
# they ran on is no part of them.
folder=(
Path("deliverables")
/ "cases"
/ str(state.get("customer_case"))
/ "executions"
/ execution_id
).as_posix(),
)
# The figures whose spread across repeated runs is reported.
SPREAD = ("people", "cost_per_month", "roles", "checks", "capabilities")
def spread(results: Sequence[CaseResult]) -> dict[str, dict[str, Any]]:
"""For each case run more than once, how far its figures moved: least, most,
mean, and the range as a share of the mean."""
by_case: dict[str, list[CaseResult]] = {}
for result in results:
by_case.setdefault(result.case, []).append(result)
found: dict[str, dict[str, Any]] = {}
for case, runs in by_case.items():
if len(runs) < 2:
continue
figures: dict[str, Any] = {"runs": len(runs)}
for name in SPREAD:
values = [
r.figures[name]
for r in runs
if isinstance(r.figures.get(name), (int, float))
]
if len(values) < 2:
continue
mean = statistics.fmean(values)
figures[name] = {
"min": min(values),
"max": max(values),
"mean": round(mean, 1),
"range_pct": round((max(values) - min(values)) / mean * 100, 1)
if mean
else 0.0,
}
found[case] = figures
return found
def markdown(
results: Sequence[CaseResult], spreads: Mapping[str, Mapping[str, Any]], stamp: str
) -> str:
"""The run, as a report that can be published."""
lines = [
"# Target operating model agent: evaluation",
"",
(
f"Run `{stamp}`. Each run is one pre-approved Execution with real models "
"on a benchmark case. Every case is fictional; no customer's data is in it."
),
"",
"## Results",
"",
(
"| Case | Run | Result | Status | Minutes | Revisions "
"| Requirements verified | Roles | People | Cost a month |"
),
"|---|---|---|---|---|---|---|---|---|---|",
]
seen: dict[str, int] = {}
for result in results:
seen[result.case] = seen.get(result.case, 0) + 1
figures = result.figures
coverage = next(
(
c.detail.split(";")[0]
for c in result.checks
if c.name.startswith("every Operational")
),
"",
)
cost = figures.get("cost_per_month")
lines.append(
f"| {result.case} | {seen[result.case]} | "
f"{'PASS' if result.passed else 'FAIL'} | {result.status} | "
f"{result.minutes} | {sum(result.revisions.values())} | {coverage} | "
f"{figures.get('roles', '')} | {figures.get('people', '')} | "
+ (
f"{cost:,} {figures.get('currency', '')} |"
if isinstance(cost, (int, float))
else " |"
)
)
failed = [(r, c) for r in results for c in r.checks if not c.passed]
lines += ["", "## Checks that failed", ""]
if failed:
lines += [
f"- **{r.case}**: {c.name}" + (f" ({c.detail})" if c.detail else "")
for r, c in failed
]
else:
lines.append("None: every run passed every check.")
if spreads:
lines += [
"",
"## Spread across repeated runs",
"",
"The same case and inputs, run again: how far the operating model moved.",
"",
"| Case | Figure | Least | Most | Mean | Range of the mean |",
"|---|---|---|---|---|---|",
]
for case, measured in spreads.items():
for name in SPREAD:
if name in measured:
f = measured[name]
lines.append(
f"| {case} | {name} | {f['min']} | {f['max']} | {f['mean']} | "
f"{f['range_pct']}% |"
)
return "\n".join(lines) + "\n"
def parse_arguments(arguments: Sequence[str]) -> tuple[list[str], int]:
"""The cases named, and how many times each is to run."""
names, repeat, rest = [], 1, list(arguments)
while rest:
argument = rest.pop(0)
if argument == "--repeat":
repeat = int(rest.pop(0))
else:
names.append(argument)
return names, repeat
def report(results: Sequence[CaseResult]) -> str:
lines = []
for result in results:
lines.append(
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
f"{result.status}, {result.minutes} min"
)
lines += [
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
+ (f": {check.detail}" if check.detail else "")
for check in result.checks
]
if result.figures:
lines.append(
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
)
lines += [f" warning: {warning}" for warning in result.warnings]
return "\n".join(lines)
async def main(arguments: Sequence[str]) -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
wanted, repeat = parse_arguments(arguments)
cases = load_cases(wanted)
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
output_root = RESULTS / stamp
output_root.mkdir(parents=True, exist_ok=True)
# An evaluation keeps nothing between runs and reads nobody's cloud.
settings = get_settings().model_copy(
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
)
results = []
for case in cases:
for execution_id in execution_ids(case, repeat):
logger.info("Running %s as %s", case["id"], execution_id)
results.append(
await run_case(
case, output_root, settings=settings, execution_id=execution_id
)
)
spreads = spread(results)
(output_root / "results.json").write_text(
json.dumps([result.as_dict() for result in results], indent=2),
encoding="utf-8",
)
(output_root / "spread.json").write_text(
json.dumps(spreads, indent=2), encoding="utf-8"
)
(output_root / "report.md").write_text(
markdown(results, spreads, stamp), encoding="utf-8"
)
print(report(results))
print(f"\nWritten to {output_root}")
return 0 if all(result.passed for result in results) else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))