The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
357 lines
13 KiB
Python
357 lines
13 KiB
Python
"""Evaluate the target operating model agent on benchmark cases with real models.
|
|
|
|
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
|
|
and the agent's own Skills and role files), then scores it with eval/scoring.py.
|
|
It costs model usage: run it deliberately.
|
|
|
|
Usage:
|
|
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
|
|
.venv/bin/python -m eval.operating_model_eval [--repeat N] [case-id ...]
|
|
|
|
The model settings are read from the environment or from `.env`. With no case named,
|
|
every case in eval/datasets/cases.json runs, one after another; `--repeat N` runs each
|
|
N times, so the spread of what is staffed and priced can be seen. Results are written
|
|
under eval/results/<time>/: each Execution's files, results.json, spread.json, and
|
|
report.md, which holds no path of the machine it ran on and can be published. The
|
|
exit code is non-zero when a case fails a check or runs over its time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import statistics
|
|
import sys
|
|
import time
|
|
from collections.abc import Mapping, Sequence
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
from execution_engine.adk.runtime import AgentRuntime
|
|
from execution_engine.agent import open_assembly
|
|
from execution_engine.config import Settings, get_settings
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
|
|
from app.operating_model import digest
|
|
from app.operating_model.workflow import ROOT, build
|
|
from eval.scoring import CaseResult, Check, score
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
CASES = HERE / "datasets" / "cases.json"
|
|
RESULTS = HERE / "results"
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
|
|
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
|
|
unknown = sorted(set(wanted) - {case["id"] for case in cases})
|
|
if unknown:
|
|
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
|
|
return [case for case in cases if not wanted or case["id"] in wanted]
|
|
|
|
|
|
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""What an Execution of the case is started with."""
|
|
given: dict[str, Any] = {}
|
|
if case.get("spec"):
|
|
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
|
|
if case.get("sources"):
|
|
given["sources"] = digest.read_folder(
|
|
(CASES.parent / case["sources"]).resolve()
|
|
)
|
|
if case.get("overlay"):
|
|
given["scope"] = yaml.safe_dump(case["overlay"])
|
|
return given
|
|
|
|
|
|
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
|
|
"""Tokens by role, when the runtime counts them."""
|
|
usage = getattr(runtime, "usage", None)
|
|
if not isinstance(usage, Mapping):
|
|
return {}
|
|
by_role: dict[str, dict[str, int]] = {}
|
|
for (role, _model), used in usage.items():
|
|
totals = by_role.setdefault(role, {})
|
|
for name, amount in vars(used).items():
|
|
if isinstance(amount, int):
|
|
totals[name] = totals.get(name, 0) + amount
|
|
return by_role
|
|
|
|
|
|
def execution_ids(case: Mapping[str, Any], repeat: int) -> list[str]:
|
|
"""The Execution of each run of a case: the case's id, or one per repeat."""
|
|
if repeat <= 1:
|
|
return [str(case["id"])]
|
|
return [f"{case['id']}-r{n}" for n in range(1, repeat + 1)]
|
|
|
|
|
|
async def run_case(
|
|
case: Mapping[str, Any],
|
|
output_root: Path,
|
|
*,
|
|
settings: Settings | None = None,
|
|
runtime: AgentRuntime | None = None,
|
|
execution_id: str | None = None,
|
|
) -> CaseResult:
|
|
"""Run one case to the end and score it. `runtime` replaces the real one, for a
|
|
scripted run of the harness itself."""
|
|
execution_id = execution_id or str(case["id"])
|
|
settings = settings or get_settings()
|
|
workflow = build(ROOT)
|
|
started = time.monotonic()
|
|
async with open_assembly(
|
|
settings,
|
|
workflow,
|
|
root=ROOT,
|
|
runtime=runtime,
|
|
recorder=InMemoryRecorder(),
|
|
output_root=output_root,
|
|
) as assembly:
|
|
used = assembly.runtime
|
|
# A case is stopped at half again its time limit: a run that loops keeps
|
|
# spending until something stops it.
|
|
limit = float(case.get("max_minutes") or 60) * 1.5
|
|
try:
|
|
interaction = await asyncio.wait_for(
|
|
assembly.engine.start(
|
|
"", pre_approved=True, execution_id=execution_id, **inputs(case)
|
|
),
|
|
timeout=limit * 60,
|
|
)
|
|
except TimeoutError:
|
|
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status="stopped",
|
|
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
|
|
minutes=round((time.monotonic() - started) / 60, 1),
|
|
tokens=_tokens(used),
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("Case %s could not run", case["id"])
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status="error",
|
|
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
|
|
minutes=round((time.monotonic() - started) / 60, 1),
|
|
)
|
|
minutes = round((time.monotonic() - started) / 60, 1)
|
|
state = interaction.state
|
|
checks, revisions, figures = score(
|
|
workflow,
|
|
state,
|
|
status=interaction.status,
|
|
max_revisions=int(case.get("max_revisions", 2)),
|
|
)
|
|
allowed = case.get("max_minutes")
|
|
if allowed:
|
|
checks.append(
|
|
Check(
|
|
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
|
|
)
|
|
)
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status=interaction.status,
|
|
checks=checks,
|
|
minutes=minutes,
|
|
revisions=revisions,
|
|
tokens=_tokens(used),
|
|
figures=figures,
|
|
warnings=list(state.get("warnings") or []),
|
|
# Relative to the run's folder: results are published, and the machine
|
|
# they ran on is no part of them.
|
|
folder=(
|
|
Path("deliverables")
|
|
/ "cases"
|
|
/ str(state.get("customer_case"))
|
|
/ "executions"
|
|
/ execution_id
|
|
).as_posix(),
|
|
)
|
|
|
|
|
|
# The figures whose spread across repeated runs is reported.
|
|
SPREAD = ("people", "cost_per_month", "roles", "checks", "capabilities")
|
|
|
|
|
|
def spread(results: Sequence[CaseResult]) -> dict[str, dict[str, Any]]:
|
|
"""For each case run more than once, how far its figures moved: least, most,
|
|
mean, and the range as a share of the mean."""
|
|
by_case: dict[str, list[CaseResult]] = {}
|
|
for result in results:
|
|
by_case.setdefault(result.case, []).append(result)
|
|
found: dict[str, dict[str, Any]] = {}
|
|
for case, runs in by_case.items():
|
|
if len(runs) < 2:
|
|
continue
|
|
figures: dict[str, Any] = {"runs": len(runs)}
|
|
for name in SPREAD:
|
|
values = [
|
|
r.figures[name]
|
|
for r in runs
|
|
if isinstance(r.figures.get(name), (int, float))
|
|
]
|
|
if len(values) < 2:
|
|
continue
|
|
mean = statistics.fmean(values)
|
|
figures[name] = {
|
|
"min": min(values),
|
|
"max": max(values),
|
|
"mean": round(mean, 1),
|
|
"range_pct": round((max(values) - min(values)) / mean * 100, 1)
|
|
if mean
|
|
else 0.0,
|
|
}
|
|
found[case] = figures
|
|
return found
|
|
|
|
|
|
def markdown(
|
|
results: Sequence[CaseResult], spreads: Mapping[str, Mapping[str, Any]], stamp: str
|
|
) -> str:
|
|
"""The run, as a report that can be published."""
|
|
lines = [
|
|
"# Target operating model agent: evaluation",
|
|
"",
|
|
(
|
|
f"Run `{stamp}`. Each run is one pre-approved Execution with real models "
|
|
"on a benchmark case. Every case is fictional; no customer's data is in it."
|
|
),
|
|
"",
|
|
"## Results",
|
|
"",
|
|
(
|
|
"| Case | Run | Result | Status | Minutes | Revisions "
|
|
"| Requirements verified | Roles | People | Cost a month |"
|
|
),
|
|
"|---|---|---|---|---|---|---|---|---|---|",
|
|
]
|
|
seen: dict[str, int] = {}
|
|
for result in results:
|
|
seen[result.case] = seen.get(result.case, 0) + 1
|
|
figures = result.figures
|
|
coverage = next(
|
|
(
|
|
c.detail.split(";")[0]
|
|
for c in result.checks
|
|
if c.name.startswith("every Operational")
|
|
),
|
|
"",
|
|
)
|
|
cost = figures.get("cost_per_month")
|
|
lines.append(
|
|
f"| {result.case} | {seen[result.case]} | "
|
|
f"{'PASS' if result.passed else 'FAIL'} | {result.status} | "
|
|
f"{result.minutes} | {sum(result.revisions.values())} | {coverage} | "
|
|
f"{figures.get('roles', '')} | {figures.get('people', '')} | "
|
|
+ (
|
|
f"{cost:,} {figures.get('currency', '')} |"
|
|
if isinstance(cost, (int, float))
|
|
else " |"
|
|
)
|
|
)
|
|
failed = [(r, c) for r in results for c in r.checks if not c.passed]
|
|
lines += ["", "## Checks that failed", ""]
|
|
if failed:
|
|
lines += [
|
|
f"- **{r.case}**: {c.name}" + (f" ({c.detail})" if c.detail else "")
|
|
for r, c in failed
|
|
]
|
|
else:
|
|
lines.append("None: every run passed every check.")
|
|
if spreads:
|
|
lines += [
|
|
"",
|
|
"## Spread across repeated runs",
|
|
"",
|
|
"The same case and inputs, run again: how far the operating model moved.",
|
|
"",
|
|
"| Case | Figure | Least | Most | Mean | Range of the mean |",
|
|
"|---|---|---|---|---|---|",
|
|
]
|
|
for case, measured in spreads.items():
|
|
for name in SPREAD:
|
|
if name in measured:
|
|
f = measured[name]
|
|
lines.append(
|
|
f"| {case} | {name} | {f['min']} | {f['max']} | {f['mean']} | "
|
|
f"{f['range_pct']}% |"
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def parse_arguments(arguments: Sequence[str]) -> tuple[list[str], int]:
|
|
"""The cases named, and how many times each is to run."""
|
|
names, repeat, rest = [], 1, list(arguments)
|
|
while rest:
|
|
argument = rest.pop(0)
|
|
if argument == "--repeat":
|
|
repeat = int(rest.pop(0))
|
|
else:
|
|
names.append(argument)
|
|
return names, repeat
|
|
|
|
|
|
def report(results: Sequence[CaseResult]) -> str:
|
|
lines = []
|
|
for result in results:
|
|
lines.append(
|
|
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
|
|
f"{result.status}, {result.minutes} min"
|
|
)
|
|
lines += [
|
|
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
|
|
+ (f": {check.detail}" if check.detail else "")
|
|
for check in result.checks
|
|
]
|
|
if result.figures:
|
|
lines.append(
|
|
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
|
|
)
|
|
lines += [f" warning: {warning}" for warning in result.warnings]
|
|
return "\n".join(lines)
|
|
|
|
|
|
async def main(arguments: Sequence[str]) -> int:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
|
|
wanted, repeat = parse_arguments(arguments)
|
|
cases = load_cases(wanted)
|
|
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
output_root = RESULTS / stamp
|
|
output_root.mkdir(parents=True, exist_ok=True)
|
|
# An evaluation keeps nothing between runs and reads nobody's cloud.
|
|
settings = get_settings().model_copy(
|
|
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
|
|
)
|
|
results = []
|
|
for case in cases:
|
|
for execution_id in execution_ids(case, repeat):
|
|
logger.info("Running %s as %s", case["id"], execution_id)
|
|
results.append(
|
|
await run_case(
|
|
case, output_root, settings=settings, execution_id=execution_id
|
|
)
|
|
)
|
|
spreads = spread(results)
|
|
(output_root / "results.json").write_text(
|
|
json.dumps([result.as_dict() for result in results], indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
(output_root / "spread.json").write_text(
|
|
json.dumps(spreads, indent=2), encoding="utf-8"
|
|
)
|
|
(output_root / "report.md").write_text(
|
|
markdown(results, spreads, stamp), encoding="utf-8"
|
|
)
|
|
print(report(results))
|
|
print(f"\nWritten to {output_root}")
|
|
return 0 if all(result.passed for result in results) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|