feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,12 +6,14 @@ It costs model usage: run it deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
|
||||
.venv/bin/python -m eval.operating_model_eval [case-id ...]
|
||||
.venv/bin/python -m eval.operating_model_eval [--repeat N] [case-id ...]
|
||||
|
||||
The model settings are read from the environment or from `.env`. With no case named,
|
||||
every case in eval/datasets/cases.json runs, one after another. Results are written
|
||||
under eval/results/<time>/: each Execution's files, and results.json. The exit code
|
||||
is non-zero when a case fails a check or runs over its time.
|
||||
every case in eval/datasets/cases.json runs, one after another; `--repeat N` runs each
|
||||
N times, so the spread of what is staffed and priced can be seen. Results are written
|
||||
under eval/results/<time>/: each Execution's files, results.json, spread.json, and
|
||||
report.md, which holds no path of the machine it ran on and can be published. The
|
||||
exit code is non-zero when a case fails a check or runs over its time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,6 +21,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -78,15 +81,24 @@ def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
|
||||
return by_role
|
||||
|
||||
|
||||
def execution_ids(case: Mapping[str, Any], repeat: int) -> list[str]:
|
||||
"""The Execution of each run of a case: the case's id, or one per repeat."""
|
||||
if repeat <= 1:
|
||||
return [str(case["id"])]
|
||||
return [f"{case['id']}-r{n}" for n in range(1, repeat + 1)]
|
||||
|
||||
|
||||
async def run_case(
|
||||
case: Mapping[str, Any],
|
||||
output_root: Path,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
runtime: AgentRuntime | None = None,
|
||||
execution_id: str | None = None,
|
||||
) -> CaseResult:
|
||||
"""Run one case to the end and score it. `runtime` replaces the real one, for a
|
||||
scripted run of the harness itself."""
|
||||
execution_id = execution_id or str(case["id"])
|
||||
settings = settings or get_settings()
|
||||
workflow = build(ROOT)
|
||||
started = time.monotonic()
|
||||
@@ -105,7 +117,7 @@ async def run_case(
|
||||
try:
|
||||
interaction = await asyncio.wait_for(
|
||||
assembly.engine.start(
|
||||
"", pre_approved=True, execution_id=case["id"], **inputs(case)
|
||||
"", pre_approved=True, execution_id=execution_id, **inputs(case)
|
||||
),
|
||||
timeout=limit * 60,
|
||||
)
|
||||
@@ -150,17 +162,140 @@ async def run_case(
|
||||
tokens=_tokens(used),
|
||||
figures=figures,
|
||||
warnings=list(state.get("warnings") or []),
|
||||
folder=str(
|
||||
output_root
|
||||
/ "deliverables"
|
||||
# Relative to the run's folder: results are published, and the machine
|
||||
# they ran on is no part of them.
|
||||
folder=(
|
||||
Path("deliverables")
|
||||
/ "cases"
|
||||
/ str(state.get("customer_case"))
|
||||
/ "executions"
|
||||
/ case["id"]
|
||||
),
|
||||
/ execution_id
|
||||
).as_posix(),
|
||||
)
|
||||
|
||||
|
||||
# The figures whose spread across repeated runs is reported.
|
||||
SPREAD = ("people", "cost_per_month", "roles", "checks", "capabilities")
|
||||
|
||||
|
||||
def spread(results: Sequence[CaseResult]) -> dict[str, dict[str, Any]]:
|
||||
"""For each case run more than once, how far its figures moved: least, most,
|
||||
mean, and the range as a share of the mean."""
|
||||
by_case: dict[str, list[CaseResult]] = {}
|
||||
for result in results:
|
||||
by_case.setdefault(result.case, []).append(result)
|
||||
found: dict[str, dict[str, Any]] = {}
|
||||
for case, runs in by_case.items():
|
||||
if len(runs) < 2:
|
||||
continue
|
||||
figures: dict[str, Any] = {"runs": len(runs)}
|
||||
for name in SPREAD:
|
||||
values = [
|
||||
r.figures[name]
|
||||
for r in runs
|
||||
if isinstance(r.figures.get(name), (int, float))
|
||||
]
|
||||
if len(values) < 2:
|
||||
continue
|
||||
mean = statistics.fmean(values)
|
||||
figures[name] = {
|
||||
"min": min(values),
|
||||
"max": max(values),
|
||||
"mean": round(mean, 1),
|
||||
"range_pct": round((max(values) - min(values)) / mean * 100, 1)
|
||||
if mean
|
||||
else 0.0,
|
||||
}
|
||||
found[case] = figures
|
||||
return found
|
||||
|
||||
|
||||
def markdown(
|
||||
results: Sequence[CaseResult], spreads: Mapping[str, Mapping[str, Any]], stamp: str
|
||||
) -> str:
|
||||
"""The run, as a report that can be published."""
|
||||
lines = [
|
||||
"# Target operating model agent: evaluation",
|
||||
"",
|
||||
(
|
||||
f"Run `{stamp}`. Each run is one pre-approved Execution with real models "
|
||||
"on a benchmark case. Every case is fictional; no customer's data is in it."
|
||||
),
|
||||
"",
|
||||
"## Results",
|
||||
"",
|
||||
(
|
||||
"| Case | Run | Result | Status | Minutes | Revisions "
|
||||
"| Requirements verified | Roles | People | Cost a month |"
|
||||
),
|
||||
"|---|---|---|---|---|---|---|---|---|---|",
|
||||
]
|
||||
seen: dict[str, int] = {}
|
||||
for result in results:
|
||||
seen[result.case] = seen.get(result.case, 0) + 1
|
||||
figures = result.figures
|
||||
coverage = next(
|
||||
(
|
||||
c.detail.split(";")[0]
|
||||
for c in result.checks
|
||||
if c.name.startswith("every Operational")
|
||||
),
|
||||
"",
|
||||
)
|
||||
cost = figures.get("cost_per_month")
|
||||
lines.append(
|
||||
f"| {result.case} | {seen[result.case]} | "
|
||||
f"{'PASS' if result.passed else 'FAIL'} | {result.status} | "
|
||||
f"{result.minutes} | {sum(result.revisions.values())} | {coverage} | "
|
||||
f"{figures.get('roles', '')} | {figures.get('people', '')} | "
|
||||
+ (
|
||||
f"{cost:,} {figures.get('currency', '')} |"
|
||||
if isinstance(cost, (int, float))
|
||||
else " |"
|
||||
)
|
||||
)
|
||||
failed = [(r, c) for r in results for c in r.checks if not c.passed]
|
||||
lines += ["", "## Checks that failed", ""]
|
||||
if failed:
|
||||
lines += [
|
||||
f"- **{r.case}**: {c.name}" + (f" ({c.detail})" if c.detail else "")
|
||||
for r, c in failed
|
||||
]
|
||||
else:
|
||||
lines.append("None: every run passed every check.")
|
||||
if spreads:
|
||||
lines += [
|
||||
"",
|
||||
"## Spread across repeated runs",
|
||||
"",
|
||||
"The same case and inputs, run again: how far the operating model moved.",
|
||||
"",
|
||||
"| Case | Figure | Least | Most | Mean | Range of the mean |",
|
||||
"|---|---|---|---|---|---|",
|
||||
]
|
||||
for case, measured in spreads.items():
|
||||
for name in SPREAD:
|
||||
if name in measured:
|
||||
f = measured[name]
|
||||
lines.append(
|
||||
f"| {case} | {name} | {f['min']} | {f['max']} | {f['mean']} | "
|
||||
f"{f['range_pct']}% |"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def parse_arguments(arguments: Sequence[str]) -> tuple[list[str], int]:
|
||||
"""The cases named, and how many times each is to run."""
|
||||
names, repeat, rest = [], 1, list(arguments)
|
||||
while rest:
|
||||
argument = rest.pop(0)
|
||||
if argument == "--repeat":
|
||||
repeat = int(rest.pop(0))
|
||||
else:
|
||||
names.append(argument)
|
||||
return names, repeat
|
||||
|
||||
|
||||
def report(results: Sequence[CaseResult]) -> str:
|
||||
lines = []
|
||||
for result in results:
|
||||
@@ -183,7 +318,8 @@ def report(results: Sequence[CaseResult]) -> str:
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
|
||||
cases = load_cases(arguments)
|
||||
wanted, repeat = parse_arguments(arguments)
|
||||
cases = load_cases(wanted)
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
output_root = RESULTS / stamp
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
@@ -193,12 +329,24 @@ async def main(arguments: Sequence[str]) -> int:
|
||||
)
|
||||
results = []
|
||||
for case in cases:
|
||||
logger.info("Running %s", case["id"])
|
||||
results.append(await run_case(case, output_root, settings=settings))
|
||||
(output_root / "results.json").write_text(
|
||||
json.dumps([result.as_dict() for result in results], indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
for execution_id in execution_ids(case, repeat):
|
||||
logger.info("Running %s as %s", case["id"], execution_id)
|
||||
results.append(
|
||||
await run_case(
|
||||
case, output_root, settings=settings, execution_id=execution_id
|
||||
)
|
||||
)
|
||||
spreads = spread(results)
|
||||
(output_root / "results.json").write_text(
|
||||
json.dumps([result.as_dict() for result in results], indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output_root / "spread.json").write_text(
|
||||
json.dumps(spreads, indent=2), encoding="utf-8"
|
||||
)
|
||||
(output_root / "report.md").write_text(
|
||||
markdown(results, spreads, stamp), encoding="utf-8"
|
||||
)
|
||||
print(report(results))
|
||||
print(f"\nWritten to {output_root}")
|
||||
return 0 if all(result.passed for result in results) else 1
|
||||
|
||||
Reference in New Issue
Block a user