Files
tom_analysis_agent/eval/operating_model_eval.py

209 lines
7.3 KiB
Python

"""Evaluate the target operating model agent on benchmark cases with real models.
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
and the agent's own Skills and role files), then scores it with eval/scoring.py.
It costs model usage: run it deliberately.
Usage:
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
.venv/bin/python -m eval.operating_model_eval [case-id ...]
The model settings are read from the environment or from `.env`. With no case named,
every case in eval/datasets/cases.json runs, one after another. Results are written
under eval/results/<time>/: each Execution's files, and results.json. The exit code
is non-zero when a case fails a check or runs over its time.
"""
from __future__ import annotations
import asyncio
import json
import logging
import sys
import time
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import yaml
from execution_engine.adk.runtime import AgentRuntime
from execution_engine.agent import open_assembly
from execution_engine.config import Settings, get_settings
from execution_engine.recorder import InMemoryRecorder
from app.operating_model import digest
from app.operating_model.workflow import ROOT, build
from eval.scoring import CaseResult, Check, score
HERE = Path(__file__).resolve().parent
CASES = HERE / "datasets" / "cases.json"
RESULTS = HERE / "results"
logger = logging.getLogger(__name__)
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
unknown = sorted(set(wanted) - {case["id"] for case in cases})
if unknown:
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
return [case for case in cases if not wanted or case["id"] in wanted]
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
"""What an Execution of the case is started with."""
given: dict[str, Any] = {}
if case.get("spec"):
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
if case.get("sources"):
given["sources"] = digest.read_folder(
(CASES.parent / case["sources"]).resolve()
)
if case.get("overlay"):
given["scope"] = yaml.safe_dump(case["overlay"])
return given
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
"""Tokens by role, when the runtime counts them."""
usage = getattr(runtime, "usage", None)
if not isinstance(usage, Mapping):
return {}
by_role: dict[str, dict[str, int]] = {}
for (role, _model), used in usage.items():
totals = by_role.setdefault(role, {})
for name, amount in vars(used).items():
if isinstance(amount, int):
totals[name] = totals.get(name, 0) + amount
return by_role
async def run_case(
case: Mapping[str, Any],
output_root: Path,
*,
settings: Settings | None = None,
runtime: AgentRuntime | None = None,
) -> CaseResult:
"""Run one case to the end and score it. `runtime` replaces the real one, for a
scripted run of the harness itself."""
settings = settings or get_settings()
workflow = build(ROOT)
started = time.monotonic()
async with open_assembly(
settings,
workflow,
root=ROOT,
runtime=runtime,
recorder=InMemoryRecorder(),
output_root=output_root,
) as assembly:
used = assembly.runtime
# A case is stopped at half again its time limit: a run that loops keeps
# spending until something stops it.
limit = float(case.get("max_minutes") or 60) * 1.5
try:
interaction = await asyncio.wait_for(
assembly.engine.start(
"", pre_approved=True, execution_id=case["id"], **inputs(case)
),
timeout=limit * 60,
)
except TimeoutError:
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
return CaseResult(
case=case["id"],
status="stopped",
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
minutes=round((time.monotonic() - started) / 60, 1),
tokens=_tokens(used),
)
except Exception as exc:
logger.exception("Case %s could not run", case["id"])
return CaseResult(
case=case["id"],
status="error",
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
minutes=round((time.monotonic() - started) / 60, 1),
)
minutes = round((time.monotonic() - started) / 60, 1)
state = interaction.state
checks, revisions, figures = score(
workflow,
state,
status=interaction.status,
max_revisions=int(case.get("max_revisions", 2)),
)
allowed = case.get("max_minutes")
if allowed:
checks.append(
Check(
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
)
)
return CaseResult(
case=case["id"],
status=interaction.status,
checks=checks,
minutes=minutes,
revisions=revisions,
tokens=_tokens(used),
figures=figures,
warnings=list(state.get("warnings") or []),
folder=str(
output_root
/ "deliverables"
/ "cases"
/ str(state.get("customer_case"))
/ "executions"
/ case["id"]
),
)
def report(results: Sequence[CaseResult]) -> str:
lines = []
for result in results:
lines.append(
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
f"{result.status}, {result.minutes} min"
)
lines += [
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
+ (f": {check.detail}" if check.detail else "")
for check in result.checks
]
if result.figures:
lines.append(
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
)
lines += [f" warning: {warning}" for warning in result.warnings]
return "\n".join(lines)
async def main(arguments: Sequence[str]) -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
cases = load_cases(arguments)
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
output_root = RESULTS / stamp
output_root.mkdir(parents=True, exist_ok=True)
# An evaluation keeps nothing between runs and reads nobody's cloud.
settings = get_settings().model_copy(
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
)
results = []
for case in cases:
logger.info("Running %s", case["id"])
results.append(await run_case(case, output_root, settings=settings))
(output_root / "results.json").write_text(
json.dumps([result.as_dict() for result in results], indent=2),
encoding="utf-8",
)
print(report(results))
print(f"\nWritten to {output_root}")
return 0 if all(result.passed for result in results) else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))