209 lines
7.3 KiB
Python
209 lines
7.3 KiB
Python
"""Evaluate the target operating model agent on benchmark cases with real models.
|
|
|
|
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
|
|
and the agent's own Skills and role files), then scores it with eval/scoring.py.
|
|
It costs model usage: run it deliberately.
|
|
|
|
Usage:
|
|
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
|
|
.venv/bin/python -m eval.operating_model_eval [case-id ...]
|
|
|
|
The model settings are read from the environment or from `.env`. With no case named,
|
|
every case in eval/datasets/cases.json runs, one after another. Results are written
|
|
under eval/results/<time>/: each Execution's files, and results.json. The exit code
|
|
is non-zero when a case fails a check or runs over its time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import sys
|
|
import time
|
|
from collections.abc import Mapping, Sequence
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
from execution_engine.adk.runtime import AgentRuntime
|
|
from execution_engine.agent import open_assembly
|
|
from execution_engine.config import Settings, get_settings
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
|
|
from app.operating_model import digest
|
|
from app.operating_model.workflow import ROOT, build
|
|
from eval.scoring import CaseResult, Check, score
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
CASES = HERE / "datasets" / "cases.json"
|
|
RESULTS = HERE / "results"
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
|
|
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
|
|
unknown = sorted(set(wanted) - {case["id"] for case in cases})
|
|
if unknown:
|
|
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
|
|
return [case for case in cases if not wanted or case["id"] in wanted]
|
|
|
|
|
|
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""What an Execution of the case is started with."""
|
|
given: dict[str, Any] = {}
|
|
if case.get("spec"):
|
|
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
|
|
if case.get("sources"):
|
|
given["sources"] = digest.read_folder(
|
|
(CASES.parent / case["sources"]).resolve()
|
|
)
|
|
if case.get("overlay"):
|
|
given["scope"] = yaml.safe_dump(case["overlay"])
|
|
return given
|
|
|
|
|
|
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
|
|
"""Tokens by role, when the runtime counts them."""
|
|
usage = getattr(runtime, "usage", None)
|
|
if not isinstance(usage, Mapping):
|
|
return {}
|
|
by_role: dict[str, dict[str, int]] = {}
|
|
for (role, _model), used in usage.items():
|
|
totals = by_role.setdefault(role, {})
|
|
for name, amount in vars(used).items():
|
|
if isinstance(amount, int):
|
|
totals[name] = totals.get(name, 0) + amount
|
|
return by_role
|
|
|
|
|
|
async def run_case(
|
|
case: Mapping[str, Any],
|
|
output_root: Path,
|
|
*,
|
|
settings: Settings | None = None,
|
|
runtime: AgentRuntime | None = None,
|
|
) -> CaseResult:
|
|
"""Run one case to the end and score it. `runtime` replaces the real one, for a
|
|
scripted run of the harness itself."""
|
|
settings = settings or get_settings()
|
|
workflow = build(ROOT)
|
|
started = time.monotonic()
|
|
async with open_assembly(
|
|
settings,
|
|
workflow,
|
|
root=ROOT,
|
|
runtime=runtime,
|
|
recorder=InMemoryRecorder(),
|
|
output_root=output_root,
|
|
) as assembly:
|
|
used = assembly.runtime
|
|
# A case is stopped at half again its time limit: a run that loops keeps
|
|
# spending until something stops it.
|
|
limit = float(case.get("max_minutes") or 60) * 1.5
|
|
try:
|
|
interaction = await asyncio.wait_for(
|
|
assembly.engine.start(
|
|
"", pre_approved=True, execution_id=case["id"], **inputs(case)
|
|
),
|
|
timeout=limit * 60,
|
|
)
|
|
except TimeoutError:
|
|
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status="stopped",
|
|
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
|
|
minutes=round((time.monotonic() - started) / 60, 1),
|
|
tokens=_tokens(used),
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("Case %s could not run", case["id"])
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status="error",
|
|
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
|
|
minutes=round((time.monotonic() - started) / 60, 1),
|
|
)
|
|
minutes = round((time.monotonic() - started) / 60, 1)
|
|
state = interaction.state
|
|
checks, revisions, figures = score(
|
|
workflow,
|
|
state,
|
|
status=interaction.status,
|
|
max_revisions=int(case.get("max_revisions", 2)),
|
|
)
|
|
allowed = case.get("max_minutes")
|
|
if allowed:
|
|
checks.append(
|
|
Check(
|
|
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
|
|
)
|
|
)
|
|
return CaseResult(
|
|
case=case["id"],
|
|
status=interaction.status,
|
|
checks=checks,
|
|
minutes=minutes,
|
|
revisions=revisions,
|
|
tokens=_tokens(used),
|
|
figures=figures,
|
|
warnings=list(state.get("warnings") or []),
|
|
folder=str(
|
|
output_root
|
|
/ "deliverables"
|
|
/ "cases"
|
|
/ str(state.get("customer_case"))
|
|
/ "executions"
|
|
/ case["id"]
|
|
),
|
|
)
|
|
|
|
|
|
def report(results: Sequence[CaseResult]) -> str:
|
|
lines = []
|
|
for result in results:
|
|
lines.append(
|
|
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
|
|
f"{result.status}, {result.minutes} min"
|
|
)
|
|
lines += [
|
|
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
|
|
+ (f": {check.detail}" if check.detail else "")
|
|
for check in result.checks
|
|
]
|
|
if result.figures:
|
|
lines.append(
|
|
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
|
|
)
|
|
lines += [f" warning: {warning}" for warning in result.warnings]
|
|
return "\n".join(lines)
|
|
|
|
|
|
async def main(arguments: Sequence[str]) -> int:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
|
|
cases = load_cases(arguments)
|
|
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
output_root = RESULTS / stamp
|
|
output_root.mkdir(parents=True, exist_ok=True)
|
|
# An evaluation keeps nothing between runs and reads nobody's cloud.
|
|
settings = get_settings().model_copy(
|
|
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
|
|
)
|
|
results = []
|
|
for case in cases:
|
|
logger.info("Running %s", case["id"])
|
|
results.append(await run_case(case, output_root, settings=settings))
|
|
(output_root / "results.json").write_text(
|
|
json.dumps([result.as_dict() for result in results], indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
print(report(results))
|
|
print(f"\nWritten to {output_root}")
|
|
return 0 if all(result.passed for result in results) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|