feat: working TOM agent
This commit is contained in:
208
eval/operating_model_eval.py
Normal file
208
eval/operating_model_eval.py
Normal file
@@ -0,0 +1,208 @@
|
||||
"""Evaluate the target operating model agent on benchmark cases with real models.
|
||||
|
||||
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
|
||||
and the agent's own Skills and role files), then scores it with eval/scoring.py.
|
||||
It costs model usage: run it deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
|
||||
.venv/bin/python -m eval.operating_model_eval [case-id ...]
|
||||
|
||||
The model settings are read from the environment or from `.env`. With no case named,
|
||||
every case in eval/datasets/cases.json runs, one after another. Results are written
|
||||
under eval/results/<time>/: each Execution's files, and results.json. The exit code
|
||||
is non-zero when a case fails a check or runs over its time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from execution_engine.adk.runtime import AgentRuntime
|
||||
from execution_engine.agent import open_assembly
|
||||
from execution_engine.config import Settings, get_settings
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
|
||||
from app.operating_model import digest
|
||||
from app.operating_model.workflow import ROOT, build
|
||||
from eval.scoring import CaseResult, Check, score
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
CASES = HERE / "datasets" / "cases.json"
|
||||
RESULTS = HERE / "results"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
|
||||
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
|
||||
unknown = sorted(set(wanted) - {case["id"] for case in cases})
|
||||
if unknown:
|
||||
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
|
||||
return [case for case in cases if not wanted or case["id"] in wanted]
|
||||
|
||||
|
||||
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""What an Execution of the case is started with."""
|
||||
given: dict[str, Any] = {}
|
||||
if case.get("spec"):
|
||||
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
|
||||
if case.get("sources"):
|
||||
given["sources"] = digest.read_folder(
|
||||
(CASES.parent / case["sources"]).resolve()
|
||||
)
|
||||
if case.get("overlay"):
|
||||
given["scope"] = yaml.safe_dump(case["overlay"])
|
||||
return given
|
||||
|
||||
|
||||
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
|
||||
"""Tokens by role, when the runtime counts them."""
|
||||
usage = getattr(runtime, "usage", None)
|
||||
if not isinstance(usage, Mapping):
|
||||
return {}
|
||||
by_role: dict[str, dict[str, int]] = {}
|
||||
for (role, _model), used in usage.items():
|
||||
totals = by_role.setdefault(role, {})
|
||||
for name, amount in vars(used).items():
|
||||
if isinstance(amount, int):
|
||||
totals[name] = totals.get(name, 0) + amount
|
||||
return by_role
|
||||
|
||||
|
||||
async def run_case(
|
||||
case: Mapping[str, Any],
|
||||
output_root: Path,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
runtime: AgentRuntime | None = None,
|
||||
) -> CaseResult:
|
||||
"""Run one case to the end and score it. `runtime` replaces the real one, for a
|
||||
scripted run of the harness itself."""
|
||||
settings = settings or get_settings()
|
||||
workflow = build(ROOT)
|
||||
started = time.monotonic()
|
||||
async with open_assembly(
|
||||
settings,
|
||||
workflow,
|
||||
root=ROOT,
|
||||
runtime=runtime,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=output_root,
|
||||
) as assembly:
|
||||
used = assembly.runtime
|
||||
# A case is stopped at half again its time limit: a run that loops keeps
|
||||
# spending until something stops it.
|
||||
limit = float(case.get("max_minutes") or 60) * 1.5
|
||||
try:
|
||||
interaction = await asyncio.wait_for(
|
||||
assembly.engine.start(
|
||||
"", pre_approved=True, execution_id=case["id"], **inputs(case)
|
||||
),
|
||||
timeout=limit * 60,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status="stopped",
|
||||
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
|
||||
minutes=round((time.monotonic() - started) / 60, 1),
|
||||
tokens=_tokens(used),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Case %s could not run", case["id"])
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status="error",
|
||||
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
|
||||
minutes=round((time.monotonic() - started) / 60, 1),
|
||||
)
|
||||
minutes = round((time.monotonic() - started) / 60, 1)
|
||||
state = interaction.state
|
||||
checks, revisions, figures = score(
|
||||
workflow,
|
||||
state,
|
||||
status=interaction.status,
|
||||
max_revisions=int(case.get("max_revisions", 2)),
|
||||
)
|
||||
allowed = case.get("max_minutes")
|
||||
if allowed:
|
||||
checks.append(
|
||||
Check(
|
||||
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
|
||||
)
|
||||
)
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status=interaction.status,
|
||||
checks=checks,
|
||||
minutes=minutes,
|
||||
revisions=revisions,
|
||||
tokens=_tokens(used),
|
||||
figures=figures,
|
||||
warnings=list(state.get("warnings") or []),
|
||||
folder=str(
|
||||
output_root
|
||||
/ "deliverables"
|
||||
/ "cases"
|
||||
/ str(state.get("customer_case"))
|
||||
/ "executions"
|
||||
/ case["id"]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def report(results: Sequence[CaseResult]) -> str:
|
||||
lines = []
|
||||
for result in results:
|
||||
lines.append(
|
||||
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
|
||||
f"{result.status}, {result.minutes} min"
|
||||
)
|
||||
lines += [
|
||||
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
|
||||
+ (f": {check.detail}" if check.detail else "")
|
||||
for check in result.checks
|
||||
]
|
||||
if result.figures:
|
||||
lines.append(
|
||||
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
|
||||
)
|
||||
lines += [f" warning: {warning}" for warning in result.warnings]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
|
||||
cases = load_cases(arguments)
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
output_root = RESULTS / stamp
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
# An evaluation keeps nothing between runs and reads nobody's cloud.
|
||||
settings = get_settings().model_copy(
|
||||
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
|
||||
)
|
||||
results = []
|
||||
for case in cases:
|
||||
logger.info("Running %s", case["id"])
|
||||
results.append(await run_case(case, output_root, settings=settings))
|
||||
(output_root / "results.json").write_text(
|
||||
json.dumps([result.as_dict() for result in results], indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(report(results))
|
||||
print(f"\nWritten to {output_root}")
|
||||
return 0 if all(result.passed for result in results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main(sys.argv[1:])))
|
||||
Reference in New Issue
Block a user