Files
gcp_solution_architecture_a…/eval/solution_eval.py
Jonathan Boniface 7ee49ab352
Some checks failed
validation / verify (push) Failing after 10s
latest refactoring
2026-10-06 20:22:37 +01:00

218 lines
8.0 KiB
Python

"""Evaluate the solution architect agent on benchmark cases with real models.
Each case runs a pre-approved Execution with scripted user answers through the full
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
eval/scoring.py. Costs model usage: run deliberately.
Usage:
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
.venv/bin/python -m eval.solution_eval [case-id ...]
"""
from __future__ import annotations
import asyncio
import json
import logging
import sys
import time
from collections import Counter
from collections.abc import Sequence
from dataclasses import asdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
from app.agent import open_assembly
from app.config import Settings, get_settings
from app.recorder import InMemoryRecorder
from app.tools.local import TerraformRunner
from eval.scoring import (
CaseResult,
CaseSpec,
load_cases,
score_execution,
template_headings,
)
ROOT = Path(__file__).resolve().parent
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
RESULTS_DIR = ROOT / "results"
class CountingRuntime:
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
def __init__(self, inner: AgentRuntime) -> None:
self._inner = inner
self._started = time.monotonic()
self.runs: Counter[str] = Counter()
self.outcomes: Counter[str] = Counter()
self.timeline: list[dict[str, Any]] = []
async def run(
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
) -> Outcome:
self.runs[role.name] += 1
start = time.monotonic()
outcome = await self._inner.run(role, task, tools)
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
self.timeline.append(
{
"role": role.name,
"deliverable": task.deliverable,
"start_s": round(start - self._started, 1),
"seconds": round(time.monotonic() - start, 1),
"outcome": type(outcome).__name__,
}
)
return outcome
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
"""Seconds until the first validation_results run ended: the run without revisions."""
for run in timeline:
if run["deliverable"] == "validation_results":
return round(run["start_s"] + run["seconds"], 1)
return None
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
by_role = {
f"{role} ({model})": asdict(counts)
for (role, model), counts in sorted(usage.items())
}
by_model: dict[str, Counter[str]] = {}
for (_, model), counts in usage.items():
by_model.setdefault(model, Counter()).update(asdict(counts))
total: Counter[str] = Counter()
for counts in by_model.values():
total.update(counts)
return {
"by_role": by_role,
"by_model": {model: dict(c) for model, c in by_model.items()},
"total": dict(total),
}
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
from app.adk.adk_runtime import AdkAgentRuntime
from app.providers import load_provider
bundle = load_provider(settings.CLOUD_PROVIDER)
adk_runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
runtime = CountingRuntime(adk_runtime)
recorder = InMemoryRecorder()
started = time.monotonic()
async with open_assembly(
settings, runtime=runtime, recorder=recorder, output_root=run_dir
) as assembly:
interaction = await assembly.engine.start(
case.workflow_request,
pre_approved=True,
allow_cloud_read=False,
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
scripted_answers=case.scripted_answers,
)
elapsed = time.monotonic() - started
runner = TerraformRunner(
binary=settings.TERRAFORM_BINARY,
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
)
result = await score_execution(
case, interaction, template_headings(bundle.output_template), runner.validate
)
ledger = interaction.state.get("ledger", {})
details = {
"execution_id": interaction.execution_id,
"seconds": round(elapsed, 1),
"first_pass_seconds": first_pass_seconds(runtime.timeline),
"orchestrator_turns": interaction.state.get("turns"),
"tokens": token_report(adk_runtime.usage),
"timeline": runtime.timeline,
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"revisions": {
kind: item["revisions"]
for kind, item in ledger.get("deliverables", {}).items()
if item["revisions"]
},
"warnings": interaction.state.get("warnings", []),
"conversation": interaction.state.get("conversation", []),
"output_dir": str(
run_dir / "deliverables" / "executions" / interaction.execution_id
),
}
return CaseResult(result.case_id, result.status, result.checks, details)
async def main(case_ids: Sequence[str]) -> int:
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
logging.basicConfig(
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
)
logging.getLogger("app").setLevel(logging.INFO)
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
if not cases:
print(f"No cases match {list(case_ids)}", file=sys.stderr)
return 2
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
run_dir = RESULTS_DIR / stamp
run_dir.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
for case in cases:
print(f"== {case.id}: running", flush=True)
try:
result = await run_case(case, settings, run_dir)
except Exception as exc: # noqa: BLE001 - record the failure and continue
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
results.append(
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
)
continue
print(
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
flush=True,
)
for check in result.checks:
mark = "ok " if check.passed else "FAIL"
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
details = result.details
print(
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
f"revisions {details['revisions'] or 'none'}",
flush=True,
)
for model, counts in details["tokens"]["by_model"].items():
print(f" tokens {model}: {counts}", flush=True)
print(f" tokens total: {details['tokens']['total']}", flush=True)
results.append(
{
"case_id": result.case_id,
"status": result.status,
"score": result.score,
"passed": result.passed,
"checks": [asdict(check) for check in result.checks],
"details": result.details,
}
)
report = run_dir / "report.json"
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
passed = sum(1 for r in results if r.get("passed"))
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
return 0 if passed == len(results) else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))