218 lines
8.0 KiB
Python
218 lines
8.0 KiB
Python
"""Evaluate the solution architect agent on benchmark cases with real models.
|
|
|
|
Each case runs a pre-approved Execution with scripted user answers through the full
|
|
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
|
|
eval/scoring.py. Costs model usage: run deliberately.
|
|
|
|
Usage:
|
|
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
|
.venv/bin/python -m eval.solution_eval [case-id ...]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
from collections.abc import Sequence
|
|
from dataclasses import asdict
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
|
from app.agent import open_assembly
|
|
from app.config import Settings, get_settings
|
|
from app.recorder import InMemoryRecorder
|
|
from app.tools.local import TerraformRunner
|
|
from eval.scoring import (
|
|
CaseResult,
|
|
CaseSpec,
|
|
load_cases,
|
|
score_execution,
|
|
template_headings,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
|
RESULTS_DIR = ROOT / "results"
|
|
|
|
|
|
class CountingRuntime:
|
|
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
|
|
|
|
def __init__(self, inner: AgentRuntime) -> None:
|
|
self._inner = inner
|
|
self._started = time.monotonic()
|
|
self.runs: Counter[str] = Counter()
|
|
self.outcomes: Counter[str] = Counter()
|
|
self.timeline: list[dict[str, Any]] = []
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
self.runs[role.name] += 1
|
|
start = time.monotonic()
|
|
outcome = await self._inner.run(role, task, tools)
|
|
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
|
|
self.timeline.append(
|
|
{
|
|
"role": role.name,
|
|
"deliverable": task.deliverable,
|
|
"start_s": round(start - self._started, 1),
|
|
"seconds": round(time.monotonic() - start, 1),
|
|
"outcome": type(outcome).__name__,
|
|
}
|
|
)
|
|
return outcome
|
|
|
|
|
|
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
|
|
"""Seconds until the first validation_results run ended: the run without revisions."""
|
|
for run in timeline:
|
|
if run["deliverable"] == "validation_results":
|
|
return round(run["start_s"] + run["seconds"], 1)
|
|
return None
|
|
|
|
|
|
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
|
by_role = {
|
|
f"{role} ({model})": asdict(counts)
|
|
for (role, model), counts in sorted(usage.items())
|
|
}
|
|
by_model: dict[str, Counter[str]] = {}
|
|
for (_, model), counts in usage.items():
|
|
by_model.setdefault(model, Counter()).update(asdict(counts))
|
|
total: Counter[str] = Counter()
|
|
for counts in by_model.values():
|
|
total.update(counts)
|
|
return {
|
|
"by_role": by_role,
|
|
"by_model": {model: dict(c) for model, c in by_model.items()},
|
|
"total": dict(total),
|
|
}
|
|
|
|
|
|
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.providers import load_provider
|
|
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
adk_runtime = AdkAgentRuntime(
|
|
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
runtime = CountingRuntime(adk_runtime)
|
|
recorder = InMemoryRecorder()
|
|
started = time.monotonic()
|
|
async with open_assembly(
|
|
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
|
) as assembly:
|
|
interaction = await assembly.engine.start(
|
|
case.workflow_request,
|
|
pre_approved=True,
|
|
allow_cloud_read=False,
|
|
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
|
|
scripted_answers=case.scripted_answers,
|
|
)
|
|
elapsed = time.monotonic() - started
|
|
|
|
runner = TerraformRunner(
|
|
binary=settings.TERRAFORM_BINARY,
|
|
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
|
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
|
|
)
|
|
result = await score_execution(
|
|
case, interaction, template_headings(bundle.output_template), runner.validate
|
|
)
|
|
ledger = interaction.state.get("ledger", {})
|
|
details = {
|
|
"execution_id": interaction.execution_id,
|
|
"seconds": round(elapsed, 1),
|
|
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
|
"orchestrator_turns": interaction.state.get("turns"),
|
|
"tokens": token_report(adk_runtime.usage),
|
|
"timeline": runtime.timeline,
|
|
"agent_runs": dict(runtime.runs),
|
|
"outcomes": dict(runtime.outcomes),
|
|
"revisions": {
|
|
kind: item["revisions"]
|
|
for kind, item in ledger.get("deliverables", {}).items()
|
|
if item["revisions"]
|
|
},
|
|
"warnings": interaction.state.get("warnings", []),
|
|
"conversation": interaction.state.get("conversation", []),
|
|
"output_dir": str(
|
|
run_dir / "deliverables" / "executions" / interaction.execution_id
|
|
),
|
|
}
|
|
return CaseResult(result.case_id, result.status, result.checks, details)
|
|
|
|
|
|
async def main(case_ids: Sequence[str]) -> int:
|
|
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
|
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
|
|
)
|
|
logging.getLogger("app").setLevel(logging.INFO)
|
|
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
|
|
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
|
|
if not cases:
|
|
print(f"No cases match {list(case_ids)}", file=sys.stderr)
|
|
return 2
|
|
|
|
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
run_dir = RESULTS_DIR / stamp
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
results: list[dict[str, Any]] = []
|
|
for case in cases:
|
|
print(f"== {case.id}: running", flush=True)
|
|
try:
|
|
result = await run_case(case, settings, run_dir)
|
|
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
|
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
|
results.append(
|
|
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
|
|
)
|
|
continue
|
|
print(
|
|
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
|
|
flush=True,
|
|
)
|
|
for check in result.checks:
|
|
mark = "ok " if check.passed else "FAIL"
|
|
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
|
|
details = result.details
|
|
print(
|
|
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
|
|
f"revisions {details['revisions'] or 'none'}",
|
|
flush=True,
|
|
)
|
|
for model, counts in details["tokens"]["by_model"].items():
|
|
print(f" tokens {model}: {counts}", flush=True)
|
|
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
|
results.append(
|
|
{
|
|
"case_id": result.case_id,
|
|
"status": result.status,
|
|
"score": result.score,
|
|
"passed": result.passed,
|
|
"checks": [asdict(check) for check in result.checks],
|
|
"details": result.details,
|
|
}
|
|
)
|
|
|
|
report = run_dir / "report.json"
|
|
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
|
passed = sum(1 for r in results if r.get("passed"))
|
|
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
|
|
return 0 if passed == len(results) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|