Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
342 lines
13 KiB
Python
342 lines
13 KiB
Python
"""Evaluate the solution architect agent on benchmark cases with real models.
|
|
|
|
Each case runs a pre-approved Execution with scripted user answers through the full
|
|
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
|
|
eval/scoring.py. Costs model usage: run deliberately.
|
|
|
|
Usage:
|
|
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
|
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
|
|
|
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
|
|
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
|
|
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
|
|
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
|
non-zero when a case fails or exceeds one of its thresholds.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
from collections.abc import Callable, Sequence
|
|
from dataclasses import asdict
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
|
from app.agent import ExecutionEngine, Interaction, open_assembly
|
|
from app.config import Settings, get_settings
|
|
from app.nodes.execution_nodes import automatic_reply
|
|
from app.recorder import InMemoryRecorder
|
|
from app.states.execution_state import execution_dir
|
|
from app.tools.local import TerraformRunner
|
|
from eval.judge import DeciderJudge, GeminiJudge, Judge
|
|
from eval.scoring import (
|
|
CaseResult,
|
|
CaseSpec,
|
|
checklist_items,
|
|
load_cases,
|
|
regressions,
|
|
score_execution,
|
|
template_headings,
|
|
)
|
|
|
|
GEMINI_PREFIX = "gemini/"
|
|
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
|
|
ROOT = Path(__file__).resolve().parent
|
|
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
|
RESULTS_DIR = ROOT / "results"
|
|
|
|
|
|
class CountingRuntime:
|
|
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
|
|
|
|
def __init__(
|
|
self, inner: AgentRuntime, on_run: Callable[[], None] | None = None
|
|
) -> None:
|
|
self._inner = inner
|
|
# Called after every agent run, so a run cut short leaves a record behind.
|
|
self._on_run = on_run
|
|
self._started = time.monotonic()
|
|
self.runs: Counter[str] = Counter()
|
|
self.outcomes: Counter[str] = Counter()
|
|
self.timeline: list[dict[str, Any]] = []
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
self.runs[role.name] += 1
|
|
start = time.monotonic()
|
|
outcome = await self._inner.run(role, task, tools)
|
|
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
|
|
self.timeline.append(
|
|
{
|
|
"role": role.name,
|
|
"deliverable": task.deliverable,
|
|
"start_s": round(start - self._started, 1),
|
|
"seconds": round(time.monotonic() - start, 1),
|
|
"outcome": type(outcome).__name__,
|
|
}
|
|
)
|
|
if self._on_run:
|
|
self._on_run()
|
|
return outcome
|
|
|
|
|
|
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
|
|
"""Seconds until the first validation_results run ended: the run without revisions."""
|
|
for run in timeline:
|
|
if run["deliverable"] == "validation_results":
|
|
return round(run["start_s"] + run["seconds"], 1)
|
|
return None
|
|
|
|
|
|
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
|
by_role = {
|
|
f"{role} ({model})": asdict(counts)
|
|
for (role, model), counts in sorted(usage.items())
|
|
}
|
|
by_model: dict[str, Counter[str]] = {}
|
|
for (_, model), counts in usage.items():
|
|
by_model.setdefault(model, Counter()).update(asdict(counts))
|
|
total: Counter[str] = Counter()
|
|
for counts in by_model.values():
|
|
total.update(counts)
|
|
return {
|
|
"by_role": by_role,
|
|
"by_model": {model: dict(c) for model, c in by_model.items()},
|
|
"total": dict(total),
|
|
}
|
|
|
|
|
|
async def run_with_scripted_user(
|
|
engine: ExecutionEngine,
|
|
case: CaseSpec,
|
|
execution_id: str,
|
|
max_user_turns: int = MAX_USER_TURNS,
|
|
) -> Interaction:
|
|
"""Run an interactive Execution, answering each prompt as the case scripts it:
|
|
Approvals are granted, cloud reads denied, and questions answered by pattern."""
|
|
interaction = await engine.start(
|
|
case.workflow_request,
|
|
customer_case=case.id,
|
|
allow_cloud_read=False,
|
|
execution_id=execution_id,
|
|
)
|
|
for _ in range(max_user_turns):
|
|
if interaction.status != "awaiting_user" or interaction.prompt is None:
|
|
break
|
|
reply = automatic_reply(
|
|
interaction.prompt, [list(pair) for pair in case.scripted_answers]
|
|
)
|
|
interaction = await engine.resume(execution_id, reply)
|
|
return interaction
|
|
|
|
|
|
async def run_case(
|
|
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
|
|
) -> CaseResult:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.providers import load_provider
|
|
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
adk_runtime = AdkAgentRuntime(
|
|
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
started = time.monotonic()
|
|
progress_path = run_dir / f"{case.id}-progress.json"
|
|
|
|
def write_progress() -> None:
|
|
"""What the run has cost so far. Overwritten after every agent run and left
|
|
in place, so an interrupted run still has its time, tokens and timeline."""
|
|
progress_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"case_id": case.id,
|
|
"finished": False,
|
|
"seconds": round(time.monotonic() - started, 1),
|
|
"agent_runs": dict(runtime.runs),
|
|
"outcomes": dict(runtime.outcomes),
|
|
"tokens": token_report(adk_runtime.usage),
|
|
"timeline": runtime.timeline,
|
|
},
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
runtime = CountingRuntime(adk_runtime, on_run=write_progress)
|
|
recorder = InMemoryRecorder()
|
|
async with open_assembly(
|
|
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
|
) as assembly:
|
|
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
|
if case.interactive:
|
|
interaction = await run_with_scripted_user(
|
|
assembly.engine, case, execution_id
|
|
)
|
|
else:
|
|
interaction = await assembly.engine.start(
|
|
case.workflow_request,
|
|
customer_case=case.id,
|
|
pre_approved=True,
|
|
allow_cloud_read=False,
|
|
execution_id=execution_id,
|
|
scripted_answers=case.scripted_answers,
|
|
)
|
|
elapsed = time.monotonic() - started
|
|
write_progress()
|
|
progress_path.write_text(
|
|
progress_path.read_text(encoding="utf-8").replace(
|
|
'"finished": false', '"finished": true', 1
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
runner = TerraformRunner(
|
|
binary=settings.TERRAFORM_BINARY,
|
|
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
|
plugin_cache_dir=settings.terraform_plugin_cache(),
|
|
)
|
|
result = await score_execution(
|
|
case,
|
|
interaction,
|
|
template_headings(bundle.output_template),
|
|
runner.validate,
|
|
judge=judge,
|
|
pillar_checklists={
|
|
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
|
|
for pillar, skill in bundle.pillar_skills.items()
|
|
},
|
|
timeline=runtime.timeline,
|
|
)
|
|
ledger = interaction.state.get("ledger", {})
|
|
tokens = token_report(adk_runtime.usage)
|
|
revisions = {
|
|
kind: item["revisions"]
|
|
for kind, item in ledger.get("deliverables", {}).items()
|
|
if item["revisions"]
|
|
}
|
|
details = {
|
|
"execution_id": interaction.execution_id,
|
|
"interactive": case.interactive,
|
|
"regressions": regressions(
|
|
case,
|
|
{
|
|
"revisions": sum(revisions.values()),
|
|
"seconds": elapsed,
|
|
"tokens": tokens["total"].get("total"),
|
|
},
|
|
),
|
|
"seconds": round(elapsed, 1),
|
|
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
|
"orchestrator_turns": interaction.state.get("turns"),
|
|
"tokens": tokens,
|
|
"timeline": runtime.timeline,
|
|
"agent_runs": dict(runtime.runs),
|
|
"outcomes": dict(runtime.outcomes),
|
|
"revisions": revisions,
|
|
# Every routing decision: the group that applied, the action, what was judged.
|
|
"decisions": interaction.state.get("decisions", []),
|
|
"warnings": interaction.state.get("warnings", []),
|
|
"conversation": interaction.state.get("conversation", []),
|
|
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
|
}
|
|
return CaseResult(result.case_id, result.status, result.checks, details)
|
|
|
|
|
|
def make_judge(settings: Settings) -> Judge:
|
|
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
|
|
language-model judge; otherwise the language-model judge."""
|
|
from app.agent import open_decider
|
|
from app.providers import load_provider
|
|
|
|
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
|
|
return DeciderJudge(open_decider(settings))
|
|
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
|
settings.CLOUD_PROVIDER
|
|
).models.default.removeprefix(GEMINI_PREFIX)
|
|
return GeminiJudge(model)
|
|
|
|
|
|
async def main(arguments: Sequence[str]) -> int:
|
|
use_judge = "--judge" in arguments
|
|
case_ids = [argument for argument in arguments if argument != "--judge"]
|
|
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
|
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
|
|
)
|
|
logging.getLogger("app").setLevel(logging.INFO)
|
|
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
|
|
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
|
|
if not cases:
|
|
print(f"No cases match {list(case_ids)}", file=sys.stderr)
|
|
return 2
|
|
|
|
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
run_dir = RESULTS_DIR / stamp
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
judge = make_judge(settings) if use_judge else None
|
|
results: list[dict[str, Any]] = []
|
|
for case in cases:
|
|
print(f"== {case.id}: running", flush=True)
|
|
try:
|
|
result = await run_case(case, settings, run_dir, judge)
|
|
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
|
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
|
results.append(
|
|
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
|
|
)
|
|
continue
|
|
print(
|
|
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
|
|
flush=True,
|
|
)
|
|
for check in result.checks:
|
|
mark = "ok " if check.passed else "FAIL"
|
|
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
|
|
details = result.details
|
|
print(
|
|
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
|
|
f"revisions {details['revisions'] or 'none'}",
|
|
flush=True,
|
|
)
|
|
for regression in details["regressions"]:
|
|
print(f" REGRESSION {regression}", flush=True)
|
|
for model, counts in details["tokens"]["by_model"].items():
|
|
print(f" tokens {model}: {counts}", flush=True)
|
|
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
|
results.append(
|
|
{
|
|
"case_id": result.case_id,
|
|
"status": result.status,
|
|
"score": result.score,
|
|
"passed": result.passed,
|
|
"checks": [asdict(check) for check in result.checks],
|
|
"details": result.details,
|
|
}
|
|
)
|
|
|
|
report = run_dir / "report.json"
|
|
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
|
passed = sum(1 for r in results if r.get("passed"))
|
|
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
|
|
print(
|
|
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
|
|
f"Report: {report}"
|
|
)
|
|
return 0 if passed == len(results) and not regressed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|