Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
393 lines
15 KiB
Python
393 lines
15 KiB
Python
"""Evaluate the solution architect agent on benchmark cases with real models.
|
|
|
|
Each case runs a pre-approved Execution with scripted user answers through the full
|
|
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
|
|
eval/scoring.py. Costs model usage: run deliberately.
|
|
|
|
Usage:
|
|
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
|
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
|
|
|
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
|
|
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
|
|
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
|
|
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
|
non-zero when a case fails or exceeds one of its thresholds.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
from collections.abc import Callable, Sequence
|
|
from dataclasses import asdict
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
|
from app.agent import ExecutionEngine, Interaction, open_assembly, skill_dirs
|
|
from app.config import Settings, get_settings
|
|
from app.nodes.execution_nodes import automatic_reply
|
|
from app.recorder import InMemoryRecorder
|
|
from app.states.execution_state import execution_dir
|
|
from app.tools.local import TerraformRunner
|
|
from eval.judge import DeciderJudge, GeminiJudge, Judge
|
|
from eval.scoring import (
|
|
CaseResult,
|
|
CaseSpec,
|
|
Check,
|
|
checklist_items,
|
|
load_cases,
|
|
regressions,
|
|
score_change,
|
|
score_execution,
|
|
template_headings,
|
|
)
|
|
|
|
GEMINI_PREFIX = "gemini/"
|
|
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
|
|
ROOT = Path(__file__).resolve().parent
|
|
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
|
RESULTS_DIR = ROOT / "results"
|
|
|
|
|
|
class CountingRuntime:
|
|
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
|
|
|
|
def __init__(
|
|
self, inner: AgentRuntime, on_run: Callable[[], None] | None = None
|
|
) -> None:
|
|
self._inner = inner
|
|
# Called after every agent run, so a run cut short leaves a record behind.
|
|
self._on_run = on_run
|
|
self._started = time.monotonic()
|
|
self.runs: Counter[str] = Counter()
|
|
self.outcomes: Counter[str] = Counter()
|
|
self.timeline: list[dict[str, Any]] = []
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
self.runs[role.name] += 1
|
|
start = time.monotonic()
|
|
outcome = await self._inner.run(role, task, tools)
|
|
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
|
|
self.timeline.append(
|
|
{
|
|
"role": role.name,
|
|
"deliverable": task.deliverable,
|
|
"start_s": round(start - self._started, 1),
|
|
"seconds": round(time.monotonic() - start, 1),
|
|
"outcome": type(outcome).__name__,
|
|
}
|
|
)
|
|
if self._on_run:
|
|
self._on_run()
|
|
return outcome
|
|
|
|
|
|
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
|
|
"""Seconds until the first validation_results run ended: the run without revisions."""
|
|
for run in timeline:
|
|
if run["deliverable"] == "validation_results":
|
|
return round(run["start_s"] + run["seconds"], 1)
|
|
return None
|
|
|
|
|
|
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
|
by_role = {
|
|
f"{role} ({model})": asdict(counts)
|
|
for (role, model), counts in sorted(usage.items())
|
|
}
|
|
by_model: dict[str, Counter[str]] = {}
|
|
for (_, model), counts in usage.items():
|
|
by_model.setdefault(model, Counter()).update(asdict(counts))
|
|
total: Counter[str] = Counter()
|
|
for counts in by_model.values():
|
|
total.update(counts)
|
|
return {
|
|
"by_role": by_role,
|
|
"by_model": {model: dict(c) for model, c in by_model.items()},
|
|
"total": dict(total),
|
|
}
|
|
|
|
|
|
async def run_with_scripted_user(
|
|
engine: ExecutionEngine,
|
|
case: CaseSpec,
|
|
execution_id: str,
|
|
max_user_turns: int = MAX_USER_TURNS,
|
|
) -> Interaction:
|
|
"""Run an interactive Execution, answering each prompt as the case scripts it:
|
|
Approvals are granted, cloud reads denied, and questions answered by pattern."""
|
|
interaction = await engine.start(
|
|
case.workflow_request,
|
|
customer_case=case.id,
|
|
allow_cloud_read=False,
|
|
execution_id=execution_id,
|
|
overlay_skills=case.overlay_skills,
|
|
)
|
|
for _ in range(max_user_turns):
|
|
if interaction.status != "awaiting_user" or interaction.prompt is None:
|
|
break
|
|
reply = automatic_reply(
|
|
interaction.prompt, [list(pair) for pair in case.scripted_answers]
|
|
)
|
|
interaction = await engine.resume(execution_id, reply)
|
|
return interaction
|
|
|
|
|
|
async def run_case(
|
|
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
|
|
) -> CaseResult:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.providers import load_provider
|
|
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
adk_runtime = AdkAgentRuntime(
|
|
skill_dirs=skill_dirs(settings, bundle),
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
started = time.monotonic()
|
|
progress_path = run_dir / f"{case.id}-progress.json"
|
|
|
|
def write_progress() -> None:
|
|
"""What the run has cost so far. Overwritten after every agent run and left
|
|
in place, so an interrupted run still has its time, tokens and timeline."""
|
|
progress_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"case_id": case.id,
|
|
"finished": False,
|
|
"seconds": round(time.monotonic() - started, 1),
|
|
"agent_runs": dict(runtime.runs),
|
|
"outcomes": dict(runtime.outcomes),
|
|
"tokens": token_report(adk_runtime.usage),
|
|
"timeline": runtime.timeline,
|
|
},
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
runtime = CountingRuntime(adk_runtime, on_run=write_progress)
|
|
recorder = InMemoryRecorder()
|
|
async with open_assembly(
|
|
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
|
) as assembly:
|
|
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
|
baseline_tokens: float | None = None
|
|
baseline_seconds: float | None = None
|
|
if case.change:
|
|
# A change case: deliver the request, then run the change against it.
|
|
base = await assembly.engine.start(
|
|
case.workflow_request,
|
|
customer_case=case.id,
|
|
pre_approved=True,
|
|
allow_cloud_read=False,
|
|
execution_id=f"{execution_id}-base",
|
|
scripted_answers=case.scripted_answers,
|
|
overlay_skills=case.overlay_skills,
|
|
)
|
|
baseline_tokens = token_report(adk_runtime.usage)["total"].get("total")
|
|
baseline_seconds = round(time.monotonic() - started, 1)
|
|
if base.status != "completed":
|
|
interaction = base # nothing to change: score what there is
|
|
else:
|
|
interaction = await assembly.engine.start(
|
|
case.change,
|
|
customer_case=case.id,
|
|
pre_approved=True,
|
|
allow_cloud_read=False,
|
|
execution_id=execution_id,
|
|
scripted_answers=case.scripted_answers,
|
|
overlay_skills=case.overlay_skills,
|
|
baseline=base.execution_id,
|
|
)
|
|
elif case.interactive:
|
|
interaction = await run_with_scripted_user(
|
|
assembly.engine, case, execution_id
|
|
)
|
|
else:
|
|
interaction = await assembly.engine.start(
|
|
case.workflow_request,
|
|
customer_case=case.id,
|
|
pre_approved=True,
|
|
allow_cloud_read=False,
|
|
execution_id=execution_id,
|
|
scripted_answers=case.scripted_answers,
|
|
overlay_skills=case.overlay_skills,
|
|
)
|
|
elapsed = time.monotonic() - started
|
|
write_progress()
|
|
progress_path.write_text(
|
|
progress_path.read_text(encoding="utf-8").replace(
|
|
'"finished": false', '"finished": true', 1
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
runner = TerraformRunner(
|
|
binary=settings.TERRAFORM_BINARY,
|
|
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
|
plugin_cache_dir=settings.terraform_plugin_cache(),
|
|
)
|
|
result = await score_execution(
|
|
case,
|
|
interaction,
|
|
template_headings(bundle.output_template),
|
|
runner.validate,
|
|
judge=judge,
|
|
pillar_checklists={
|
|
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
|
|
for pillar, skill in bundle.pillar_skills.items()
|
|
},
|
|
timeline=runtime.timeline,
|
|
)
|
|
ledger = interaction.state.get("ledger", {})
|
|
tokens = token_report(adk_runtime.usage)
|
|
change_checks: tuple[Check, ...] = ()
|
|
if case.change:
|
|
total = tokens["total"].get("total")
|
|
change_tokens = (
|
|
total - baseline_tokens
|
|
if total is not None and baseline_tokens is not None
|
|
else None
|
|
)
|
|
change_checks = tuple(score_change(interaction, baseline_tokens, change_tokens))
|
|
revisions = {
|
|
kind: item["revisions"]
|
|
for kind, item in ledger.get("deliverables", {}).items()
|
|
if item["revisions"]
|
|
}
|
|
details = {
|
|
"execution_id": interaction.execution_id,
|
|
"interactive": case.interactive,
|
|
"regressions": regressions(
|
|
case,
|
|
{
|
|
"revisions": sum(revisions.values()),
|
|
"seconds": elapsed,
|
|
"tokens": tokens["total"].get("total"),
|
|
},
|
|
),
|
|
"seconds": round(elapsed, 1),
|
|
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
|
"orchestrator_turns": interaction.state.get("turns"),
|
|
"tokens": tokens,
|
|
"timeline": runtime.timeline,
|
|
"agent_runs": dict(runtime.runs),
|
|
"outcomes": dict(runtime.outcomes),
|
|
"revisions": revisions,
|
|
# Every routing decision: the group that applied, the action, what was judged.
|
|
"decisions": interaction.state.get("decisions", []),
|
|
"warnings": interaction.state.get("warnings", []),
|
|
"conversation": interaction.state.get("conversation", []),
|
|
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
|
}
|
|
if case.change:
|
|
details["baseline"] = {
|
|
"execution_id": (interaction.state.get("baseline") or {}).get(
|
|
"execution_id"
|
|
),
|
|
"seconds": baseline_seconds,
|
|
"tokens": baseline_tokens,
|
|
}
|
|
return CaseResult(
|
|
result.case_id, result.status, (*result.checks, *change_checks), details
|
|
)
|
|
|
|
|
|
def make_judge(settings: Settings) -> Judge:
|
|
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
|
|
language-model judge; otherwise the language-model judge."""
|
|
from app.agent import open_decider
|
|
from app.providers import load_provider
|
|
|
|
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
|
|
return DeciderJudge(open_decider(settings))
|
|
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
|
settings.CLOUD_PROVIDER
|
|
).models.default.removeprefix(GEMINI_PREFIX)
|
|
return GeminiJudge(model)
|
|
|
|
|
|
async def main(arguments: Sequence[str]) -> int:
|
|
use_judge = "--judge" in arguments
|
|
case_ids = [argument for argument in arguments if argument != "--judge"]
|
|
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
|
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
|
logging.basicConfig(
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
|
|
)
|
|
logging.getLogger("app").setLevel(logging.INFO)
|
|
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
|
|
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
|
|
if not cases:
|
|
print(f"No cases match {list(case_ids)}", file=sys.stderr)
|
|
return 2
|
|
|
|
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
run_dir = RESULTS_DIR / stamp
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
judge = make_judge(settings) if use_judge else None
|
|
results: list[dict[str, Any]] = []
|
|
for case in cases:
|
|
print(f"== {case.id}: running", flush=True)
|
|
try:
|
|
result = await run_case(case, settings, run_dir, judge)
|
|
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
|
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
|
results.append(
|
|
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
|
|
)
|
|
continue
|
|
print(
|
|
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
|
|
flush=True,
|
|
)
|
|
for check in result.checks:
|
|
mark = "ok " if check.passed else "FAIL"
|
|
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
|
|
details = result.details
|
|
print(
|
|
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
|
|
f"revisions {details['revisions'] or 'none'}",
|
|
flush=True,
|
|
)
|
|
for regression in details["regressions"]:
|
|
print(f" REGRESSION {regression}", flush=True)
|
|
for model, counts in details["tokens"]["by_model"].items():
|
|
print(f" tokens {model}: {counts}", flush=True)
|
|
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
|
results.append(
|
|
{
|
|
"case_id": result.case_id,
|
|
"status": result.status,
|
|
"score": result.score,
|
|
"passed": result.passed,
|
|
"checks": [asdict(check) for check in result.checks],
|
|
"details": result.details,
|
|
}
|
|
)
|
|
|
|
report = run_dir / "report.json"
|
|
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
|
passed = sum(1 for r in results if r.get("passed"))
|
|
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
|
|
print(
|
|
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
|
|
f"Report: {report}"
|
|
)
|
|
return 0 if passed == len(results) and not regressed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|