Files
gcp_solution_architecture_a…/eval/solution_eval.py
Jonathan Boniface afe78640f8
Some checks failed
validation / verify (push) Failing after 10s
feat: start an Execution from a scope spec (ADR-0008, iteration 9 step 1)
A scope spec is a YAML file of what the customer wants and the facts a design
depends on, complete or partly filled. It sits beside the Intent, which stays
in the customer's words.

- skills/scoping holds a provider-neutral catalogue of 63 fields; the Cloud
  provider's bundle supplies the values it allows for the fields left to it.
- app/scope.py reads a spec against the catalogue with no model: it refuses an
  unknown field or a value of the wrong type, and yields what is stated, what
  is left to the agent, what is unknown and how pressing each gap is. A field
  may say ask, decide or not_applicable.
- ExecutionEngine.start(scope=...), /generate, A2A metadata and evaluation
  cases accept one. Its statement is the request, its engagement reference
  names the Customer case, and its run options apply. A pre-approved start is
  refused when an essential field is not settled.
- The spec as supplied is kept unaltered beside a completed copy with
  assumptions, gaps, provenance, readiness and a status.
- Every Specialist is shown the spec and given the Skill: a stated choice
  binds, exclusions are out of scope.
- Covers workloads and platform foundations; case-08-landing-zone starts from
  a benchmark spec.

Plans for scoping (iteration 9) and the work breakdown structure (iteration
10, nothing built), and the design spec they come from. Not run with real
models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 17:53:00 +01:00

404 lines
15 KiB
Python

"""Evaluate the solution architect agent on benchmark cases with real models.
Each case runs a pre-approved Execution with scripted user answers through the full
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
eval/scoring.py. Costs model usage: run deliberately.
Usage:
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
A case marked "interactive" runs with a scripted user in the loop. The exit code is
non-zero when a case fails or exceeds one of its thresholds.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
import time
from collections import Counter
from collections.abc import Callable, Sequence
from dataclasses import asdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
from app.agent import ExecutionEngine, Interaction, open_assembly, skill_dirs
from app.config import Settings, get_settings
from app.nodes.execution_nodes import automatic_reply
from app.recorder import InMemoryRecorder
from app.states.execution_state import execution_dir
from app.tools.local import TerraformRunner
from eval.judge import DeciderJudge, GeminiJudge, Judge
from eval.scoring import (
CaseResult,
CaseSpec,
Check,
checklist_items,
load_cases,
regressions,
score_change,
score_execution,
template_headings,
)
GEMINI_PREFIX = "gemini/"
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
ROOT = Path(__file__).resolve().parent
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
RESULTS_DIR = ROOT / "results"
class CountingRuntime:
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
def __init__(
self, inner: AgentRuntime, on_run: Callable[[], None] | None = None
) -> None:
self._inner = inner
# Called after every agent run, so a run cut short leaves a record behind.
self._on_run = on_run
self._started = time.monotonic()
self.runs: Counter[str] = Counter()
self.outcomes: Counter[str] = Counter()
self.timeline: list[dict[str, Any]] = []
async def run(
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
) -> Outcome:
self.runs[role.name] += 1
start = time.monotonic()
outcome = await self._inner.run(role, task, tools)
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
self.timeline.append(
{
"role": role.name,
"deliverable": task.deliverable,
"start_s": round(start - self._started, 1),
"seconds": round(time.monotonic() - start, 1),
"outcome": type(outcome).__name__,
}
)
if self._on_run:
self._on_run()
return outcome
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
"""Seconds until the first validation_results run ended: the run without revisions."""
for run in timeline:
if run["deliverable"] == "validation_results":
return round(run["start_s"] + run["seconds"], 1)
return None
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
by_role = {
f"{role} ({model})": asdict(counts)
for (role, model), counts in sorted(usage.items())
}
by_model: dict[str, Counter[str]] = {}
for (_, model), counts in usage.items():
by_model.setdefault(model, Counter()).update(asdict(counts))
total: Counter[str] = Counter()
for counts in by_model.values():
total.update(counts)
return {
"by_role": by_role,
"by_model": {model: dict(c) for model, c in by_model.items()},
"total": dict(total),
}
async def run_with_scripted_user(
engine: ExecutionEngine,
case: CaseSpec,
execution_id: str,
max_user_turns: int = MAX_USER_TURNS,
) -> Interaction:
"""Run an interactive Execution, answering each prompt as the case scripts it:
Approvals are granted, cloud reads denied, and questions answered by pattern."""
interaction = await engine.start(
case.workflow_request,
customer_case=case.id,
allow_cloud_read=False,
execution_id=execution_id,
overlay_skills=case.overlay_skills,
terraform_modules=case.terraform_modules,
scope=case.scope or None,
)
for _ in range(max_user_turns):
if interaction.status != "awaiting_user" or interaction.prompt is None:
break
reply = automatic_reply(
interaction.prompt, [list(pair) for pair in case.scripted_answers]
)
interaction = await engine.resume(execution_id, reply)
return interaction
async def run_case(
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
) -> CaseResult:
from app.adk.adk_runtime import AdkAgentRuntime
from app.providers import load_provider
bundle = load_provider(settings.CLOUD_PROVIDER)
adk_runtime = AdkAgentRuntime(
skill_dirs=skill_dirs(settings, bundle),
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
started = time.monotonic()
progress_path = run_dir / f"{case.id}-progress.json"
def write_progress() -> None:
"""What the run has cost so far. Overwritten after every agent run and left
in place, so an interrupted run still has its time, tokens and timeline."""
progress_path.write_text(
json.dumps(
{
"case_id": case.id,
"finished": False,
"seconds": round(time.monotonic() - started, 1),
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"tokens": token_report(adk_runtime.usage),
"timeline": runtime.timeline,
},
indent=2,
),
encoding="utf-8",
)
runtime = CountingRuntime(adk_runtime, on_run=write_progress)
recorder = InMemoryRecorder()
async with open_assembly(
settings, runtime=runtime, recorder=recorder, output_root=run_dir
) as assembly:
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
baseline_tokens: float | None = None
baseline_seconds: float | None = None
if case.change:
# A change case: deliver the request, then run the change against it.
base = await assembly.engine.start(
case.workflow_request,
customer_case=case.id,
pre_approved=True,
allow_cloud_read=False,
execution_id=f"{execution_id}-base",
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
terraform_modules=case.terraform_modules,
scope=case.scope or None,
)
baseline_tokens = token_report(adk_runtime.usage)["total"].get("total")
baseline_seconds = round(time.monotonic() - started, 1)
if base.status != "completed":
interaction = base # nothing to change: score what there is
else:
interaction = await assembly.engine.start(
case.change,
customer_case=case.id,
pre_approved=True,
allow_cloud_read=False,
execution_id=execution_id,
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
terraform_modules=case.terraform_modules,
scope=case.scope or None,
baseline=base.execution_id,
)
elif case.interactive:
interaction = await run_with_scripted_user(
assembly.engine, case, execution_id
)
else:
interaction = await assembly.engine.start(
case.workflow_request,
customer_case=case.id,
pre_approved=True,
allow_cloud_read=False,
execution_id=execution_id,
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
terraform_modules=case.terraform_modules,
scope=case.scope or None,
)
elapsed = time.monotonic() - started
write_progress()
progress_path.write_text(
progress_path.read_text(encoding="utf-8").replace(
'"finished": false', '"finished": true', 1
),
encoding="utf-8",
)
runner = TerraformRunner(
binary=settings.TERRAFORM_BINARY,
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
plugin_cache_dir=settings.terraform_plugin_cache(),
module_namespaces=bundle.terraform_module_namespaces
if case.terraform_modules
else (),
)
result = await score_execution(
case,
interaction,
template_headings(bundle.output_template),
runner.validate,
judge=judge,
pillar_checklists={
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
for pillar, skill in bundle.pillar_skills.items()
},
timeline=runtime.timeline,
)
ledger = interaction.state.get("ledger", {})
tokens = token_report(adk_runtime.usage)
change_checks: tuple[Check, ...] = ()
if case.change:
total = tokens["total"].get("total")
change_tokens = (
total - baseline_tokens
if total is not None and baseline_tokens is not None
else None
)
change_checks = tuple(score_change(interaction, baseline_tokens, change_tokens))
revisions = {
kind: item["revisions"]
for kind, item in ledger.get("deliverables", {}).items()
if item["revisions"]
}
details = {
"execution_id": interaction.execution_id,
"interactive": case.interactive,
"regressions": regressions(
case,
{
"revisions": sum(revisions.values()),
"seconds": elapsed,
"tokens": tokens["total"].get("total"),
},
),
"seconds": round(elapsed, 1),
"first_pass_seconds": first_pass_seconds(runtime.timeline),
"orchestrator_turns": interaction.state.get("turns"),
"tokens": tokens,
"timeline": runtime.timeline,
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"revisions": revisions,
# Every routing decision: the group that applied, the action, what was judged.
"decisions": interaction.state.get("decisions", []),
"warnings": interaction.state.get("warnings", []),
"conversation": interaction.state.get("conversation", []),
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
}
if case.change:
details["baseline"] = {
"execution_id": (interaction.state.get("baseline") or {}).get(
"execution_id"
),
"seconds": baseline_seconds,
"tokens": baseline_tokens,
}
return CaseResult(
result.case_id, result.status, (*result.checks, *change_checks), details
)
def make_judge(settings: Settings) -> Judge:
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
language-model judge; otherwise the language-model judge."""
from app.agent import open_decider
from app.providers import load_provider
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
return DeciderJudge(open_decider(settings))
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
settings.CLOUD_PROVIDER
).models.default.removeprefix(GEMINI_PREFIX)
return GeminiJudge(model)
async def main(arguments: Sequence[str]) -> int:
use_judge = "--judge" in arguments
case_ids = [argument for argument in arguments if argument != "--judge"]
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
logging.basicConfig(
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
)
logging.getLogger("app").setLevel(logging.INFO)
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
if not cases:
print(f"No cases match {list(case_ids)}", file=sys.stderr)
return 2
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
run_dir = RESULTS_DIR / stamp
run_dir.mkdir(parents=True, exist_ok=True)
judge = make_judge(settings) if use_judge else None
results: list[dict[str, Any]] = []
for case in cases:
print(f"== {case.id}: running", flush=True)
try:
result = await run_case(case, settings, run_dir, judge)
except Exception as exc: # noqa: BLE001 - record the failure and continue
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
results.append(
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
)
continue
print(
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
flush=True,
)
for check in result.checks:
mark = "ok " if check.passed else "FAIL"
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
details = result.details
print(
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
f"revisions {details['revisions'] or 'none'}",
flush=True,
)
for regression in details["regressions"]:
print(f" REGRESSION {regression}", flush=True)
for model, counts in details["tokens"]["by_model"].items():
print(f" tokens {model}: {counts}", flush=True)
print(f" tokens total: {details['tokens']['total']}", flush=True)
results.append(
{
"case_id": result.case_id,
"status": result.status,
"score": result.score,
"passed": result.passed,
"checks": [asdict(check) for check in result.checks],
"details": result.details,
}
)
report = run_dir / "report.json"
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
passed = sum(1 for r in results if r.get("passed"))
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
print(
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
f"Report: {report}"
)
return 0 if passed == len(results) and not regressed else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))