feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -194,5 +194,78 @@
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "case-04-interactive-web-app",
|
||||
"name": "Internal booking web application with a relational database (interactive)",
|
||||
"workflow_request": "Design an internal web application on Google Cloud for staff to book meeting rooms and equipment, backed by a managed relational database, available only to employees signed in with the corporate identity provider, with application secrets kept out of code and configuration.",
|
||||
"required_products": [
|
||||
"Cloud Run",
|
||||
"Cloud SQL",
|
||||
"Secret Manager"
|
||||
],
|
||||
"scripted_answers": [
|
||||
[
|
||||
"region|location|geograph",
|
||||
"A single region, europe-west1 (Belgium). All users are in the EU."
|
||||
],
|
||||
[
|
||||
"user|staff|employee|how many|concurren|traffic|volume|scale|peak|requests per|rps",
|
||||
"About 4,000 employees; 300 concurrent users at the 09:00 peak, around 20 requests per second."
|
||||
],
|
||||
[
|
||||
"availability|uptime|sla|rto|rpo|disaster|recover|backup",
|
||||
"99.5% during business hours (07:00-19:00 CET). RTO 4 hours, RPO 1 hour. Daily backups kept 14 days."
|
||||
],
|
||||
[
|
||||
"latency|response time|performance",
|
||||
"Pages should load within 1 second at p95."
|
||||
],
|
||||
[
|
||||
"identity|sign.?in|login|sso|auth|idp|access",
|
||||
"Employees sign in with Google Workspace accounts; no access from outside the company. Admins are a Google group."
|
||||
],
|
||||
[
|
||||
"security|compliance|encrypt|privacy|gdpr|secret|credential|password",
|
||||
"GDPR applies to employee names and emails. Google-managed keys are fine. Database passwords and API keys must live in a secrets store and rotate every 90 days."
|
||||
],
|
||||
[
|
||||
"data|database|schema|relational|postgres|mysql|size|storage",
|
||||
"PostgreSQL. Under 20 GB of data in the first three years; bookings, rooms, equipment and audit history."
|
||||
],
|
||||
[
|
||||
"cost|budget|spend",
|
||||
"Under 600 EUR a month; the database can be small."
|
||||
],
|
||||
[
|
||||
"current|existing|today|on-prem|migrat|legacy",
|
||||
"Bookings are kept in a shared spreadsheet today; there is no data to migrate."
|
||||
],
|
||||
[
|
||||
"depend|integrat|calendar|email|notif|downstream|upstream",
|
||||
"Send booking confirmation emails through the existing SMTP relay; no calendar integration in phase one."
|
||||
],
|
||||
[
|
||||
"operat|monitor|logging|alert|team|support|deploy|ci|pipeline",
|
||||
"Two developers own it, deploy containers from Cloud Build with Terraform, and want Cloud Monitoring alerts to a Google Chat space."
|
||||
],
|
||||
[
|
||||
"network|vpc|private|internet|egress|ingress",
|
||||
"No public database endpoint; the database must use private IP only."
|
||||
],
|
||||
[
|
||||
"sustainab|carbon",
|
||||
"No specific requirement."
|
||||
],
|
||||
[
|
||||
"retention|audit|archive|delete",
|
||||
"Keep booking history 2 years, then delete; keep audit logs 1 year."
|
||||
],
|
||||
[
|
||||
"approve|confirm|proceed|look right|any changes|happy",
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
],
|
||||
"interactive": true
|
||||
}
|
||||
]
|
||||
|
||||
77
eval/judge.py
Normal file
77
eval/judge.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
||||
|
||||
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
||||
question. The shape (one state, many typed questions) is deliberately the one a
|
||||
decision model takes, so the Gemini judge here can be swapped for one later.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"You grade a document from a cloud solution architecture engagement. Answer every "
|
||||
"question about the document only, strictly and without benefit of the doubt. "
|
||||
'Reply with one JSON object mapping each question id to a number: for a "yes_no" '
|
||||
'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 '
|
||||
"(weakly), 0.67 (mostly) or 1 (fully)."
|
||||
)
|
||||
|
||||
|
||||
class Judge(Protocol):
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
"""Score each question, keyed by id, from 0 to 1.
|
||||
|
||||
`questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class GeminiJudge:
|
||||
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
||||
|
||||
def __init__(self, model: str) -> None:
|
||||
from google import genai
|
||||
|
||||
self._client = genai.Client()
|
||||
self._model = model
|
||||
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
from google.genai import types
|
||||
|
||||
prompt = (
|
||||
f"{INSTRUCTIONS}\n\nQuestions:\n"
|
||||
f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n"
|
||||
f"Document:\n<document>\n{document}\n</document>"
|
||||
)
|
||||
response = await self._client.aio.models.generate_content(
|
||||
model=self._model,
|
||||
contents=prompt,
|
||||
config=types.GenerateContentConfig(
|
||||
temperature=0, response_mime_type="application/json"
|
||||
),
|
||||
)
|
||||
return parse_scores(response.text or "", questions)
|
||||
|
||||
|
||||
def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]:
|
||||
"""The judge's reply as scores; a missing or malformed answer scores 0."""
|
||||
try:
|
||||
raw = json.loads(text)
|
||||
except ValueError:
|
||||
raw = {}
|
||||
if not isinstance(raw, dict):
|
||||
raw = {}
|
||||
scores = {}
|
||||
for key in questions:
|
||||
value = raw.get(key, 0)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
value = 1.0 if value is True else 0.0
|
||||
scores[key] = min(1.0, max(0.0, float(value)))
|
||||
return scores
|
||||
227
eval/scoring.py
227
eval/scoring.py
@@ -2,10 +2,16 @@
|
||||
|
||||
Each check is a small, explainable rule tied to the upstream Skill's requirements.
|
||||
A case passes when its score reaches PASS_THRESHOLD.
|
||||
|
||||
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
|
||||
quality) run when a judge is given, and are what lets a better or worse Skill move the
|
||||
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
|
||||
reported regression without changing the score.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
@@ -15,9 +21,43 @@ from typing import Any
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.chain import (
|
||||
COMPONENTS_FILE,
|
||||
PRODUCTS_FILE,
|
||||
RECOMMENDATIONS_FILE,
|
||||
REQUIREMENTS_FILE,
|
||||
VERDICTS_FILE,
|
||||
read_graph,
|
||||
read_list,
|
||||
validation_outcome,
|
||||
)
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
from app.workflows.target_architecture import (
|
||||
DIAGRAM_ATTACHMENT,
|
||||
GRAPH_ATTACHMENT,
|
||||
GraphError,
|
||||
chain_gaps,
|
||||
parse_graph,
|
||||
render_mermaid,
|
||||
)
|
||||
from eval.judge import Judge
|
||||
|
||||
PASS_THRESHOLD = 0.8
|
||||
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
|
||||
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
|
||||
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
|
||||
|
||||
# What a reviewer would ask of design recommendations, graded 0 to 1 each.
|
||||
QUALITY_RUBRIC: Mapping[str, str] = {
|
||||
"specific": "Are the recommendations specific to this workload and its stated "
|
||||
"requirements, as opposed to generic best practice that would fit any workload?",
|
||||
"actionable": "Does each recommendation name the concrete setting, feature or "
|
||||
"configuration to apply, so an engineer could act on it without more research?",
|
||||
"trade_offs": "Where pillars pull in different directions (for example cost "
|
||||
"against reliability), is the trade-off named and a resolution recommended?",
|
||||
"traceable": "Can each recommendation be traced to a requirement or constraint "
|
||||
"of the workload?",
|
||||
}
|
||||
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
|
||||
|
||||
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
@@ -32,6 +72,11 @@ class CaseSpec:
|
||||
workflow_request: str
|
||||
required_products: tuple[str, ...]
|
||||
scripted_answers: tuple[tuple[str, str], ...]
|
||||
# Run with a user in the loop (scripted): Approvals and questions are answered
|
||||
# turn by turn, and Specialists are checked for asking before they submit.
|
||||
interactive: bool = False
|
||||
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
|
||||
thresholds: Mapping[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -62,8 +107,6 @@ class CaseResult:
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[CaseSpec]:
|
||||
import json
|
||||
|
||||
return [
|
||||
CaseSpec(
|
||||
id=item["id"],
|
||||
@@ -73,6 +116,8 @@ def load_cases(path: Path) -> list[CaseSpec]:
|
||||
scripted_answers=tuple(
|
||||
tuple(pair) for pair in item.get("scripted_answers", [])
|
||||
),
|
||||
interactive=bool(item.get("interactive", False)),
|
||||
thresholds=dict(item.get("thresholds", {})),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
@@ -87,6 +132,53 @@ def template_headings(template: Path) -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def checklist_items(skill_file: Path) -> list[str]:
|
||||
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
|
||||
text = skill_file.read_text(encoding="utf-8")
|
||||
match = re.search(
|
||||
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
|
||||
text,
|
||||
re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
return []
|
||||
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
|
||||
# items to check are the leaves, each prefixed with its category.
|
||||
bullets: list[tuple[int, str]] = []
|
||||
for line in match.group(1).splitlines():
|
||||
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
|
||||
if bullet:
|
||||
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
|
||||
bullets.append((len(bullet.group(1)), text.replace("**", "")))
|
||||
elif bullets and line.startswith(" ") and line.strip():
|
||||
indent, text = bullets[-1]
|
||||
bullets[-1] = (indent, f"{text} {line.strip()}")
|
||||
|
||||
items: list[str] = []
|
||||
category = ""
|
||||
for index, (indent, text) in enumerate(bullets):
|
||||
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
|
||||
if has_children:
|
||||
category = text.rstrip(":").strip()
|
||||
elif indent == 0:
|
||||
category = ""
|
||||
items.append(text)
|
||||
else:
|
||||
items.append(f"{category}: {text}" if category else text)
|
||||
return items
|
||||
|
||||
|
||||
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
|
||||
"""Each threshold the run exceeded, as "metric: measured > limit"."""
|
||||
over = []
|
||||
for metric in THRESHOLD_METRICS:
|
||||
limit = case.thresholds.get(metric)
|
||||
value = measured.get(metric)
|
||||
if limit is not None and value is not None and value > limit:
|
||||
over.append(f"{metric}: {value:g} > {limit:g}")
|
||||
return over
|
||||
|
||||
|
||||
def _normalise(text: str) -> str:
|
||||
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
|
||||
|
||||
@@ -103,6 +195,10 @@ async def score_execution(
|
||||
interaction: Interaction,
|
||||
headings: Sequence[str],
|
||||
terraform_validate: TerraformValidate,
|
||||
*,
|
||||
judge: Judge | None = None,
|
||||
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
|
||||
timeline: Sequence[Mapping[str, Any]] = (),
|
||||
) -> CaseResult:
|
||||
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
|
||||
"deliverables", {}
|
||||
@@ -179,7 +275,7 @@ async def score_execution(
|
||||
)
|
||||
)
|
||||
|
||||
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
|
||||
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
|
||||
"architecture.mmd", ""
|
||||
)
|
||||
mermaid = await mermaid_check(diagram)
|
||||
@@ -204,4 +300,129 @@ async def score_execution(
|
||||
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
||||
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
||||
|
||||
checks.append(_chain_check(deliverables))
|
||||
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
|
||||
checks.append(
|
||||
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
|
||||
)
|
||||
|
||||
if case.interactive:
|
||||
checks.append(_asks_before_submitting(timeline))
|
||||
|
||||
if judge is not None:
|
||||
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
|
||||
checks.append(
|
||||
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
|
||||
)
|
||||
checks.append(await _quality(judge, recommendations))
|
||||
|
||||
return CaseResult(case.id, interaction.status, tuple(checks))
|
||||
|
||||
|
||||
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
|
||||
"""The Requirements chain holds end to end: every requirement reaches a Check,
|
||||
directly or through a component's Element, and nothing along it is missing."""
|
||||
name = "requirements_chain_complete"
|
||||
graph = read_graph(deliverables)
|
||||
if graph is None:
|
||||
return Check(name, False, "no well-formed Target architecture")
|
||||
gaps = chain_gaps(
|
||||
graph,
|
||||
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
|
||||
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
|
||||
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
|
||||
recommendations=read_list(
|
||||
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
|
||||
),
|
||||
)
|
||||
return Check(name, not gaps, "; ".join(gaps))
|
||||
|
||||
|
||||
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
|
||||
"""Validation ended with every Check passed and the dry-run not failed."""
|
||||
name = "every_check_passes"
|
||||
text = attachments.get(VERDICTS_FILE)
|
||||
if text is None:
|
||||
return Check(name, False, f"no {VERDICTS_FILE}")
|
||||
outcome = validation_outcome(text)
|
||||
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
|
||||
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
|
||||
]
|
||||
if outcome["dry_run"].get("verdict") == "fail":
|
||||
open_checks.append("dry-run failed")
|
||||
if not outcome["passed"] and not open_checks:
|
||||
open_checks.append("no verdicts")
|
||||
return Check(name, not open_checks, "; ".join(open_checks))
|
||||
|
||||
|
||||
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
|
||||
name = "diagram_matches_graph"
|
||||
try:
|
||||
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
|
||||
except GraphError as exc:
|
||||
return Check(name, False, exc.problems[0])
|
||||
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
|
||||
return Check(name, matches, "" if matches else "the diagram is not the graph's")
|
||||
|
||||
|
||||
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
|
||||
"""In an interactive Execution every Specialist confirms with the user before its
|
||||
first submission (spec risk D32)."""
|
||||
asked: set[str] = set()
|
||||
silent: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for run in timeline:
|
||||
deliverable = run.get("deliverable")
|
||||
if not deliverable:
|
||||
continue
|
||||
if run.get("outcome") == "NeedsInput":
|
||||
asked.add(deliverable)
|
||||
elif run.get("outcome") == "Submitted" and deliverable not in seen:
|
||||
seen.add(deliverable)
|
||||
if deliverable not in asked:
|
||||
silent.append(deliverable)
|
||||
return Check(
|
||||
"specialists_ask_before_submitting",
|
||||
bool(seen) and not silent,
|
||||
"submitted without asking: " + ", ".join(silent) if silent else "",
|
||||
)
|
||||
|
||||
|
||||
async def _checklist_coverage(
|
||||
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
|
||||
) -> Check:
|
||||
name = "pillar_checklists_covered"
|
||||
questions = {
|
||||
f"{pillar}.{index}": {
|
||||
"kind": "yes_no",
|
||||
"text": f"Do the recommendations address this {pillar} checklist item, "
|
||||
f"either by recommending it or by explaining why it does not apply? {item}",
|
||||
}
|
||||
for pillar, items in checklists.items()
|
||||
for index, item in enumerate(items)
|
||||
}
|
||||
if not recommendations or not questions:
|
||||
return Check(name, False, "no design recommendations or no checklists")
|
||||
scores = await judge.answer(recommendations, questions)
|
||||
per_pillar = {
|
||||
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
|
||||
/ len(items)
|
||||
for pillar, items in checklists.items()
|
||||
if items
|
||||
}
|
||||
weakest = min(per_pillar.values())
|
||||
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
|
||||
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
|
||||
|
||||
|
||||
async def _quality(judge: Judge, recommendations: str) -> Check:
|
||||
name = "recommendations_quality"
|
||||
if not recommendations:
|
||||
return Check(name, False, "no design recommendations")
|
||||
scores = await judge.answer(
|
||||
recommendations,
|
||||
{key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items()},
|
||||
)
|
||||
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
||||
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
||||
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
||||
|
||||
@@ -6,7 +6,12 @@ eval/scoring.py. Costs model usage: run deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
||||
.venv/bin/python -m eval.solution_eval [case-id ...]
|
||||
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
||||
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
|
||||
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
|
||||
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
||||
non-zero when a case fails or exceeds one of its thresholds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,6 +19,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
@@ -24,18 +30,25 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
||||
from app.agent import open_assembly
|
||||
from app.agent import ExecutionEngine, Interaction, open_assembly
|
||||
from app.config import Settings, get_settings
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.execution_state import execution_dir
|
||||
from app.tools.local import TerraformRunner
|
||||
from eval.judge import GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
checklist_items,
|
||||
load_cases,
|
||||
regressions,
|
||||
score_execution,
|
||||
template_headings,
|
||||
)
|
||||
|
||||
GEMINI_PREFIX = "gemini/"
|
||||
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
||||
RESULTS_DIR = ROOT / "results"
|
||||
@@ -96,7 +109,33 @@ def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
|
||||
async def run_with_scripted_user(
|
||||
engine: ExecutionEngine,
|
||||
case: CaseSpec,
|
||||
execution_id: str,
|
||||
max_user_turns: int = MAX_USER_TURNS,
|
||||
) -> Interaction:
|
||||
"""Run an interactive Execution, answering each prompt as the case scripts it:
|
||||
Approvals are granted, cloud reads denied, and questions answered by pattern."""
|
||||
interaction = await engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
for _ in range(max_user_turns):
|
||||
if interaction.status != "awaiting_user" or interaction.prompt is None:
|
||||
break
|
||||
reply = automatic_reply(
|
||||
interaction.prompt, [list(pair) for pair in case.scripted_answers]
|
||||
)
|
||||
interaction = await engine.resume(execution_id, reply)
|
||||
return interaction
|
||||
|
||||
|
||||
async def run_case(
|
||||
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
|
||||
) -> CaseResult:
|
||||
from app.adk.adk_runtime import AdkAgentRuntime
|
||||
from app.providers import load_provider
|
||||
|
||||
@@ -111,48 +150,84 @@ async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseRes
|
||||
async with open_assembly(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
interaction = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
||||
if case.interactive:
|
||||
interaction = await run_with_scripted_user(
|
||||
assembly.engine, case, execution_id
|
||||
)
|
||||
else:
|
||||
interaction = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
runner = TerraformRunner(
|
||||
binary=settings.TERRAFORM_BINARY,
|
||||
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
||||
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
|
||||
plugin_cache_dir=settings.terraform_plugin_cache(),
|
||||
)
|
||||
result = await score_execution(
|
||||
case, interaction, template_headings(bundle.output_template), runner.validate
|
||||
case,
|
||||
interaction,
|
||||
template_headings(bundle.output_template),
|
||||
runner.validate,
|
||||
judge=judge,
|
||||
pillar_checklists={
|
||||
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
|
||||
for pillar, skill in bundle.pillar_skills.items()
|
||||
},
|
||||
timeline=runtime.timeline,
|
||||
)
|
||||
ledger = interaction.state.get("ledger", {})
|
||||
tokens = token_report(adk_runtime.usage)
|
||||
revisions = {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
if item["revisions"]
|
||||
}
|
||||
details = {
|
||||
"execution_id": interaction.execution_id,
|
||||
"interactive": case.interactive,
|
||||
"regressions": regressions(
|
||||
case,
|
||||
{
|
||||
"revisions": sum(revisions.values()),
|
||||
"seconds": elapsed,
|
||||
"tokens": tokens["total"].get("total"),
|
||||
},
|
||||
),
|
||||
"seconds": round(elapsed, 1),
|
||||
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
||||
"orchestrator_turns": interaction.state.get("turns"),
|
||||
"tokens": token_report(adk_runtime.usage),
|
||||
"tokens": tokens,
|
||||
"timeline": runtime.timeline,
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"revisions": {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
if item["revisions"]
|
||||
},
|
||||
"revisions": revisions,
|
||||
"warnings": interaction.state.get("warnings", []),
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(
|
||||
run_dir / "deliverables" / "executions" / interaction.execution_id
|
||||
),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
}
|
||||
return CaseResult(result.case_id, result.status, result.checks, details)
|
||||
|
||||
|
||||
async def main(case_ids: Sequence[str]) -> int:
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
from app.providers import load_provider
|
||||
|
||||
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
||||
settings.CLOUD_PROVIDER
|
||||
).models.default.removeprefix(GEMINI_PREFIX)
|
||||
return GeminiJudge(model)
|
||||
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
use_judge = "--judge" in arguments
|
||||
case_ids = [argument for argument in arguments if argument != "--judge"]
|
||||
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
||||
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
||||
logging.basicConfig(
|
||||
@@ -168,11 +243,12 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = RESULTS_DIR / stamp
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
judge = make_judge(settings) if use_judge else None
|
||||
results: list[dict[str, Any]] = []
|
||||
for case in cases:
|
||||
print(f"== {case.id}: running", flush=True)
|
||||
try:
|
||||
result = await run_case(case, settings, run_dir)
|
||||
result = await run_case(case, settings, run_dir, judge)
|
||||
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
||||
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
||||
results.append(
|
||||
@@ -192,6 +268,8 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
f"revisions {details['revisions'] or 'none'}",
|
||||
flush=True,
|
||||
)
|
||||
for regression in details["regressions"]:
|
||||
print(f" REGRESSION {regression}", flush=True)
|
||||
for model, counts in details["tokens"]["by_model"].items():
|
||||
print(f" tokens {model}: {counts}", flush=True)
|
||||
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
||||
@@ -209,8 +287,12 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
report = run_dir / "report.json"
|
||||
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
||||
passed = sum(1 for r in results if r.get("passed"))
|
||||
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
|
||||
return 0 if passed == len(results) else 1
|
||||
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
|
||||
print(
|
||||
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
|
||||
f"Report: {report}"
|
||||
)
|
||||
return 0 if passed == len(results) and not regressed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user