latest refactoring
Some checks failed
validation / verify (push) Failing after 10s

This commit is contained in:
2026-10-06 20:22:37 +01:00
parent e94b2493e7
commit 7ee49ab352
414 changed files with 15768 additions and 3896 deletions

View File

@@ -1,26 +0,0 @@
[
{
"id": "case-01-event-driven-http",
"name": "Event-Driven Regional HTTP Workload",
"workflow_request": "Design an event-driven regional HTTP application that accepts incoming webhooks, validates signatures, durably enqueues payloads to Pub/Sub, and retains raw events in Cloud Storage for 30-day replay audit.",
"rubric": {
"required_products": ["Cloud Run", "Cloud Pub/Sub", "Cloud Storage", "Cloud IAM"],
"must_defer_in_discover": true,
"requires_mermaid": true,
"requires_terraform": true,
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
}
},
{
"id": "case-02-high-scale-ingestion",
"name": "High Scale Ingestion Pipeline",
"workflow_request": "Create a high-scale containerized data ingestion pipeline with least-privilege IAM service accounts and automated validation.",
"rubric": {
"required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
"must_defer_in_discover": true,
"requires_mermaid": true,
"requires_terraform": true,
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
}
}
]

View File

@@ -0,0 +1,198 @@
[
{
"id": "case-01-event-driven-http",
"name": "Event-driven regional webhook ingestion",
"workflow_request": "Design an event-driven regional HTTP application on Google Cloud that accepts incoming webhooks, validates their signatures, durably enqueues payloads for asynchronous processing, and retains raw events for 30 days for replay and audit.",
"required_products": [
"Cloud Run",
"Pub/Sub",
"Cloud Storage"
],
"scripted_answers": [
[
"region|location|geograph",
"A single region, europe-west2 (London). Data must stay in the UK."
],
[
"availability|uptime|sla|rto|rpo|disaster|recover",
"99.9% monthly availability. RTO 1 hour, RPO 5 minutes for accepted events. No multi-region DR in phase one."
],
[
"traffic|volume|throughput|scale|peak|requests per|rps|payload",
"Average 50 webhooks per second, peaks of 500 per second for 10 minutes. Payloads up to 256 KB."
],
[
"latency|response time|performance",
"Acknowledge each webhook within 300 ms at p95."
],
[
"security|compliance|encrypt|privacy|gdpr|pci|access|identity|auth",
"UK GDPR applies. Google-managed encryption keys are acceptable. Senders sign requests with HMAC-SHA256 shared secrets; only processing services may read events."
],
[
"cost|budget|spend",
"Under 1,500 GBP a month at average load; prefer pay-per-use."
],
[
"current|existing|today|on-prem|migrat|legacy",
"This is a new workload; nothing runs today."
],
[
"depend|integrat|downstream|upstream|consum",
"Processed events go to an existing internal REST API that is reachable over HTTPS; it can be slow or unavailable for minutes."
],
[
"operat|monitor|logging|alert|team|support",
"A small platform team of three uses Terraform and wants managed services and Cloud Monitoring alerts."
],
[
"sustainab|carbon",
"Prefer low-carbon regions when there is no other constraint, but the UK residency requirement wins."
],
[
"retention|replay|audit|archive",
"Keep raw events 30 days, then delete. Replay must be possible for any event in that window."
],
[
"approve|confirm|proceed|look right|any changes|happy",
"Approved. Please proceed."
]
]
},
{
"id": "case-02-high-scale-ingestion",
"name": "High-scale containerised data ingestion pipeline",
"workflow_request": "Create a high-scale containerised data ingestion pipeline on Google Cloud with least-privilege service identities and automated validation of incoming records.",
"required_products": [
"Pub/Sub",
"Cloud Storage",
"BigQuery"
],
"scripted_answers": [
[
"region|location|geograph",
"us-central1, single region."
],
[
"source|producer|incoming|where.*data|format|schema",
"About 200 IoT gateways send JSON records over HTTPS; each record has a published JSON schema version."
],
[
"availability|uptime|sla|rto|rpo|disaster|recover",
"99.5% availability; RTO 4 hours, RPO 15 minutes. Losing accepted records is not acceptable."
],
[
"traffic|volume|throughput|scale|peak|records per|rps|size",
"2 TB a day, about 20,000 records per second at peak, records under 10 KB."
],
[
"latency|fresh|real.?time|batch",
"Records must be queryable for analytics within 5 minutes of arrival."
],
[
"security|compliance|encrypt|privacy|access|identity|auth|least",
"No regulated data. Each component gets its own service account with least privilege; gateways authenticate with per-device credentials."
],
[
"validat|invalid|reject|quarantine|dead",
"Invalid records go to a quarantine location with the validation error, and are reprocessable after a fix."
],
[
"cost|budget|spend",
"Around 8,000 USD a month; optimise storage costs for data older than 90 days."
],
[
"current|existing|today|on-prem|migrat|legacy",
"Today a single Kafka VM on premises collects records; it is being retired."
],
[
"analytic|query|consum|report|downstream|depend",
"Analysts query the data with SQL; dashboards refresh every 15 minutes."
],
[
"operat|monitor|logging|alert|team|support|container|kubernetes|gke",
"The team runs containers and prefers serverless containers over managing Kubernetes clusters."
],
[
"retention|archive|keep",
"Keep raw records one year and curated tables three years."
],
[
"approve|confirm|proceed|look right|any changes|happy",
"Approved. Please proceed."
]
]
},
{
"id": "case-03-internal-web-app",
"name": "Internal booking web application with a relational database",
"workflow_request": "Design an internal web application on Google Cloud for staff to book meeting rooms and equipment, backed by a managed relational database, available only to employees signed in with the corporate identity provider, with application secrets kept out of code and configuration.",
"required_products": [
"Cloud Run",
"Cloud SQL",
"Secret Manager"
],
"scripted_answers": [
[
"region|location|geograph",
"A single region, europe-west1 (Belgium). All users are in the EU."
],
[
"user|staff|employee|how many|concurren|traffic|volume|scale|peak|requests per|rps",
"About 4,000 employees; 300 concurrent users at the 09:00 peak, around 20 requests per second."
],
[
"availability|uptime|sla|rto|rpo|disaster|recover|backup",
"99.5% during business hours (07:00-19:00 CET). RTO 4 hours, RPO 1 hour. Daily backups kept 14 days."
],
[
"latency|response time|performance",
"Pages should load within 1 second at p95."
],
[
"identity|sign.?in|login|sso|auth|idp|access",
"Employees sign in with Google Workspace accounts; no access from outside the company. Admins are a Google group."
],
[
"security|compliance|encrypt|privacy|gdpr|secret|credential|password",
"GDPR applies to employee names and emails. Google-managed keys are fine. Database passwords and API keys must live in a secrets store and rotate every 90 days."
],
[
"data|database|schema|relational|postgres|mysql|size|storage",
"PostgreSQL. Under 20 GB of data in the first three years; bookings, rooms, equipment and audit history."
],
[
"cost|budget|spend",
"Under 600 EUR a month; the database can be small."
],
[
"current|existing|today|on-prem|migrat|legacy",
"Bookings are kept in a shared spreadsheet today; there is no data to migrate."
],
[
"depend|integrat|calendar|email|notif|downstream|upstream",
"Send booking confirmation emails through the existing SMTP relay; no calendar integration in phase one."
],
[
"operat|monitor|logging|alert|team|support|deploy|ci|pipeline",
"Two developers own it, deploy containers from Cloud Build with Terraform, and want Cloud Monitoring alerts to a Google Chat space."
],
[
"network|vpc|private|internet|egress|ingress",
"No public database endpoint; the database must use private IP only."
],
[
"sustainab|carbon",
"No specific requirement."
],
[
"retention|audit|archive|delete",
"Keep booking history 2 years, then delete; keep audit logs 1 year."
],
[
"approve|confirm|proceed|look right|any changes|happy",
"Approved. Please proceed."
]
]
}
]

View File

@@ -1,73 +0,0 @@
"""Evaluation Harness Runner for GCP Solution Architecture Agent.
Uses google.adk.evaluation.Evaluator with PostgreSQL database metrics persistence.
"""
import json
import logging
from pathlib import Path
from typing import Any, Dict, List
from app.adk.evaluation import ADKEvaluator
from app.config import get_settings
logger = logging.getLogger(__name__)
class EvalHarness:
"""Offline Evaluation Harness for running benchmark suites against ADK agents."""
def __init__(self, dataset_path: Path | None = None) -> None:
settings = get_settings()
self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH
self.evaluator = ADKEvaluator()
def load_benchmark_cases(self) -> List[Dict[str, Any]]:
"""Load benchmark dataset JSON."""
if not self.dataset_path.is_file():
logger.error("Benchmark dataset not found at %s", self.dataset_path)
return []
with open(self.dataset_path, "r", encoding="utf-8") as f:
return json.load(f)
def run_eval_suite(self) -> Dict[str, Any]:
"""Execute all benchmark test cases through ADKEvaluator and compile scoring metrics."""
cases = self.load_benchmark_cases()
if not cases:
return {"status": "error", "message": "No benchmark cases loaded."}
results = []
total_passed = 0
for case in cases:
logger.info("Evaluating ADK benchmark case: %s", case.get("id"))
eval_result = self.evaluator.evaluate_benchmark_case(case)
results.append(eval_result)
if eval_result.get("passed"):
total_passed += 1
pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0
summary = {
"total_cases": len(cases),
"passed_cases": total_passed,
"failed_cases": len(cases) - total_passed,
"pass_rate_percentage": pass_rate,
"results": results,
}
return summary
def main() -> None:
"""CLI Runner for Evaluation Harness."""
harness = EvalHarness()
summary = harness.run_eval_suite()
print("=== GCP Solution Architecture Agent ADK Benchmark Summary ===")
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
main()

View File

@@ -1,100 +0,0 @@
"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
import re
from typing import Any, Dict, List
def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
"""Score Phase 0 Requirements Discovery output."""
score = 0.0
feedback = []
if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
score += 0.5
else:
feedback.append("Product selection was not explicitly deferred in discovery phase.")
if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
score += 0.5
else:
feedback.append("Missing functional or non-functional requirement sections.")
return {"score": score, "max_score": 1.0, "feedback": feedback}
def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
"""Score Phase 1 Architecture & Product Selection output."""
score = 0.0
max_score = 3.0
feedback = []
# Check product choices
required_prods = rubric.get("required_products", [])
found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
if len(found_prods) == len(required_prods):
score += 1.0
else:
missing = set(required_prods) - set(found_prods)
feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
# Check Mermaid diagram
if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
score += 1.0
else:
feedback.append("Invalid or missing Mermaid diagram syntax.")
# Check Terraform code
if "resource" in terraform_code and "google_" in terraform_code:
score += 1.0
else:
feedback.append("Terraform code does not contain Google Cloud resources.")
return {"score": score, "max_score": max_score, "feedback": feedback}
def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
"""Score Phase 3 Guide Packaging output."""
score = 0.0
max_score = 1.0
feedback = []
required_sections = rubric.get("requires_guide_sections", [])
found_sections = [sec for sec in required_sections if sec in solution_guide]
if len(found_sections) == len(required_sections):
score += 1.0
else:
missing = set(required_sections) - set(found_sections)
feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
return {"score": score, "max_score": max_score, "feedback": feedback}
def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
"""Run full evaluation suite for a benchmark case output."""
rubric = case.get("rubric", {})
req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
des_eval = evaluate_design_phase(
final_state.get("architecture_doc", ""),
final_state.get("mermaid_diagram", ""),
final_state.get("terraform_code", ""),
rubric,
)
pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
percentage = (total_score / total_max) * 100.0
all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
return {
"case_id": case.get("id"),
"case_name": case.get("name"),
"total_score": total_score,
"max_score": total_max,
"percentage": percentage,
"passed": percentage >= 80.0,
"feedback": all_feedback,
}

View File

@@ -1,58 +0,0 @@
"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
import logging
from typing import Any, Dict, List
from eval.metrics import evaluate_case_run
logger = logging.getLogger(__name__)
class SkillOptimizer:
"""Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Generate prompt tuning recommendations based on evaluation failures."""
recommendations = []
results = eval_summary.get("results", [])
for res in results:
if not res.get("passed"):
feedback_items = res.get("feedback", [])
for feedback in feedback_items:
rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
if rec:
recommendations.append(rec)
return recommendations
def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
"""Maps specific feedback strings to target SKILL.md prompt enhancements."""
if "deferred" in feedback.lower():
return {
"target_skill": "requirements_discovery",
"phase": "discover",
"issue": feedback,
"recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
}
elif "mermaid" in feedback.lower():
return {
"target_skill": "architecture_design",
"phase": "design",
"issue": feedback,
"recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
}
elif "terraform" in feedback.lower():
return {
"target_skill": "architecture_design",
"phase": "design",
"issue": feedback,
"recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
}
elif "section" in feedback.lower():
return {
"target_skill": "packaging_guide",
"phase": "package",
"issue": feedback,
"recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
}
return None

207
eval/scoring.py Normal file
View File

@@ -0,0 +1,207 @@
"""Score a finished Execution against a benchmark case.
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
"""
from __future__ import annotations
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.local import mermaid_check
from app.workflows.deliverables import DeliverableKind
PASS_THRESHOLD = 0.8
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
@dataclass(frozen=True)
class CaseSpec:
id: str
name: str
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, str], ...]
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass(frozen=True)
class CaseResult:
case_id: str
status: str
checks: tuple[Check, ...]
details: Mapping[str, Any] = field(default_factory=dict)
@property
def score(self) -> float:
return (
sum(c.passed for c in self.checks) / len(self.checks)
if self.checks
else 0.0
)
@property
def passed(self) -> bool:
return self.score >= PASS_THRESHOLD
def load_cases(path: Path) -> list[CaseSpec]:
import json
return [
CaseSpec(
id=item["id"],
name=item["name"],
workflow_request=item["workflow_request"],
required_products=tuple(item["required_products"]),
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
def template_headings(template: Path) -> list[str]:
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
return [
heading
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
if "[" not in heading
]
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
def _mentions(text: str, product: str) -> bool:
return (
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
is not None
)
async def score_execution(
case: CaseSpec,
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
)
def content(kind: DeliverableKind) -> str:
item = deliverables.get(kind.value)
return item["content"] if item else ""
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
item = deliverables.get(kind.value)
return item.get("attachments", {}) if item else {}
checks: list[Check] = []
checks.append(
Check(
"completed",
interaction.status == "completed",
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
)
)
guide = content(DeliverableKind.SOLUTION_GUIDE)
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
checks.append(
Check(
"guide_follows_template",
bool(guide) and not missing_sections,
"missing: " + "; ".join(missing_sections) if missing_sections else "",
)
)
selection = content(DeliverableKind.PRODUCT_SELECTION)
selected_text = selection + "\n" + guide
missing_products = [
p for p in case.required_products if not _mentions(selected_text, p)
]
checks.append(
Check(
"required_products_selected",
bool(selection) and not missing_products,
"missing: " + ", ".join(missing_products) if missing_products else "",
)
)
lowered = selection.lower()
checks.append(
Check(
"selection_has_alternatives",
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
)
)
checks.append(
Check(
"selection_cites_documentation",
bool(_DOC_LINK.search(selection)),
)
)
requirements = content(DeliverableKind.REQUIREMENTS)
leaked = [
product
for product in case.required_products
if _mentions(requirements, product)
and not _mentions(case.workflow_request, product)
]
checks.append(
Check(
"requirements_name_no_products",
bool(requirements) and not leaked,
"names: " + ", ".join(leaked) if leaked else "",
)
)
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
checks.append(
Check(
"diagram_is_valid_mermaid",
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
mermaid.message,
)
)
terraform_files = {
name.removeprefix("terraform/"): text
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
if name.endswith((".tf", ".tfvars", ".tf.json"))
}
if terraform_files:
validation = await terraform_validate(terraform_files)
terraform_ok = validation.message.startswith("terraform validate passed")
terraform_detail = validation.message[:1500]
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
return CaseResult(case.id, interaction.status, tuple(checks))

217
eval/solution_eval.py Normal file
View File

@@ -0,0 +1,217 @@
"""Evaluate the solution architect agent on benchmark cases with real models.
Each case runs a pre-approved Execution with scripted user answers through the full
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
eval/scoring.py. Costs model usage: run deliberately.
Usage:
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
.venv/bin/python -m eval.solution_eval [case-id ...]
"""
from __future__ import annotations
import asyncio
import json
import logging
import sys
import time
from collections import Counter
from collections.abc import Sequence
from dataclasses import asdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
from app.agent import open_assembly
from app.config import Settings, get_settings
from app.recorder import InMemoryRecorder
from app.tools.local import TerraformRunner
from eval.scoring import (
CaseResult,
CaseSpec,
load_cases,
score_execution,
template_headings,
)
ROOT = Path(__file__).resolve().parent
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
RESULTS_DIR = ROOT / "results"
class CountingRuntime:
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
def __init__(self, inner: AgentRuntime) -> None:
self._inner = inner
self._started = time.monotonic()
self.runs: Counter[str] = Counter()
self.outcomes: Counter[str] = Counter()
self.timeline: list[dict[str, Any]] = []
async def run(
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
) -> Outcome:
self.runs[role.name] += 1
start = time.monotonic()
outcome = await self._inner.run(role, task, tools)
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
self.timeline.append(
{
"role": role.name,
"deliverable": task.deliverable,
"start_s": round(start - self._started, 1),
"seconds": round(time.monotonic() - start, 1),
"outcome": type(outcome).__name__,
}
)
return outcome
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
"""Seconds until the first validation_results run ended: the run without revisions."""
for run in timeline:
if run["deliverable"] == "validation_results":
return round(run["start_s"] + run["seconds"], 1)
return None
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
by_role = {
f"{role} ({model})": asdict(counts)
for (role, model), counts in sorted(usage.items())
}
by_model: dict[str, Counter[str]] = {}
for (_, model), counts in usage.items():
by_model.setdefault(model, Counter()).update(asdict(counts))
total: Counter[str] = Counter()
for counts in by_model.values():
total.update(counts)
return {
"by_role": by_role,
"by_model": {model: dict(c) for model, c in by_model.items()},
"total": dict(total),
}
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
from app.adk.adk_runtime import AdkAgentRuntime
from app.providers import load_provider
bundle = load_provider(settings.CLOUD_PROVIDER)
adk_runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
runtime = CountingRuntime(adk_runtime)
recorder = InMemoryRecorder()
started = time.monotonic()
async with open_assembly(
settings, runtime=runtime, recorder=recorder, output_root=run_dir
) as assembly:
interaction = await assembly.engine.start(
case.workflow_request,
pre_approved=True,
allow_cloud_read=False,
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
scripted_answers=case.scripted_answers,
)
elapsed = time.monotonic() - started
runner = TerraformRunner(
binary=settings.TERRAFORM_BINARY,
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
)
result = await score_execution(
case, interaction, template_headings(bundle.output_template), runner.validate
)
ledger = interaction.state.get("ledger", {})
details = {
"execution_id": interaction.execution_id,
"seconds": round(elapsed, 1),
"first_pass_seconds": first_pass_seconds(runtime.timeline),
"orchestrator_turns": interaction.state.get("turns"),
"tokens": token_report(adk_runtime.usage),
"timeline": runtime.timeline,
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"revisions": {
kind: item["revisions"]
for kind, item in ledger.get("deliverables", {}).items()
if item["revisions"]
},
"warnings": interaction.state.get("warnings", []),
"conversation": interaction.state.get("conversation", []),
"output_dir": str(
run_dir / "deliverables" / "executions" / interaction.execution_id
),
}
return CaseResult(result.case_id, result.status, result.checks, details)
async def main(case_ids: Sequence[str]) -> int:
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
logging.basicConfig(
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
)
logging.getLogger("app").setLevel(logging.INFO)
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
if not cases:
print(f"No cases match {list(case_ids)}", file=sys.stderr)
return 2
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
run_dir = RESULTS_DIR / stamp
run_dir.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
for case in cases:
print(f"== {case.id}: running", flush=True)
try:
result = await run_case(case, settings, run_dir)
except Exception as exc: # noqa: BLE001 - record the failure and continue
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
results.append(
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
)
continue
print(
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
flush=True,
)
for check in result.checks:
mark = "ok " if check.passed else "FAIL"
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
details = result.details
print(
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
f"revisions {details['revisions'] or 'none'}",
flush=True,
)
for model, counts in details["tokens"]["by_model"].items():
print(f" tokens {model}: {counts}", flush=True)
print(f" tokens total: {details['tokens']['total']}", flush=True)
results.append(
{
"case_id": result.case_id,
"status": result.status,
"score": result.score,
"passed": result.passed,
"checks": [asdict(check) for check in result.checks],
"details": result.details,
}
)
report = run_dir / "report.json"
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
passed = sum(1 for r in results if r.get("passed"))
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
return 0 if passed == len(results) else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))

View File

@@ -1,32 +0,0 @@
"""Tests for Evaluation Harness and Metrics."""
import pytest
from eval.eval_harness import EvalHarness
from eval.optimizer import SkillOptimizer
def test_eval_harness_benchmark():
"""Run EvalHarness against benchmark cases."""
harness = EvalHarness()
summary = harness.run_eval_suite()
assert summary["total_cases"] >= 1
assert summary["passed_cases"] >= 1
assert summary["pass_rate_percentage"] == 100.0
def test_skill_optimizer_recommendations():
"""Verify optimizer generates tuning recommendations for simulated failures."""
optimizer = SkillOptimizer()
simulated_summary = {
"results": [
{
"case_id": "test-case",
"passed": False,
"feedback": ["Product selection was not explicitly deferred in discovery phase."],
}
]
}
recs = optimizer.generate_tuning_recommendations(simulated_summary)
assert len(recs) == 1
assert recs[0]["target_skill"] == "requirements_discovery"