This commit is contained in:
@@ -1,26 +0,0 @@
|
||||
[
|
||||
{
|
||||
"id": "case-01-event-driven-http",
|
||||
"name": "Event-Driven Regional HTTP Workload",
|
||||
"workflow_request": "Design an event-driven regional HTTP application that accepts incoming webhooks, validates signatures, durably enqueues payloads to Pub/Sub, and retains raw events in Cloud Storage for 30-day replay audit.",
|
||||
"rubric": {
|
||||
"required_products": ["Cloud Run", "Cloud Pub/Sub", "Cloud Storage", "Cloud IAM"],
|
||||
"must_defer_in_discover": true,
|
||||
"requires_mermaid": true,
|
||||
"requires_terraform": true,
|
||||
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "case-02-high-scale-ingestion",
|
||||
"name": "High Scale Ingestion Pipeline",
|
||||
"workflow_request": "Create a high-scale containerized data ingestion pipeline with least-privilege IAM service accounts and automated validation.",
|
||||
"rubric": {
|
||||
"required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
|
||||
"must_defer_in_discover": true,
|
||||
"requires_mermaid": true,
|
||||
"requires_terraform": true,
|
||||
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
|
||||
}
|
||||
}
|
||||
]
|
||||
198
eval/datasets/solution_cases.json
Normal file
198
eval/datasets/solution_cases.json
Normal file
@@ -0,0 +1,198 @@
|
||||
[
|
||||
{
|
||||
"id": "case-01-event-driven-http",
|
||||
"name": "Event-driven regional webhook ingestion",
|
||||
"workflow_request": "Design an event-driven regional HTTP application on Google Cloud that accepts incoming webhooks, validates their signatures, durably enqueues payloads for asynchronous processing, and retains raw events for 30 days for replay and audit.",
|
||||
"required_products": [
|
||||
"Cloud Run",
|
||||
"Pub/Sub",
|
||||
"Cloud Storage"
|
||||
],
|
||||
"scripted_answers": [
|
||||
[
|
||||
"region|location|geograph",
|
||||
"A single region, europe-west2 (London). Data must stay in the UK."
|
||||
],
|
||||
[
|
||||
"availability|uptime|sla|rto|rpo|disaster|recover",
|
||||
"99.9% monthly availability. RTO 1 hour, RPO 5 minutes for accepted events. No multi-region DR in phase one."
|
||||
],
|
||||
[
|
||||
"traffic|volume|throughput|scale|peak|requests per|rps|payload",
|
||||
"Average 50 webhooks per second, peaks of 500 per second for 10 minutes. Payloads up to 256 KB."
|
||||
],
|
||||
[
|
||||
"latency|response time|performance",
|
||||
"Acknowledge each webhook within 300 ms at p95."
|
||||
],
|
||||
[
|
||||
"security|compliance|encrypt|privacy|gdpr|pci|access|identity|auth",
|
||||
"UK GDPR applies. Google-managed encryption keys are acceptable. Senders sign requests with HMAC-SHA256 shared secrets; only processing services may read events."
|
||||
],
|
||||
[
|
||||
"cost|budget|spend",
|
||||
"Under 1,500 GBP a month at average load; prefer pay-per-use."
|
||||
],
|
||||
[
|
||||
"current|existing|today|on-prem|migrat|legacy",
|
||||
"This is a new workload; nothing runs today."
|
||||
],
|
||||
[
|
||||
"depend|integrat|downstream|upstream|consum",
|
||||
"Processed events go to an existing internal REST API that is reachable over HTTPS; it can be slow or unavailable for minutes."
|
||||
],
|
||||
[
|
||||
"operat|monitor|logging|alert|team|support",
|
||||
"A small platform team of three uses Terraform and wants managed services and Cloud Monitoring alerts."
|
||||
],
|
||||
[
|
||||
"sustainab|carbon",
|
||||
"Prefer low-carbon regions when there is no other constraint, but the UK residency requirement wins."
|
||||
],
|
||||
[
|
||||
"retention|replay|audit|archive",
|
||||
"Keep raw events 30 days, then delete. Replay must be possible for any event in that window."
|
||||
],
|
||||
[
|
||||
"approve|confirm|proceed|look right|any changes|happy",
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "case-02-high-scale-ingestion",
|
||||
"name": "High-scale containerised data ingestion pipeline",
|
||||
"workflow_request": "Create a high-scale containerised data ingestion pipeline on Google Cloud with least-privilege service identities and automated validation of incoming records.",
|
||||
"required_products": [
|
||||
"Pub/Sub",
|
||||
"Cloud Storage",
|
||||
"BigQuery"
|
||||
],
|
||||
"scripted_answers": [
|
||||
[
|
||||
"region|location|geograph",
|
||||
"us-central1, single region."
|
||||
],
|
||||
[
|
||||
"source|producer|incoming|where.*data|format|schema",
|
||||
"About 200 IoT gateways send JSON records over HTTPS; each record has a published JSON schema version."
|
||||
],
|
||||
[
|
||||
"availability|uptime|sla|rto|rpo|disaster|recover",
|
||||
"99.5% availability; RTO 4 hours, RPO 15 minutes. Losing accepted records is not acceptable."
|
||||
],
|
||||
[
|
||||
"traffic|volume|throughput|scale|peak|records per|rps|size",
|
||||
"2 TB a day, about 20,000 records per second at peak, records under 10 KB."
|
||||
],
|
||||
[
|
||||
"latency|fresh|real.?time|batch",
|
||||
"Records must be queryable for analytics within 5 minutes of arrival."
|
||||
],
|
||||
[
|
||||
"security|compliance|encrypt|privacy|access|identity|auth|least",
|
||||
"No regulated data. Each component gets its own service account with least privilege; gateways authenticate with per-device credentials."
|
||||
],
|
||||
[
|
||||
"validat|invalid|reject|quarantine|dead",
|
||||
"Invalid records go to a quarantine location with the validation error, and are reprocessable after a fix."
|
||||
],
|
||||
[
|
||||
"cost|budget|spend",
|
||||
"Around 8,000 USD a month; optimise storage costs for data older than 90 days."
|
||||
],
|
||||
[
|
||||
"current|existing|today|on-prem|migrat|legacy",
|
||||
"Today a single Kafka VM on premises collects records; it is being retired."
|
||||
],
|
||||
[
|
||||
"analytic|query|consum|report|downstream|depend",
|
||||
"Analysts query the data with SQL; dashboards refresh every 15 minutes."
|
||||
],
|
||||
[
|
||||
"operat|monitor|logging|alert|team|support|container|kubernetes|gke",
|
||||
"The team runs containers and prefers serverless containers over managing Kubernetes clusters."
|
||||
],
|
||||
[
|
||||
"retention|archive|keep",
|
||||
"Keep raw records one year and curated tables three years."
|
||||
],
|
||||
[
|
||||
"approve|confirm|proceed|look right|any changes|happy",
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "case-03-internal-web-app",
|
||||
"name": "Internal booking web application with a relational database",
|
||||
"workflow_request": "Design an internal web application on Google Cloud for staff to book meeting rooms and equipment, backed by a managed relational database, available only to employees signed in with the corporate identity provider, with application secrets kept out of code and configuration.",
|
||||
"required_products": [
|
||||
"Cloud Run",
|
||||
"Cloud SQL",
|
||||
"Secret Manager"
|
||||
],
|
||||
"scripted_answers": [
|
||||
[
|
||||
"region|location|geograph",
|
||||
"A single region, europe-west1 (Belgium). All users are in the EU."
|
||||
],
|
||||
[
|
||||
"user|staff|employee|how many|concurren|traffic|volume|scale|peak|requests per|rps",
|
||||
"About 4,000 employees; 300 concurrent users at the 09:00 peak, around 20 requests per second."
|
||||
],
|
||||
[
|
||||
"availability|uptime|sla|rto|rpo|disaster|recover|backup",
|
||||
"99.5% during business hours (07:00-19:00 CET). RTO 4 hours, RPO 1 hour. Daily backups kept 14 days."
|
||||
],
|
||||
[
|
||||
"latency|response time|performance",
|
||||
"Pages should load within 1 second at p95."
|
||||
],
|
||||
[
|
||||
"identity|sign.?in|login|sso|auth|idp|access",
|
||||
"Employees sign in with Google Workspace accounts; no access from outside the company. Admins are a Google group."
|
||||
],
|
||||
[
|
||||
"security|compliance|encrypt|privacy|gdpr|secret|credential|password",
|
||||
"GDPR applies to employee names and emails. Google-managed keys are fine. Database passwords and API keys must live in a secrets store and rotate every 90 days."
|
||||
],
|
||||
[
|
||||
"data|database|schema|relational|postgres|mysql|size|storage",
|
||||
"PostgreSQL. Under 20 GB of data in the first three years; bookings, rooms, equipment and audit history."
|
||||
],
|
||||
[
|
||||
"cost|budget|spend",
|
||||
"Under 600 EUR a month; the database can be small."
|
||||
],
|
||||
[
|
||||
"current|existing|today|on-prem|migrat|legacy",
|
||||
"Bookings are kept in a shared spreadsheet today; there is no data to migrate."
|
||||
],
|
||||
[
|
||||
"depend|integrat|calendar|email|notif|downstream|upstream",
|
||||
"Send booking confirmation emails through the existing SMTP relay; no calendar integration in phase one."
|
||||
],
|
||||
[
|
||||
"operat|monitor|logging|alert|team|support|deploy|ci|pipeline",
|
||||
"Two developers own it, deploy containers from Cloud Build with Terraform, and want Cloud Monitoring alerts to a Google Chat space."
|
||||
],
|
||||
[
|
||||
"network|vpc|private|internet|egress|ingress",
|
||||
"No public database endpoint; the database must use private IP only."
|
||||
],
|
||||
[
|
||||
"sustainab|carbon",
|
||||
"No specific requirement."
|
||||
],
|
||||
[
|
||||
"retention|audit|archive|delete",
|
||||
"Keep booking history 2 years, then delete; keep audit logs 1 year."
|
||||
],
|
||||
[
|
||||
"approve|confirm|proceed|look right|any changes|happy",
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,73 +0,0 @@
|
||||
"""Evaluation Harness Runner for GCP Solution Architecture Agent.
|
||||
|
||||
Uses google.adk.evaluation.Evaluator with PostgreSQL database metrics persistence.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.adk.evaluation import ADKEvaluator
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalHarness:
|
||||
"""Offline Evaluation Harness for running benchmark suites against ADK agents."""
|
||||
|
||||
def __init__(self, dataset_path: Path | None = None) -> None:
|
||||
settings = get_settings()
|
||||
self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH
|
||||
self.evaluator = ADKEvaluator()
|
||||
|
||||
def load_benchmark_cases(self) -> List[Dict[str, Any]]:
|
||||
"""Load benchmark dataset JSON."""
|
||||
if not self.dataset_path.is_file():
|
||||
logger.error("Benchmark dataset not found at %s", self.dataset_path)
|
||||
return []
|
||||
|
||||
with open(self.dataset_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
def run_eval_suite(self) -> Dict[str, Any]:
|
||||
"""Execute all benchmark test cases through ADKEvaluator and compile scoring metrics."""
|
||||
cases = self.load_benchmark_cases()
|
||||
if not cases:
|
||||
return {"status": "error", "message": "No benchmark cases loaded."}
|
||||
|
||||
results = []
|
||||
total_passed = 0
|
||||
|
||||
for case in cases:
|
||||
logger.info("Evaluating ADK benchmark case: %s", case.get("id"))
|
||||
eval_result = self.evaluator.evaluate_benchmark_case(case)
|
||||
results.append(eval_result)
|
||||
|
||||
if eval_result.get("passed"):
|
||||
total_passed += 1
|
||||
|
||||
pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0
|
||||
|
||||
summary = {
|
||||
"total_cases": len(cases),
|
||||
"passed_cases": total_passed,
|
||||
"failed_cases": len(cases) - total_passed,
|
||||
"pass_rate_percentage": pass_rate,
|
||||
"results": results,
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI Runner for Evaluation Harness."""
|
||||
harness = EvalHarness()
|
||||
summary = harness.run_eval_suite()
|
||||
print("=== GCP Solution Architecture Agent ADK Benchmark Summary ===")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
100
eval/metrics.py
100
eval/metrics.py
@@ -1,100 +0,0 @@
|
||||
"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
|
||||
"""Score Phase 0 Requirements Discovery output."""
|
||||
score = 0.0
|
||||
feedback = []
|
||||
|
||||
if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
|
||||
score += 0.5
|
||||
else:
|
||||
feedback.append("Product selection was not explicitly deferred in discovery phase.")
|
||||
|
||||
if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
|
||||
score += 0.5
|
||||
else:
|
||||
feedback.append("Missing functional or non-functional requirement sections.")
|
||||
|
||||
return {"score": score, "max_score": 1.0, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Score Phase 1 Architecture & Product Selection output."""
|
||||
score = 0.0
|
||||
max_score = 3.0
|
||||
feedback = []
|
||||
|
||||
# Check product choices
|
||||
required_prods = rubric.get("required_products", [])
|
||||
found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
|
||||
if len(found_prods) == len(required_prods):
|
||||
score += 1.0
|
||||
else:
|
||||
missing = set(required_prods) - set(found_prods)
|
||||
feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
|
||||
|
||||
# Check Mermaid diagram
|
||||
if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
|
||||
score += 1.0
|
||||
else:
|
||||
feedback.append("Invalid or missing Mermaid diagram syntax.")
|
||||
|
||||
# Check Terraform code
|
||||
if "resource" in terraform_code and "google_" in terraform_code:
|
||||
score += 1.0
|
||||
else:
|
||||
feedback.append("Terraform code does not contain Google Cloud resources.")
|
||||
|
||||
return {"score": score, "max_score": max_score, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Score Phase 3 Guide Packaging output."""
|
||||
score = 0.0
|
||||
max_score = 1.0
|
||||
feedback = []
|
||||
|
||||
required_sections = rubric.get("requires_guide_sections", [])
|
||||
found_sections = [sec for sec in required_sections if sec in solution_guide]
|
||||
|
||||
if len(found_sections) == len(required_sections):
|
||||
score += 1.0
|
||||
else:
|
||||
missing = set(required_sections) - set(found_sections)
|
||||
feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
|
||||
|
||||
return {"score": score, "max_score": max_score, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Run full evaluation suite for a benchmark case output."""
|
||||
rubric = case.get("rubric", {})
|
||||
|
||||
req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
|
||||
des_eval = evaluate_design_phase(
|
||||
final_state.get("architecture_doc", ""),
|
||||
final_state.get("mermaid_diagram", ""),
|
||||
final_state.get("terraform_code", ""),
|
||||
rubric,
|
||||
)
|
||||
pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
|
||||
|
||||
total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
|
||||
total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
|
||||
percentage = (total_score / total_max) * 100.0
|
||||
|
||||
all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
|
||||
|
||||
return {
|
||||
"case_id": case.get("id"),
|
||||
"case_name": case.get("name"),
|
||||
"total_score": total_score,
|
||||
"max_score": total_max,
|
||||
"percentage": percentage,
|
||||
"passed": percentage >= 80.0,
|
||||
"feedback": all_feedback,
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
from eval.metrics import evaluate_case_run
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillOptimizer:
|
||||
"""Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
|
||||
|
||||
def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Generate prompt tuning recommendations based on evaluation failures."""
|
||||
recommendations = []
|
||||
results = eval_summary.get("results", [])
|
||||
|
||||
for res in results:
|
||||
if not res.get("passed"):
|
||||
feedback_items = res.get("feedback", [])
|
||||
for feedback in feedback_items:
|
||||
rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
|
||||
if rec:
|
||||
recommendations.append(rec)
|
||||
|
||||
return recommendations
|
||||
|
||||
def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
|
||||
"""Maps specific feedback strings to target SKILL.md prompt enhancements."""
|
||||
if "deferred" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "requirements_discovery",
|
||||
"phase": "discover",
|
||||
"issue": feedback,
|
||||
"recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
|
||||
}
|
||||
elif "mermaid" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "architecture_design",
|
||||
"phase": "design",
|
||||
"issue": feedback,
|
||||
"recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
|
||||
}
|
||||
elif "terraform" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "architecture_design",
|
||||
"phase": "design",
|
||||
"issue": feedback,
|
||||
"recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
|
||||
}
|
||||
elif "section" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "packaging_guide",
|
||||
"phase": "package",
|
||||
"issue": feedback,
|
||||
"recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
|
||||
}
|
||||
return None
|
||||
207
eval/scoring.py
Normal file
207
eval/scoring.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""Score a finished Execution against a benchmark case.
|
||||
|
||||
Each check is a small, explainable rule tied to the upstream Skill's requirements.
|
||||
A case passes when its score reaches PASS_THRESHOLD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
PASS_THRESHOLD = 0.8
|
||||
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
|
||||
|
||||
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
|
||||
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseSpec:
|
||||
id: str
|
||||
name: str
|
||||
workflow_request: str
|
||||
required_products: tuple[str, ...]
|
||||
scripted_answers: tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
passed: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseResult:
|
||||
case_id: str
|
||||
status: str
|
||||
checks: tuple[Check, ...]
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def score(self) -> float:
|
||||
return (
|
||||
sum(c.passed for c in self.checks) / len(self.checks)
|
||||
if self.checks
|
||||
else 0.0
|
||||
)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.score >= PASS_THRESHOLD
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[CaseSpec]:
|
||||
import json
|
||||
|
||||
return [
|
||||
CaseSpec(
|
||||
id=item["id"],
|
||||
name=item["name"],
|
||||
workflow_request=item["workflow_request"],
|
||||
required_products=tuple(item["required_products"]),
|
||||
scripted_answers=tuple(
|
||||
tuple(pair) for pair in item.get("scripted_answers", [])
|
||||
),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
|
||||
|
||||
def template_headings(template: Path) -> list[str]:
|
||||
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
|
||||
return [
|
||||
heading
|
||||
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
|
||||
if "[" not in heading
|
||||
]
|
||||
|
||||
|
||||
def _normalise(text: str) -> str:
|
||||
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
|
||||
|
||||
|
||||
def _mentions(text: str, product: str) -> bool:
|
||||
return (
|
||||
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
async def score_execution(
|
||||
case: CaseSpec,
|
||||
interaction: Interaction,
|
||||
headings: Sequence[str],
|
||||
terraform_validate: TerraformValidate,
|
||||
) -> CaseResult:
|
||||
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
|
||||
"deliverables", {}
|
||||
)
|
||||
|
||||
def content(kind: DeliverableKind) -> str:
|
||||
item = deliverables.get(kind.value)
|
||||
return item["content"] if item else ""
|
||||
|
||||
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
|
||||
item = deliverables.get(kind.value)
|
||||
return item.get("attachments", {}) if item else {}
|
||||
|
||||
checks: list[Check] = []
|
||||
|
||||
checks.append(
|
||||
Check(
|
||||
"completed",
|
||||
interaction.status == "completed",
|
||||
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
|
||||
)
|
||||
)
|
||||
|
||||
guide = content(DeliverableKind.SOLUTION_GUIDE)
|
||||
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
|
||||
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
|
||||
checks.append(
|
||||
Check(
|
||||
"guide_follows_template",
|
||||
bool(guide) and not missing_sections,
|
||||
"missing: " + "; ".join(missing_sections) if missing_sections else "",
|
||||
)
|
||||
)
|
||||
|
||||
selection = content(DeliverableKind.PRODUCT_SELECTION)
|
||||
selected_text = selection + "\n" + guide
|
||||
missing_products = [
|
||||
p for p in case.required_products if not _mentions(selected_text, p)
|
||||
]
|
||||
checks.append(
|
||||
Check(
|
||||
"required_products_selected",
|
||||
bool(selection) and not missing_products,
|
||||
"missing: " + ", ".join(missing_products) if missing_products else "",
|
||||
)
|
||||
)
|
||||
|
||||
lowered = selection.lower()
|
||||
checks.append(
|
||||
Check(
|
||||
"selection_has_alternatives",
|
||||
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
Check(
|
||||
"selection_cites_documentation",
|
||||
bool(_DOC_LINK.search(selection)),
|
||||
)
|
||||
)
|
||||
|
||||
requirements = content(DeliverableKind.REQUIREMENTS)
|
||||
leaked = [
|
||||
product
|
||||
for product in case.required_products
|
||||
if _mentions(requirements, product)
|
||||
and not _mentions(case.workflow_request, product)
|
||||
]
|
||||
checks.append(
|
||||
Check(
|
||||
"requirements_name_no_products",
|
||||
bool(requirements) and not leaked,
|
||||
"names: " + ", ".join(leaked) if leaked else "",
|
||||
)
|
||||
)
|
||||
|
||||
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
|
||||
"architecture.mmd", ""
|
||||
)
|
||||
mermaid = await mermaid_check(diagram)
|
||||
checks.append(
|
||||
Check(
|
||||
"diagram_is_valid_mermaid",
|
||||
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
|
||||
mermaid.message,
|
||||
)
|
||||
)
|
||||
|
||||
terraform_files = {
|
||||
name.removeprefix("terraform/"): text
|
||||
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
|
||||
if name.endswith((".tf", ".tfvars", ".tf.json"))
|
||||
}
|
||||
if terraform_files:
|
||||
validation = await terraform_validate(terraform_files)
|
||||
terraform_ok = validation.message.startswith("terraform validate passed")
|
||||
terraform_detail = validation.message[:1500]
|
||||
else:
|
||||
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
||||
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
||||
|
||||
return CaseResult(case.id, interaction.status, tuple(checks))
|
||||
217
eval/solution_eval.py
Normal file
217
eval/solution_eval.py
Normal file
@@ -0,0 +1,217 @@
|
||||
"""Evaluate the solution architect agent on benchmark cases with real models.
|
||||
|
||||
Each case runs a pre-approved Execution with scripted user answers through the full
|
||||
assembly (ADK runtime, remote MCP servers, Terraform tools), then scores it with
|
||||
eval/scoring.py. Costs model usage: run deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
||||
.venv/bin/python -m eval.solution_eval [case-id ...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
||||
from app.agent import open_assembly
|
||||
from app.config import Settings, get_settings
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.local import TerraformRunner
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
load_cases,
|
||||
score_execution,
|
||||
template_headings,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
||||
RESULTS_DIR = ROOT / "results"
|
||||
|
||||
|
||||
class CountingRuntime:
|
||||
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
|
||||
|
||||
def __init__(self, inner: AgentRuntime) -> None:
|
||||
self._inner = inner
|
||||
self._started = time.monotonic()
|
||||
self.runs: Counter[str] = Counter()
|
||||
self.outcomes: Counter[str] = Counter()
|
||||
self.timeline: list[dict[str, Any]] = []
|
||||
|
||||
async def run(
|
||||
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
||||
) -> Outcome:
|
||||
self.runs[role.name] += 1
|
||||
start = time.monotonic()
|
||||
outcome = await self._inner.run(role, task, tools)
|
||||
self.outcomes[f"{role.name}:{type(outcome).__name__}"] += 1
|
||||
self.timeline.append(
|
||||
{
|
||||
"role": role.name,
|
||||
"deliverable": task.deliverable,
|
||||
"start_s": round(start - self._started, 1),
|
||||
"seconds": round(time.monotonic() - start, 1),
|
||||
"outcome": type(outcome).__name__,
|
||||
}
|
||||
)
|
||||
return outcome
|
||||
|
||||
|
||||
def first_pass_seconds(timeline: Sequence[dict[str, Any]]) -> float | None:
|
||||
"""Seconds until the first validation_results run ended: the run without revisions."""
|
||||
for run in timeline:
|
||||
if run["deliverable"] == "validation_results":
|
||||
return round(run["start_s"] + run["seconds"], 1)
|
||||
return None
|
||||
|
||||
|
||||
def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
||||
by_role = {
|
||||
f"{role} ({model})": asdict(counts)
|
||||
for (role, model), counts in sorted(usage.items())
|
||||
}
|
||||
by_model: dict[str, Counter[str]] = {}
|
||||
for (_, model), counts in usage.items():
|
||||
by_model.setdefault(model, Counter()).update(asdict(counts))
|
||||
total: Counter[str] = Counter()
|
||||
for counts in by_model.values():
|
||||
total.update(counts)
|
||||
return {
|
||||
"by_role": by_role,
|
||||
"by_model": {model: dict(c) for model, c in by_model.items()},
|
||||
"total": dict(total),
|
||||
}
|
||||
|
||||
|
||||
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
|
||||
from app.adk.adk_runtime import AdkAgentRuntime
|
||||
from app.providers import load_provider
|
||||
|
||||
bundle = load_provider(settings.CLOUD_PROVIDER)
|
||||
adk_runtime = AdkAgentRuntime(
|
||||
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
||||
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
||||
)
|
||||
runtime = CountingRuntime(adk_runtime)
|
||||
recorder = InMemoryRecorder()
|
||||
started = time.monotonic()
|
||||
async with open_assembly(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
interaction = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
runner = TerraformRunner(
|
||||
binary=settings.TERRAFORM_BINARY,
|
||||
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
||||
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
|
||||
)
|
||||
result = await score_execution(
|
||||
case, interaction, template_headings(bundle.output_template), runner.validate
|
||||
)
|
||||
ledger = interaction.state.get("ledger", {})
|
||||
details = {
|
||||
"execution_id": interaction.execution_id,
|
||||
"seconds": round(elapsed, 1),
|
||||
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
||||
"orchestrator_turns": interaction.state.get("turns"),
|
||||
"tokens": token_report(adk_runtime.usage),
|
||||
"timeline": runtime.timeline,
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"revisions": {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
if item["revisions"]
|
||||
},
|
||||
"warnings": interaction.state.get("warnings", []),
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(
|
||||
run_dir / "deliverables" / "executions" / interaction.execution_id
|
||||
),
|
||||
}
|
||||
return CaseResult(result.case_id, result.status, result.checks, details)
|
||||
|
||||
|
||||
async def main(case_ids: Sequence[str]) -> int:
|
||||
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
||||
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
logging.getLogger("app").setLevel(logging.INFO)
|
||||
settings = get_settings().model_copy(update={"CHECKPOINTER": "memory"})
|
||||
cases = [c for c in load_cases(CASES_PATH) if not case_ids or c.id in case_ids]
|
||||
if not cases:
|
||||
print(f"No cases match {list(case_ids)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = RESULTS_DIR / stamp
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
for case in cases:
|
||||
print(f"== {case.id}: running", flush=True)
|
||||
try:
|
||||
result = await run_case(case, settings, run_dir)
|
||||
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
||||
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
||||
results.append(
|
||||
{"case_id": case.id, "error": f"{type(exc).__name__}: {exc}"}
|
||||
)
|
||||
continue
|
||||
print(
|
||||
f" score {result.score:.0%} ({'PASS' if result.passed else 'FAIL'})",
|
||||
flush=True,
|
||||
)
|
||||
for check in result.checks:
|
||||
mark = "ok " if check.passed else "FAIL"
|
||||
print(f" {mark} {check.name} {check.detail[:160]}", flush=True)
|
||||
details = result.details
|
||||
print(
|
||||
f" time {details['seconds']}s, first pass {details['first_pass_seconds']}s, "
|
||||
f"revisions {details['revisions'] or 'none'}",
|
||||
flush=True,
|
||||
)
|
||||
for model, counts in details["tokens"]["by_model"].items():
|
||||
print(f" tokens {model}: {counts}", flush=True)
|
||||
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
||||
results.append(
|
||||
{
|
||||
"case_id": result.case_id,
|
||||
"status": result.status,
|
||||
"score": result.score,
|
||||
"passed": result.passed,
|
||||
"checks": [asdict(check) for check in result.checks],
|
||||
"details": result.details,
|
||||
}
|
||||
)
|
||||
|
||||
report = run_dir / "report.json"
|
||||
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
||||
passed = sum(1 for r in results if r.get("passed"))
|
||||
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
|
||||
return 0 if passed == len(results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main(sys.argv[1:])))
|
||||
@@ -1,32 +0,0 @@
|
||||
"""Tests for Evaluation Harness and Metrics."""
|
||||
|
||||
import pytest
|
||||
from eval.eval_harness import EvalHarness
|
||||
from eval.optimizer import SkillOptimizer
|
||||
|
||||
|
||||
def test_eval_harness_benchmark():
|
||||
"""Run EvalHarness against benchmark cases."""
|
||||
harness = EvalHarness()
|
||||
summary = harness.run_eval_suite()
|
||||
|
||||
assert summary["total_cases"] >= 1
|
||||
assert summary["passed_cases"] >= 1
|
||||
assert summary["pass_rate_percentage"] == 100.0
|
||||
|
||||
|
||||
def test_skill_optimizer_recommendations():
|
||||
"""Verify optimizer generates tuning recommendations for simulated failures."""
|
||||
optimizer = SkillOptimizer()
|
||||
simulated_summary = {
|
||||
"results": [
|
||||
{
|
||||
"case_id": "test-case",
|
||||
"passed": False,
|
||||
"feedback": ["Product selection was not explicitly deferred in discovery phase."],
|
||||
}
|
||||
]
|
||||
}
|
||||
recs = optimizer.generate_tuning_recommendations(simulated_summary)
|
||||
assert len(recs) == 1
|
||||
assert recs[0]["target_skill"] == "requirements_discovery"
|
||||
Reference in New Issue
Block a user