101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
|
|
|
|
import re
|
|
from typing import Any, Dict, List
|
|
|
|
|
|
def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
|
|
"""Score Phase 0 Requirements Discovery output."""
|
|
score = 0.0
|
|
feedback = []
|
|
|
|
if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
|
|
score += 0.5
|
|
else:
|
|
feedback.append("Product selection was not explicitly deferred in discovery phase.")
|
|
|
|
if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
|
|
score += 0.5
|
|
else:
|
|
feedback.append("Missing functional or non-functional requirement sections.")
|
|
|
|
return {"score": score, "max_score": 1.0, "feedback": feedback}
|
|
|
|
|
|
def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Score Phase 1 Architecture & Product Selection output."""
|
|
score = 0.0
|
|
max_score = 3.0
|
|
feedback = []
|
|
|
|
# Check product choices
|
|
required_prods = rubric.get("required_products", [])
|
|
found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
|
|
if len(found_prods) == len(required_prods):
|
|
score += 1.0
|
|
else:
|
|
missing = set(required_prods) - set(found_prods)
|
|
feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
|
|
|
|
# Check Mermaid diagram
|
|
if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
|
|
score += 1.0
|
|
else:
|
|
feedback.append("Invalid or missing Mermaid diagram syntax.")
|
|
|
|
# Check Terraform code
|
|
if "resource" in terraform_code and "google_" in terraform_code:
|
|
score += 1.0
|
|
else:
|
|
feedback.append("Terraform code does not contain Google Cloud resources.")
|
|
|
|
return {"score": score, "max_score": max_score, "feedback": feedback}
|
|
|
|
|
|
def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Score Phase 3 Guide Packaging output."""
|
|
score = 0.0
|
|
max_score = 1.0
|
|
feedback = []
|
|
|
|
required_sections = rubric.get("requires_guide_sections", [])
|
|
found_sections = [sec for sec in required_sections if sec in solution_guide]
|
|
|
|
if len(found_sections) == len(required_sections):
|
|
score += 1.0
|
|
else:
|
|
missing = set(required_sections) - set(found_sections)
|
|
feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
|
|
|
|
return {"score": score, "max_score": max_score, "feedback": feedback}
|
|
|
|
|
|
def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Run full evaluation suite for a benchmark case output."""
|
|
rubric = case.get("rubric", {})
|
|
|
|
req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
|
|
des_eval = evaluate_design_phase(
|
|
final_state.get("architecture_doc", ""),
|
|
final_state.get("mermaid_diagram", ""),
|
|
final_state.get("terraform_code", ""),
|
|
rubric,
|
|
)
|
|
pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
|
|
|
|
total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
|
|
total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
|
|
percentage = (total_score / total_max) * 100.0
|
|
|
|
all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
|
|
|
|
return {
|
|
"case_id": case.get("id"),
|
|
"case_name": case.get("name"),
|
|
"total_score": total_score,
|
|
"max_score": total_max,
|
|
"percentage": percentage,
|
|
"passed": percentage >= 80.0,
|
|
"feedback": all_feedback,
|
|
}
|