fix: refactored manually

eval/metrics.py

@@ -0,0 +1,100 @@
+"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
+
+import re
+from typing import Any, Dict, List
+
+
+def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
+    """Score Phase 0 Requirements Discovery output."""
+    score = 0.0
+    feedback = []
+
+    if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
+        score += 0.5
+    else:
+        feedback.append("Product selection was not explicitly deferred in discovery phase.")
+
+    if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
+        score += 0.5
+    else:
+        feedback.append("Missing functional or non-functional requirement sections.")
+
+    return {"score": score, "max_score": 1.0, "feedback": feedback}
+
+
+def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
+    """Score Phase 1 Architecture & Product Selection output."""
+    score = 0.0
+    max_score = 3.0
+    feedback = []
+
+    # Check product choices
+    required_prods = rubric.get("required_products", [])
+    found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
+    if len(found_prods) == len(required_prods):
+        score += 1.0
+    else:
+        missing = set(required_prods) - set(found_prods)
+        feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
+
+    # Check Mermaid diagram
+    if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
+        score += 1.0
+    else:
+        feedback.append("Invalid or missing Mermaid diagram syntax.")
+
+    # Check Terraform code
+    if "resource" in terraform_code and "google_" in terraform_code:
+        score += 1.0
+    else:
+        feedback.append("Terraform code does not contain Google Cloud resources.")
+
+    return {"score": score, "max_score": max_score, "feedback": feedback}
+
+
+def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
+    """Score Phase 3 Guide Packaging output."""
+    score = 0.0
+    max_score = 1.0
+    feedback = []
+
+    required_sections = rubric.get("requires_guide_sections", [])
+    found_sections = [sec for sec in required_sections if sec in solution_guide]
+
+    if len(found_sections) == len(required_sections):
+        score += 1.0
+    else:
+        missing = set(required_sections) - set(found_sections)
+        feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
+
+    return {"score": score, "max_score": max_score, "feedback": feedback}
+
+
+def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
+    """Run full evaluation suite for a benchmark case output."""
+    rubric = case.get("rubric", {})
+
+    req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
+    des_eval = evaluate_design_phase(
+        final_state.get("architecture_doc", ""),
+        final_state.get("mermaid_diagram", ""),
+        final_state.get("terraform_code", ""),
+        rubric,
+    )
+    pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
+
+    total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
+    total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
+    percentage = (total_score / total_max) * 100.0
+
+    all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
+
+    return {
+        "case_id": case.get("id"),
+        "case_name": case.get("name"),
+        "total_score": total_score,
+        "max_score": total_max,
+        "percentage": percentage,
+        "passed": percentage >= 80.0,
+        "feedback": all_feedback,
+    }