"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent.""" import re from typing import Any, Dict, List def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]: """Score Phase 0 Requirements Discovery output.""" score = 0.0 feedback = [] if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc: score += 0.5 else: feedback.append("Product selection was not explicitly deferred in discovery phase.") if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc: score += 0.5 else: feedback.append("Missing functional or non-functional requirement sections.") return {"score": score, "max_score": 1.0, "feedback": feedback} def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]: """Score Phase 1 Architecture & Product Selection output.""" score = 0.0 max_score = 3.0 feedback = [] # Check product choices required_prods = rubric.get("required_products", []) found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()] if len(found_prods) == len(required_prods): score += 1.0 else: missing = set(required_prods) - set(found_prods) feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}") # Check Mermaid diagram if re.search(r"\b(flowchart|graph)\b", mermaid_diagram): score += 1.0 else: feedback.append("Invalid or missing Mermaid diagram syntax.") # Check Terraform code if "resource" in terraform_code and "google_" in terraform_code: score += 1.0 else: feedback.append("Terraform code does not contain Google Cloud resources.") return {"score": score, "max_score": max_score, "feedback": feedback} def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]: """Score Phase 3 Guide Packaging output.""" score = 0.0 max_score = 1.0 feedback = [] required_sections = rubric.get("requires_guide_sections", []) found_sections = [sec for sec in required_sections if sec in solution_guide] if len(found_sections) == len(required_sections): score += 1.0 else: missing = set(required_sections) - set(found_sections) feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}") return {"score": score, "max_score": max_score, "feedback": feedback} def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]: """Run full evaluation suite for a benchmark case output.""" rubric = case.get("rubric", {}) req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", "")) des_eval = evaluate_design_phase( final_state.get("architecture_doc", ""), final_state.get("mermaid_diagram", ""), final_state.get("terraform_code", ""), rubric, ) pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric) total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"] total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"] percentage = (total_score / total_max) * 100.0 all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"] return { "case_id": case.get("id"), "case_name": case.get("name"), "total_score": total_score, "max_score": total_max, "percentage": percentage, "passed": percentage >= 80.0, "feedback": all_feedback, }