fix: refactored manually

eval/__init__.py

@@ -0,0 +1 @@
+"""Eval package."""

eval/datasets/benchmark_cases.json

@@ -0,0 +1,26 @@
+[
+  {
+    "id": "case-01-event-driven-http",
+    "name": "Event-Driven Regional HTTP Workload",
+    "workflow_request": "Design an event-driven regional HTTP application that accepts incoming webhooks, validates signatures, durably enqueues payloads to Pub/Sub, and retains raw events in Cloud Storage for 30-day replay audit.",
+    "rubric": {
+      "required_products": ["Cloud Run", "Cloud Pub/Sub", "Cloud Storage", "Cloud IAM"],
+      "must_defer_in_discover": true,
+      "requires_mermaid": true,
+      "requires_terraform": true,
+      "requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
+    }
+  },
+  {
+    "id": "case-02-high-scale-ingestion",
+    "name": "High Scale Ingestion Pipeline",
+    "workflow_request": "Create a high-scale containerized data ingestion pipeline with least-privilege IAM service accounts and automated validation.",
+    "rubric": {
+      "required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
+      "must_defer_in_discover": true,
+      "requires_mermaid": true,
+      "requires_terraform": true,
+      "requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
+    }
+  }
+]

eval/eval_harness.py

@@ -0,0 +1,81 @@
+"""Evaluation Harness Runner for GCP Solution Architecture Agent."""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List
+
+from app.config import get_settings
+from app.skills.loader import SkillLoader
+from app.workflows.gcp_architecture_graph import create_agent
+from eval.metrics import evaluate_case_run
+
+logger = logging.getLogger(__name__)
+
+
+class EvalHarness:
+    """Offline Evaluation Harness for running benchmark suites against agent workflows."""
+
+    def __init__(self, dataset_path: Path | None = None) -> None:
+        settings = get_settings()
+        self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH
+        self.skill_loader = SkillLoader(settings.SKILLS_DIR)
+        self.skill_loader.load_skills()
+        self.agent = create_agent(self.skill_loader)
+
+    def load_benchmark_cases(self) -> List[Dict[str, Any]]:
+        """Load benchmark dataset JSON."""
+        if not self.dataset_path.is_file():
+            logger.error("Benchmark dataset not found at %s", self.dataset_path)
+            return []
+
+        with open(self.dataset_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def run_eval_suite(self) -> Dict[str, Any]:
+        """Execute all benchmark test cases and compile scoring metrics."""
+        cases = self.load_benchmark_cases()
+        if not cases:
+            return {"status": "error", "message": "No benchmark cases loaded."}
+
+        results = []
+        total_passed = 0
+
+        for case in cases:
+            logger.info("Evaluating benchmark case: %s", case.get("id"))
+            initial_state = {
+                "workflow_request": case.get("workflow_request", ""),
+                "target_dir": ".",
+                "active_skills": [],
+            }
+
+            final_state = self.agent.invoke(initial_state)
+            eval_result = evaluate_case_run(final_state, case)
+            results.append(eval_result)
+
+            if eval_result.get("passed"):
+                total_passed += 1
+
+        pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0
+
+        summary = {
+            "total_cases": len(cases),
+            "passed_cases": total_passed,
+            "failed_cases": len(cases) - total_passed,
+            "pass_rate_percentage": pass_rate,
+            "results": results,
+        }
+
+        return summary
+
+
+def main() -> None:
+    """CLI Runner for Evaluation Harness."""
+    harness = EvalHarness()
+    summary = harness.run_eval_suite()
+    print("=== GCP Solution Architecture Agent Benchmark Summary ===")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

eval/metrics.py

@@ -0,0 +1,100 @@
+"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
+
+import re
+from typing import Any, Dict, List
+
+
+def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
+    """Score Phase 0 Requirements Discovery output."""
+    score = 0.0
+    feedback = []
+
+    if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
+        score += 0.5
+    else:
+        feedback.append("Product selection was not explicitly deferred in discovery phase.")
+
+    if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
+        score += 0.5
+    else:
+        feedback.append("Missing functional or non-functional requirement sections.")
+
+    return {"score": score, "max_score": 1.0, "feedback": feedback}
+
+
+def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
+    """Score Phase 1 Architecture & Product Selection output."""
+    score = 0.0
+    max_score = 3.0
+    feedback = []
+
+    # Check product choices
+    required_prods = rubric.get("required_products", [])
+    found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
+    if len(found_prods) == len(required_prods):
+        score += 1.0
+    else:
+        missing = set(required_prods) - set(found_prods)
+        feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
+
+    # Check Mermaid diagram
+    if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
+        score += 1.0
+    else:
+        feedback.append("Invalid or missing Mermaid diagram syntax.")
+
+    # Check Terraform code
+    if "resource" in terraform_code and "google_" in terraform_code:
+        score += 1.0
+    else:
+        feedback.append("Terraform code does not contain Google Cloud resources.")
+
+    return {"score": score, "max_score": max_score, "feedback": feedback}
+
+
+def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
+    """Score Phase 3 Guide Packaging output."""
+    score = 0.0
+    max_score = 1.0
+    feedback = []
+
+    required_sections = rubric.get("requires_guide_sections", [])
+    found_sections = [sec for sec in required_sections if sec in solution_guide]
+
+    if len(found_sections) == len(required_sections):
+        score += 1.0
+    else:
+        missing = set(required_sections) - set(found_sections)
+        feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
+
+    return {"score": score, "max_score": max_score, "feedback": feedback}
+
+
+def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
+    """Run full evaluation suite for a benchmark case output."""
+    rubric = case.get("rubric", {})
+
+    req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
+    des_eval = evaluate_design_phase(
+        final_state.get("architecture_doc", ""),
+        final_state.get("mermaid_diagram", ""),
+        final_state.get("terraform_code", ""),
+        rubric,
+    )
+    pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
+
+    total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
+    total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
+    percentage = (total_score / total_max) * 100.0
+
+    all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
+
+    return {
+        "case_id": case.get("id"),
+        "case_name": case.get("name"),
+        "total_score": total_score,
+        "max_score": total_max,
+        "percentage": percentage,
+        "passed": percentage >= 80.0,
+        "feedback": all_feedback,
+    }

eval/optimizer.py

@@ -0,0 +1,58 @@
+"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
+
+import logging
+from typing import Any, Dict, List
+from eval.metrics import evaluate_case_run
+
+logger = logging.getLogger(__name__)
+
+
+class SkillOptimizer:
+    """Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
+
+    def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Generate prompt tuning recommendations based on evaluation failures."""
+        recommendations = []
+        results = eval_summary.get("results", [])
+
+        for res in results:
+            if not res.get("passed"):
+                feedback_items = res.get("feedback", [])
+                for feedback in feedback_items:
+                    rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
+                    if rec:
+                        recommendations.append(rec)
+
+        return recommendations
+
+    def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
+        """Maps specific feedback strings to target SKILL.md prompt enhancements."""
+        if "deferred" in feedback.lower():
+            return {
+                "target_skill": "requirements_discovery",
+                "phase": "discover",
+                "issue": feedback,
+                "recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
+            }
+        elif "mermaid" in feedback.lower():
+            return {
+                "target_skill": "architecture_design",
+                "phase": "design",
+                "issue": feedback,
+                "recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
+            }
+        elif "terraform" in feedback.lower():
+            return {
+                "target_skill": "architecture_design",
+                "phase": "design",
+                "issue": feedback,
+                "recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
+            }
+        elif "section" in feedback.lower():
+            return {
+                "target_skill": "packaging_guide",
+                "phase": "package",
+                "issue": feedback,
+                "recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
+            }
+        return None

eval/test_eval_harness.py

@@ -0,0 +1,32 @@
+"""Tests for Evaluation Harness and Metrics."""
+
+import pytest
+from eval.eval_harness import EvalHarness
+from eval.optimizer import SkillOptimizer
+
+
+def test_eval_harness_benchmark():
+    """Run EvalHarness against benchmark cases."""
+    harness = EvalHarness()
+    summary = harness.run_eval_suite()
+
+    assert summary["total_cases"] >= 1
+    assert summary["passed_cases"] >= 1
+    assert summary["pass_rate_percentage"] == 100.0
+
+
+def test_skill_optimizer_recommendations():
+    """Verify optimizer generates tuning recommendations for simulated failures."""
+    optimizer = SkillOptimizer()
+    simulated_summary = {
+        "results": [
+            {
+                "case_id": "test-case",
+                "passed": False,
+                "feedback": ["Product selection was not explicitly deferred in discovery phase."],
+            }
+        ]
+    }
+    recs = optimizer.generate_tuning_recommendations(simulated_summary)
+    assert len(recs) == 1
+    assert recs[0]["target_skill"] == "requirements_discovery"