fix: refactored manually

eval/eval_harness.py

@@ -0,0 +1,81 @@
+"""Evaluation Harness Runner for GCP Solution Architecture Agent."""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List
+
+from app.config import get_settings
+from app.skills.loader import SkillLoader
+from app.workflows.gcp_architecture_graph import create_agent
+from eval.metrics import evaluate_case_run
+
+logger = logging.getLogger(__name__)
+
+
+class EvalHarness:
+    """Offline Evaluation Harness for running benchmark suites against agent workflows."""
+
+    def __init__(self, dataset_path: Path | None = None) -> None:
+        settings = get_settings()
+        self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH
+        self.skill_loader = SkillLoader(settings.SKILLS_DIR)
+        self.skill_loader.load_skills()
+        self.agent = create_agent(self.skill_loader)
+
+    def load_benchmark_cases(self) -> List[Dict[str, Any]]:
+        """Load benchmark dataset JSON."""
+        if not self.dataset_path.is_file():
+            logger.error("Benchmark dataset not found at %s", self.dataset_path)
+            return []
+
+        with open(self.dataset_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def run_eval_suite(self) -> Dict[str, Any]:
+        """Execute all benchmark test cases and compile scoring metrics."""
+        cases = self.load_benchmark_cases()
+        if not cases:
+            return {"status": "error", "message": "No benchmark cases loaded."}
+
+        results = []
+        total_passed = 0
+
+        for case in cases:
+            logger.info("Evaluating benchmark case: %s", case.get("id"))
+            initial_state = {
+                "workflow_request": case.get("workflow_request", ""),
+                "target_dir": ".",
+                "active_skills": [],
+            }
+
+            final_state = self.agent.invoke(initial_state)
+            eval_result = evaluate_case_run(final_state, case)
+            results.append(eval_result)
+
+            if eval_result.get("passed"):
+                total_passed += 1
+
+        pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0
+
+        summary = {
+            "total_cases": len(cases),
+            "passed_cases": total_passed,
+            "failed_cases": len(cases) - total_passed,
+            "pass_rate_percentage": pass_rate,
+            "results": results,
+        }
+
+        return summary
+
+
+def main() -> None:
+    """CLI Runner for Evaluation Harness."""
+    harness = EvalHarness()
+    summary = harness.run_eval_suite()
+    print("=== GCP Solution Architecture Agent Benchmark Summary ===")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()