fix: refactored manually

eval/optimizer.py

@@ -0,0 +1,58 @@
+"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
+
+import logging
+from typing import Any, Dict, List
+from eval.metrics import evaluate_case_run
+
+logger = logging.getLogger(__name__)
+
+
+class SkillOptimizer:
+    """Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
+
+    def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Generate prompt tuning recommendations based on evaluation failures."""
+        recommendations = []
+        results = eval_summary.get("results", [])
+
+        for res in results:
+            if not res.get("passed"):
+                feedback_items = res.get("feedback", [])
+                for feedback in feedback_items:
+                    rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
+                    if rec:
+                        recommendations.append(rec)
+
+        return recommendations
+
+    def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
+        """Maps specific feedback strings to target SKILL.md prompt enhancements."""
+        if "deferred" in feedback.lower():
+            return {
+                "target_skill": "requirements_discovery",
+                "phase": "discover",
+                "issue": feedback,
+                "recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
+            }
+        elif "mermaid" in feedback.lower():
+            return {
+                "target_skill": "architecture_design",
+                "phase": "design",
+                "issue": feedback,
+                "recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
+            }
+        elif "terraform" in feedback.lower():
+            return {
+                "target_skill": "architecture_design",
+                "phase": "design",
+                "issue": feedback,
+                "recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
+            }
+        elif "section" in feedback.lower():
+            return {
+                "target_skill": "packaging_guide",
+                "phase": "package",
+                "issue": feedback,
+                "recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
+            }
+        return None