"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts.""" import logging from typing import Any, Dict, List from eval.metrics import evaluate_case_run logger = logging.getLogger(__name__) class SkillOptimizer: """Analyzes benchmark evaluation feedback and recommends skill prompt adjustments.""" def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]: """Generate prompt tuning recommendations based on evaluation failures.""" recommendations = [] results = eval_summary.get("results", []) for res in results: if not res.get("passed"): feedback_items = res.get("feedback", []) for feedback in feedback_items: rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback) if rec: recommendations.append(rec) return recommendations def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None: """Maps specific feedback strings to target SKILL.md prompt enhancements.""" if "deferred" in feedback.lower(): return { "target_skill": "requirements_discovery", "phase": "discover", "issue": feedback, "recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.", } elif "mermaid" in feedback.lower(): return { "target_skill": "architecture_design", "phase": "design", "issue": feedback, "recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.", } elif "terraform" in feedback.lower(): return { "target_skill": "architecture_design", "phase": "design", "issue": feedback, "recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.", } elif "section" in feedback.lower(): return { "target_skill": "packaging_guide", "phase": "package", "issue": feedback, "recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.", } return None