59 lines
2.5 KiB
Python
59 lines
2.5 KiB
Python
"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
|
|
|
|
import logging
|
|
from typing import Any, Dict, List
|
|
from eval.metrics import evaluate_case_run
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class SkillOptimizer:
|
|
"""Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
|
|
|
|
def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""Generate prompt tuning recommendations based on evaluation failures."""
|
|
recommendations = []
|
|
results = eval_summary.get("results", [])
|
|
|
|
for res in results:
|
|
if not res.get("passed"):
|
|
feedback_items = res.get("feedback", [])
|
|
for feedback in feedback_items:
|
|
rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
|
|
if rec:
|
|
recommendations.append(rec)
|
|
|
|
return recommendations
|
|
|
|
def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
|
|
"""Maps specific feedback strings to target SKILL.md prompt enhancements."""
|
|
if "deferred" in feedback.lower():
|
|
return {
|
|
"target_skill": "requirements_discovery",
|
|
"phase": "discover",
|
|
"issue": feedback,
|
|
"recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
|
|
}
|
|
elif "mermaid" in feedback.lower():
|
|
return {
|
|
"target_skill": "architecture_design",
|
|
"phase": "design",
|
|
"issue": feedback,
|
|
"recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
|
|
}
|
|
elif "terraform" in feedback.lower():
|
|
return {
|
|
"target_skill": "architecture_design",
|
|
"phase": "design",
|
|
"issue": feedback,
|
|
"recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
|
|
}
|
|
elif "section" in feedback.lower():
|
|
return {
|
|
"target_skill": "packaging_guide",
|
|
"phase": "package",
|
|
"issue": feedback,
|
|
"recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
|
|
}
|
|
return None
|