This commit is contained in:
@@ -40,7 +40,7 @@ from app.workflows.target_architecture import (
|
||||
parse_graph,
|
||||
render_mermaid,
|
||||
)
|
||||
from eval.judge import Judge
|
||||
from eval.judge import Judge, JudgeUnavailable
|
||||
|
||||
PASS_THRESHOLD = 0.8
|
||||
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
|
||||
@@ -403,7 +403,10 @@ async def _checklist_coverage(
|
||||
}
|
||||
if not recommendations or not questions:
|
||||
return Check(name, False, "no design recommendations or no checklists")
|
||||
scores = await judge.answer(recommendations, questions)
|
||||
try:
|
||||
scores = await judge.answer(recommendations, questions)
|
||||
except JudgeUnavailable as exc:
|
||||
return Check(name, False, f"judge unavailable: {exc}")
|
||||
per_pillar = {
|
||||
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
|
||||
/ len(items)
|
||||
@@ -419,10 +422,16 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
|
||||
name = "recommendations_quality"
|
||||
if not recommendations:
|
||||
return Check(name, False, "no design recommendations")
|
||||
scores = await judge.answer(
|
||||
recommendations,
|
||||
{key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items()},
|
||||
)
|
||||
try:
|
||||
scores = await judge.answer(
|
||||
recommendations,
|
||||
{
|
||||
key: {"kind": "graded", "text": text}
|
||||
for key, text in QUALITY_RUBRIC.items()
|
||||
},
|
||||
)
|
||||
except JudgeUnavailable as exc:
|
||||
return Check(name, False, f"judge unavailable: {exc}")
|
||||
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
||||
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
||||
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
||||
|
||||
Reference in New Issue
Block a user