fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -40,7 +40,7 @@ from app.workflows.target_architecture import (
parse_graph,
render_mermaid,
)
from eval.judge import Judge
from eval.judge import Judge, JudgeUnavailable
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
@@ -403,7 +403,10 @@ async def _checklist_coverage(
}
if not recommendations or not questions:
return Check(name, False, "no design recommendations or no checklists")
scores = await judge.answer(recommendations, questions)
try:
scores = await judge.answer(recommendations, questions)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
per_pillar = {
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
/ len(items)
@@ -419,10 +422,16 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
name = "recommendations_quality"
if not recommendations:
return Check(name, False, "no design recommendations")
scores = await judge.answer(
recommendations,
{key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items()},
)
try:
scores = await judge.answer(
recommendations,
{
key: {"kind": "graded", "text": text}
for key, text in QUALITY_RUBRIC.items()
},
)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")