fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -475,3 +475,56 @@ async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
assert any("Approved." in reply for reply in replies)
assert any("europe-west2." in reply for reply in replies)
assert final.state["deliverables"]["requirements"]["content"]
# --- Jev as the judge -------------------------------------------------------------------
async def test_jev_judges_every_question_in_one_call_with_no_grading_in_the_state():
from app.adk.fake_runtime import FakeDecider
from app.adk.runtime import Answer
from eval.judge import DeciderJudge
decider = FakeDecider(
{
"covered": Answer(0.91, 0.82),
"missed": Answer(0.2, 0.6),
"specific": Answer(2.0, 0.7), # "mostly", on a scale of four
}
)
judge = DeciderJudge(decider)
scores = await judge.answer(
"## Reliability\nUse two zones.",
{
"covered": {"kind": "yes_no", "text": "Is redundancy addressed?"},
"missed": {"kind": "yes_no", "text": "Are backups tested?"},
"specific": {"kind": "graded", "text": "Is it specific?"},
},
)
assert scores == {"covered": 1.0, "missed": 0.0, "specific": 2 / 3}
[(state, questions, scope)] = decider.asked
assert state == "## Reliability\nUse two zones." # the document, nothing else
assert questions["covered"].kind == "noul"
assert questions["specific"].options == ("not at all", "weakly", "mostly", "fully")
assert scope == "evaluation"
async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_passed():
from app.adk.runtime import NoDecider
from eval.judge import DeciderJudge
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=DeciderJudge(NoDecider()),
pillar_checklists=CHECKLISTS,
)
assert check(result, "pillar_checklists_covered").detail == (
"judge unavailable: Jev gave no answer"
)
assert not check(result, "recommendations_quality").passed