This commit is contained in:
@@ -475,3 +475,56 @@ async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
|
||||
assert any("Approved." in reply for reply in replies)
|
||||
assert any("europe-west2." in reply for reply in replies)
|
||||
assert final.state["deliverables"]["requirements"]["content"]
|
||||
|
||||
|
||||
# --- Jev as the judge -------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_jev_judges_every_question_in_one_call_with_no_grading_in_the_state():
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer
|
||||
from eval.judge import DeciderJudge
|
||||
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"covered": Answer(0.91, 0.82),
|
||||
"missed": Answer(0.2, 0.6),
|
||||
"specific": Answer(2.0, 0.7), # "mostly", on a scale of four
|
||||
}
|
||||
)
|
||||
judge = DeciderJudge(decider)
|
||||
|
||||
scores = await judge.answer(
|
||||
"## Reliability\nUse two zones.",
|
||||
{
|
||||
"covered": {"kind": "yes_no", "text": "Is redundancy addressed?"},
|
||||
"missed": {"kind": "yes_no", "text": "Are backups tested?"},
|
||||
"specific": {"kind": "graded", "text": "Is it specific?"},
|
||||
},
|
||||
)
|
||||
|
||||
assert scores == {"covered": 1.0, "missed": 0.0, "specific": 2 / 3}
|
||||
[(state, questions, scope)] = decider.asked
|
||||
assert state == "## Reliability\nUse two zones." # the document, nothing else
|
||||
assert questions["covered"].kind == "noul"
|
||||
assert questions["specific"].options == ("not at all", "weakly", "mostly", "fully")
|
||||
assert scope == "evaluation"
|
||||
|
||||
|
||||
async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_passed():
|
||||
from app.adk.runtime import NoDecider
|
||||
from eval.judge import DeciderJudge
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
judge=DeciderJudge(NoDecider()),
|
||||
pillar_checklists=CHECKLISTS,
|
||||
)
|
||||
|
||||
assert check(result, "pillar_checklists_covered").detail == (
|
||||
"judge unavailable: Jev gave no answer"
|
||||
)
|
||||
assert not check(result, "recommendations_quality").passed
|
||||
|
||||
Reference in New Issue
Block a user