This commit is contained in:
@@ -1,8 +1,11 @@
|
||||
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
||||
|
||||
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
||||
question. The shape (one state, many typed questions) is deliberately the one a
|
||||
decision model takes, so the Gemini judge here can be swapped for one later.
|
||||
question. The shape is the one a decision model takes: one state, many typed
|
||||
questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is
|
||||
the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask
|
||||
every question of every run and returns the same answer for the same input.
|
||||
`GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,6 +14,8 @@ import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.adk.runtime import Decider, Question
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"You grade a document from a cloud solution architecture engagement. Answer every "
|
||||
"question about the document only, strictly and without benefit of the doubt. "
|
||||
@@ -31,6 +36,48 @@ class Judge(Protocol):
|
||||
...
|
||||
|
||||
|
||||
GRADES = ("not at all", "weakly", "mostly", "fully")
|
||||
|
||||
|
||||
class JudgeUnavailable(RuntimeError):
|
||||
"""The judge gave no answer; the check it serves is unanswered, not failed."""
|
||||
|
||||
|
||||
class DeciderJudge:
|
||||
"""Jev as the judge. The document is the state and carries no grading
|
||||
instructions; each question is asked on its own terms, all in one call."""
|
||||
|
||||
def __init__(self, decider: Decider, scope: str = "evaluation") -> None:
|
||||
self._decider = decider
|
||||
self._scope = scope
|
||||
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
asked = {
|
||||
key: Question("score", item["text"], GRADES)
|
||||
if item["kind"] == "graded"
|
||||
else Question("noul", item["text"])
|
||||
for key, item in questions.items()
|
||||
}
|
||||
answers = await self._decider.decide(document, asked, scope=self._scope)
|
||||
if answers is None:
|
||||
raise JudgeUnavailable("Jev gave no answer")
|
||||
scores = {}
|
||||
for key, question in asked.items():
|
||||
value = float(answers[key].value)
|
||||
# A graded answer is a level from 0 to 3; a yes-or-no answer is the
|
||||
# probability of yes, counted as a yes at an even chance or better.
|
||||
scores[key] = (
|
||||
min(1.0, max(0.0, value / (len(GRADES) - 1)))
|
||||
if question.kind == "score"
|
||||
else 1.0
|
||||
if value >= 0.5
|
||||
else 0.0
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
class GeminiJudge:
|
||||
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user