125 lines
4.4 KiB
Python
125 lines
4.4 KiB
Python
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
|
|
|
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
|
question. The shape is the one a decision model takes: one state, many typed
|
|
questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is
|
|
the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask
|
|
every question of every run and returns the same answer for the same input.
|
|
`GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping
|
|
from typing import Any, Protocol
|
|
|
|
from app.adk.runtime import Decider, Question
|
|
|
|
INSTRUCTIONS = (
|
|
"You grade a document from a cloud solution architecture engagement. Answer every "
|
|
"question about the document only, strictly and without benefit of the doubt. "
|
|
'Reply with one JSON object mapping each question id to a number: for a "yes_no" '
|
|
'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 '
|
|
"(weakly), 0.67 (mostly) or 1 (fully)."
|
|
)
|
|
|
|
|
|
class Judge(Protocol):
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
"""Score each question, keyed by id, from 0 to 1.
|
|
|
|
`questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}.
|
|
"""
|
|
...
|
|
|
|
|
|
GRADES = ("not at all", "weakly", "mostly", "fully")
|
|
|
|
|
|
class JudgeUnavailable(RuntimeError):
|
|
"""The judge gave no answer; the check it serves is unanswered, not failed."""
|
|
|
|
|
|
class DeciderJudge:
|
|
"""Jev as the judge. The document is the state and carries no grading
|
|
instructions; each question is asked on its own terms, all in one call."""
|
|
|
|
def __init__(self, decider: Decider, scope: str = "evaluation") -> None:
|
|
self._decider = decider
|
|
self._scope = scope
|
|
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
asked = {
|
|
key: Question("score", item["text"], GRADES)
|
|
if item["kind"] == "graded"
|
|
else Question("noul", item["text"])
|
|
for key, item in questions.items()
|
|
}
|
|
answers = await self._decider.decide(document, asked, scope=self._scope)
|
|
if answers is None:
|
|
raise JudgeUnavailable("Jev gave no answer")
|
|
scores = {}
|
|
for key, question in asked.items():
|
|
value = float(answers[key].value)
|
|
# A graded answer is a level from 0 to 3; a yes-or-no answer is the
|
|
# probability of yes, counted as a yes at an even chance or better.
|
|
scores[key] = (
|
|
min(1.0, max(0.0, value / (len(GRADES) - 1)))
|
|
if question.kind == "score"
|
|
else 1.0
|
|
if value >= 0.5
|
|
else 0.0
|
|
)
|
|
return scores
|
|
|
|
|
|
class GeminiJudge:
|
|
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
|
|
|
def __init__(self, model: str) -> None:
|
|
from google import genai
|
|
|
|
self._client = genai.Client()
|
|
self._model = model
|
|
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
from google.genai import types
|
|
|
|
prompt = (
|
|
f"{INSTRUCTIONS}\n\nQuestions:\n"
|
|
f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n"
|
|
f"Document:\n<document>\n{document}\n</document>"
|
|
)
|
|
response = await self._client.aio.models.generate_content(
|
|
model=self._model,
|
|
contents=prompt,
|
|
config=types.GenerateContentConfig(
|
|
temperature=0, response_mime_type="application/json"
|
|
),
|
|
)
|
|
return parse_scores(response.text or "", questions)
|
|
|
|
|
|
def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]:
|
|
"""The judge's reply as scores; a missing or malformed answer scores 0."""
|
|
try:
|
|
raw = json.loads(text)
|
|
except ValueError:
|
|
raw = {}
|
|
if not isinstance(raw, dict):
|
|
raw = {}
|
|
scores = {}
|
|
for key in questions:
|
|
value = raw.get(key, 0)
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
value = 1.0 if value is True else 0.0
|
|
scores[key] = min(1.0, max(0.0, float(value)))
|
|
return scores
|