"""Judges: graded answers about a Deliverable that string matching cannot give. A judge takes a document and a set of questions and returns a score from 0 to 1 per question. The shape is the one a decision model takes: one state, many typed questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask every question of every run and returns the same answer for the same input. `GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set. """ from __future__ import annotations import json from collections.abc import Mapping from typing import Any, Protocol from app.adk.runtime import Decider, Question INSTRUCTIONS = ( "You grade a document from a cloud solution architecture engagement. Answer every " "question about the document only, strictly and without benefit of the doubt. " 'Reply with one JSON object mapping each question id to a number: for a "yes_no" ' 'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 ' "(weakly), 0.67 (mostly) or 1 (fully)." ) class Judge(Protocol): async def answer( self, document: str, questions: Mapping[str, Mapping[str, str]] ) -> Mapping[str, float]: """Score each question, keyed by id, from 0 to 1. `questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}. """ ... GRADES = ("not at all", "weakly", "mostly", "fully") class JudgeUnavailable(RuntimeError): """The judge gave no answer; the check it serves is unanswered, not failed.""" class DeciderJudge: """Jev as the judge. The document is the state and carries no grading instructions; each question is asked on its own terms, all in one call.""" def __init__(self, decider: Decider, scope: str = "evaluation") -> None: self._decider = decider self._scope = scope async def answer( self, document: str, questions: Mapping[str, Mapping[str, str]] ) -> Mapping[str, float]: asked = { key: Question("score", item["text"], GRADES) if item["kind"] == "graded" else Question("noul", item["text"]) for key, item in questions.items() } answers = await self._decider.decide(document, asked, scope=self._scope) if answers is None: raise JudgeUnavailable("Jev gave no answer") scores = {} for key, question in asked.items(): value = float(answers[key].value) # A graded answer is a level from 0 to 3; a yes-or-no answer is the # probability of yes, counted as a yes at an even chance or better. scores[key] = ( min(1.0, max(0.0, value / (len(GRADES) - 1))) if question.kind == "score" else 1.0 if value >= 0.5 else 0.0 ) return scores class GeminiJudge: """A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does.""" def __init__(self, model: str) -> None: from google import genai self._client = genai.Client() self._model = model async def answer( self, document: str, questions: Mapping[str, Mapping[str, str]] ) -> Mapping[str, float]: from google.genai import types prompt = ( f"{INSTRUCTIONS}\n\nQuestions:\n" f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n" f"Document:\n\n{document}\n" ) response = await self._client.aio.models.generate_content( model=self._model, contents=prompt, config=types.GenerateContentConfig( temperature=0, response_mime_type="application/json" ), ) return parse_scores(response.text or "", questions) def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]: """The judge's reply as scores; a missing or malformed answer scores 0.""" try: raw = json.loads(text) except ValueError: raw = {} if not isinstance(raw, dict): raw = {} scores = {} for key in questions: value = raw.get(key, 0) if isinstance(value, bool) or not isinstance(value, (int, float)): value = 1.0 if value is True else 0.0 scores[key] = min(1.0, max(0.0, float(value))) return scores