"""Judges: graded answers about a Deliverable that string matching cannot give. A judge takes a document and a set of questions and returns a score from 0 to 1 per question. The shape (one state, many typed questions) is deliberately the one a decision model takes, so the Gemini judge here can be swapped for one later. """ from __future__ import annotations import json from collections.abc import Mapping from typing import Any, Protocol INSTRUCTIONS = ( "You grade a document from a cloud solution architecture engagement. Answer every " "question about the document only, strictly and without benefit of the doubt. " 'Reply with one JSON object mapping each question id to a number: for a "yes_no" ' 'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 ' "(weakly), 0.67 (mostly) or 1 (fully)." ) class Judge(Protocol): async def answer( self, document: str, questions: Mapping[str, Mapping[str, str]] ) -> Mapping[str, float]: """Score each question, keyed by id, from 0 to 1. `questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}. """ ... class GeminiJudge: """A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does.""" def __init__(self, model: str) -> None: from google import genai self._client = genai.Client() self._model = model async def answer( self, document: str, questions: Mapping[str, Mapping[str, str]] ) -> Mapping[str, float]: from google.genai import types prompt = ( f"{INSTRUCTIONS}\n\nQuestions:\n" f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n" f"Document:\n\n{document}\n" ) response = await self._client.aio.models.generate_content( model=self._model, contents=prompt, config=types.GenerateContentConfig( temperature=0, response_mime_type="application/json" ), ) return parse_scores(response.text or "", questions) def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]: """The judge's reply as scores; a missing or malformed answer scores 0.""" try: raw = json.loads(text) except ValueError: raw = {} if not isinstance(raw, dict): raw = {} scores = {} for key in questions: value = raw.get(key, 0) if isinstance(value, bool) or not isinstance(value, (int, float)): value = 1.0 if value is True else 0.0 scores[key] = min(1.0, max(0.0, float(value))) return scores