Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
123 lines
4.4 KiB
Python
123 lines
4.4 KiB
Python
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
|
|
|
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
|
question. The shape is the one a decision model takes: one state, many typed
|
|
questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is
|
|
the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask
|
|
every question of every run and returns the same answer for the same input.
|
|
`GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping
|
|
from typing import Any, Protocol
|
|
|
|
from app.adk.runtime import Decider, Question
|
|
from app.tools.assessment import GRADES
|
|
|
|
INSTRUCTIONS = (
|
|
"You grade a document from a cloud solution architecture engagement. Answer every "
|
|
"question about the document only, strictly and without benefit of the doubt. "
|
|
'Reply with one JSON object mapping each question id to a number: for a "yes_no" '
|
|
'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 '
|
|
"(weakly), 0.67 (mostly) or 1 (fully)."
|
|
)
|
|
|
|
|
|
class Judge(Protocol):
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
"""Score each question, keyed by id, from 0 to 1.
|
|
|
|
`questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}.
|
|
"""
|
|
...
|
|
|
|
|
|
class JudgeUnavailable(RuntimeError):
|
|
"""The judge gave no answer; the check it serves is unanswered, not failed."""
|
|
|
|
|
|
class DeciderJudge:
|
|
"""Jev as the judge. The document is the state and carries no grading
|
|
instructions; each question is asked on its own terms, all in one call."""
|
|
|
|
def __init__(self, decider: Decider, scope: str = "evaluation") -> None:
|
|
self._decider = decider
|
|
self._scope = scope
|
|
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
asked = {
|
|
key: Question("score", item["text"], GRADES)
|
|
if item["kind"] == "graded"
|
|
else Question("noul", item["text"])
|
|
for key, item in questions.items()
|
|
}
|
|
answers = await self._decider.decide(document, asked, scope=self._scope)
|
|
if answers is None:
|
|
raise JudgeUnavailable("Jev gave no answer")
|
|
scores = {}
|
|
for key, question in asked.items():
|
|
value = float(answers[key].value)
|
|
# A graded answer is a level from 0 to 3; a yes-or-no answer is the
|
|
# probability of yes, counted as a yes at an even chance or better.
|
|
scores[key] = (
|
|
min(1.0, max(0.0, value / (len(GRADES) - 1)))
|
|
if question.kind == "score"
|
|
else 1.0
|
|
if value >= 0.5
|
|
else 0.0
|
|
)
|
|
return scores
|
|
|
|
|
|
class GeminiJudge:
|
|
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
|
|
|
def __init__(self, model: str) -> None:
|
|
from google import genai
|
|
|
|
self._client = genai.Client()
|
|
self._model = model
|
|
|
|
async def answer(
|
|
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
|
) -> Mapping[str, float]:
|
|
from google.genai import types
|
|
|
|
prompt = (
|
|
f"{INSTRUCTIONS}\n\nQuestions:\n"
|
|
f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n"
|
|
f"Document:\n<document>\n{document}\n</document>"
|
|
)
|
|
response = await self._client.aio.models.generate_content(
|
|
model=self._model,
|
|
contents=prompt,
|
|
config=types.GenerateContentConfig(
|
|
temperature=0, response_mime_type="application/json"
|
|
),
|
|
)
|
|
return parse_scores(response.text or "", questions)
|
|
|
|
|
|
def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]:
|
|
"""The judge's reply as scores; a missing or malformed answer scores 0."""
|
|
try:
|
|
raw = json.loads(text)
|
|
except ValueError:
|
|
raw = {}
|
|
if not isinstance(raw, dict):
|
|
raw = {}
|
|
scores = {}
|
|
for key in questions:
|
|
value = raw.get(key, 0)
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
value = 1.0 if value is True else 0.0
|
|
scores[key] = min(1.0, max(0.0, float(value)))
|
|
return scores
|