This commit is contained in:
@@ -1,8 +1,11 @@
|
||||
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
||||
|
||||
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
||||
question. The shape (one state, many typed questions) is deliberately the one a
|
||||
decision model takes, so the Gemini judge here can be swapped for one later.
|
||||
question. The shape is the one a decision model takes: one state, many typed
|
||||
questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is
|
||||
the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask
|
||||
every question of every run and returns the same answer for the same input.
|
||||
`GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,6 +14,8 @@ import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.adk.runtime import Decider, Question
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"You grade a document from a cloud solution architecture engagement. Answer every "
|
||||
"question about the document only, strictly and without benefit of the doubt. "
|
||||
@@ -31,6 +36,48 @@ class Judge(Protocol):
|
||||
...
|
||||
|
||||
|
||||
GRADES = ("not at all", "weakly", "mostly", "fully")
|
||||
|
||||
|
||||
class JudgeUnavailable(RuntimeError):
|
||||
"""The judge gave no answer; the check it serves is unanswered, not failed."""
|
||||
|
||||
|
||||
class DeciderJudge:
|
||||
"""Jev as the judge. The document is the state and carries no grading
|
||||
instructions; each question is asked on its own terms, all in one call."""
|
||||
|
||||
def __init__(self, decider: Decider, scope: str = "evaluation") -> None:
|
||||
self._decider = decider
|
||||
self._scope = scope
|
||||
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
asked = {
|
||||
key: Question("score", item["text"], GRADES)
|
||||
if item["kind"] == "graded"
|
||||
else Question("noul", item["text"])
|
||||
for key, item in questions.items()
|
||||
}
|
||||
answers = await self._decider.decide(document, asked, scope=self._scope)
|
||||
if answers is None:
|
||||
raise JudgeUnavailable("Jev gave no answer")
|
||||
scores = {}
|
||||
for key, question in asked.items():
|
||||
value = float(answers[key].value)
|
||||
# A graded answer is a level from 0 to 3; a yes-or-no answer is the
|
||||
# probability of yes, counted as a yes at an even chance or better.
|
||||
scores[key] = (
|
||||
min(1.0, max(0.0, value / (len(GRADES) - 1)))
|
||||
if question.kind == "score"
|
||||
else 1.0
|
||||
if value >= 0.5
|
||||
else 0.0
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
class GeminiJudge:
|
||||
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ from app.workflows.target_architecture import (
|
||||
parse_graph,
|
||||
render_mermaid,
|
||||
)
|
||||
from eval.judge import Judge
|
||||
from eval.judge import Judge, JudgeUnavailable
|
||||
|
||||
PASS_THRESHOLD = 0.8
|
||||
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
|
||||
@@ -403,7 +403,10 @@ async def _checklist_coverage(
|
||||
}
|
||||
if not recommendations or not questions:
|
||||
return Check(name, False, "no design recommendations or no checklists")
|
||||
scores = await judge.answer(recommendations, questions)
|
||||
try:
|
||||
scores = await judge.answer(recommendations, questions)
|
||||
except JudgeUnavailable as exc:
|
||||
return Check(name, False, f"judge unavailable: {exc}")
|
||||
per_pillar = {
|
||||
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
|
||||
/ len(items)
|
||||
@@ -419,10 +422,16 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
|
||||
name = "recommendations_quality"
|
||||
if not recommendations:
|
||||
return Check(name, False, "no design recommendations")
|
||||
scores = await judge.answer(
|
||||
recommendations,
|
||||
{key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items()},
|
||||
)
|
||||
try:
|
||||
scores = await judge.answer(
|
||||
recommendations,
|
||||
{
|
||||
key: {"kind": "graded", "text": text}
|
||||
for key, text in QUALITY_RUBRIC.items()
|
||||
},
|
||||
)
|
||||
except JudgeUnavailable as exc:
|
||||
return Check(name, False, f"judge unavailable: {exc}")
|
||||
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
||||
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
||||
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
||||
|
||||
@@ -8,8 +8,9 @@ Usage:
|
||||
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
||||
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
||||
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
|
||||
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
|
||||
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
|
||||
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
|
||||
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
||||
non-zero when a case fails or exceeds one of its thresholds.
|
||||
"""
|
||||
@@ -36,7 +37,7 @@ from app.nodes.execution_nodes import automatic_reply
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.execution_state import execution_dir
|
||||
from app.tools.local import TerraformRunner
|
||||
from eval.judge import GeminiJudge, Judge
|
||||
from eval.judge import DeciderJudge, GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
@@ -209,6 +210,8 @@ async def run_case(
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"revisions": revisions,
|
||||
# Every routing decision: the group that applied, the action, what was judged.
|
||||
"decisions": interaction.state.get("decisions", []),
|
||||
"warnings": interaction.state.get("warnings", []),
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
@@ -217,8 +220,13 @@ async def run_case(
|
||||
|
||||
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
|
||||
language-model judge; otherwise the language-model judge."""
|
||||
from app.agent import open_decider
|
||||
from app.providers import load_provider
|
||||
|
||||
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
|
||||
return DeciderJudge(open_decider(settings))
|
||||
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
||||
settings.CLOUD_PROVIDER
|
||||
).models.default.removeprefix(GEMINI_PREFIX)
|
||||
|
||||
Reference in New Issue
Block a user