fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -1,8 +1,11 @@
"""Judges: graded answers about a Deliverable that string matching cannot give.
A judge takes a document and a set of questions and returns a score from 0 to 1 per
question. The shape (one state, many typed questions) is deliberately the one a
decision model takes, so the Gemini judge here can be swapped for one later.
question. The shape is the one a decision model takes: one state, many typed
questions, answered in a single call. `DeciderJudge` puts Jev in that seat, which is
the judge to prefer for narrow, typed criteria at volume: it is cheap enough to ask
every question of every run and returns the same answer for the same input.
`GeminiJudge` is a language-model judge for comparison, and for when no Jev key is set.
"""
from __future__ import annotations
@@ -11,6 +14,8 @@ import json
from collections.abc import Mapping
from typing import Any, Protocol
from app.adk.runtime import Decider, Question
INSTRUCTIONS = (
"You grade a document from a cloud solution architecture engagement. Answer every "
"question about the document only, strictly and without benefit of the doubt. "
@@ -31,6 +36,48 @@ class Judge(Protocol):
...
GRADES = ("not at all", "weakly", "mostly", "fully")
class JudgeUnavailable(RuntimeError):
"""The judge gave no answer; the check it serves is unanswered, not failed."""
class DeciderJudge:
"""Jev as the judge. The document is the state and carries no grading
instructions; each question is asked on its own terms, all in one call."""
def __init__(self, decider: Decider, scope: str = "evaluation") -> None:
self._decider = decider
self._scope = scope
async def answer(
self, document: str, questions: Mapping[str, Mapping[str, str]]
) -> Mapping[str, float]:
asked = {
key: Question("score", item["text"], GRADES)
if item["kind"] == "graded"
else Question("noul", item["text"])
for key, item in questions.items()
}
answers = await self._decider.decide(document, asked, scope=self._scope)
if answers is None:
raise JudgeUnavailable("Jev gave no answer")
scores = {}
for key, question in asked.items():
value = float(answers[key].value)
# A graded answer is a level from 0 to 3; a yes-or-no answer is the
# probability of yes, counted as a yes at an even chance or better.
scores[key] = (
min(1.0, max(0.0, value / (len(GRADES) - 1)))
if question.kind == "score"
else 1.0
if value >= 0.5
else 0.0
)
return scores
class GeminiJudge:
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""