This commit is contained in:
@@ -8,8 +8,9 @@ Usage:
|
||||
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
||||
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
||||
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
|
||||
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
|
||||
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
|
||||
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
|
||||
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
||||
non-zero when a case fails or exceeds one of its thresholds.
|
||||
"""
|
||||
@@ -36,7 +37,7 @@ from app.nodes.execution_nodes import automatic_reply
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.execution_state import execution_dir
|
||||
from app.tools.local import TerraformRunner
|
||||
from eval.judge import GeminiJudge, Judge
|
||||
from eval.judge import DeciderJudge, GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
@@ -209,6 +210,8 @@ async def run_case(
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"revisions": revisions,
|
||||
# Every routing decision: the group that applied, the action, what was judged.
|
||||
"decisions": interaction.state.get("decisions", []),
|
||||
"warnings": interaction.state.get("warnings", []),
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
@@ -217,8 +220,13 @@ async def run_case(
|
||||
|
||||
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
|
||||
language-model judge; otherwise the language-model judge."""
|
||||
from app.agent import open_decider
|
||||
from app.providers import load_provider
|
||||
|
||||
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
|
||||
return DeciderJudge(open_decider(settings))
|
||||
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
||||
settings.CLOUD_PROVIDER
|
||||
).models.default.removeprefix(GEMINI_PREFIX)
|
||||
|
||||
Reference in New Issue
Block a user