fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -8,8 +8,9 @@ Usage:
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
--judge adds the judged checks (pillar checklist coverage, recommendation quality).
The judge is Jev when TYPESAFE_API_KEY is set; otherwise, or with EVAL_JUDGE=gemini,
a language model: the provider bundle's default unless EVAL_JUDGE_MODEL names another.
A case marked "interactive" runs with a scripted user in the loop. The exit code is
non-zero when a case fails or exceeds one of its thresholds.
"""
@@ -36,7 +37,7 @@ from app.nodes.execution_nodes import automatic_reply
from app.recorder import InMemoryRecorder
from app.states.execution_state import execution_dir
from app.tools.local import TerraformRunner
from eval.judge import GeminiJudge, Judge
from eval.judge import DeciderJudge, GeminiJudge, Judge
from eval.scoring import (
CaseResult,
CaseSpec,
@@ -209,6 +210,8 @@ async def run_case(
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"revisions": revisions,
# Every routing decision: the group that applied, the action, what was judged.
"decisions": interaction.state.get("decisions", []),
"warnings": interaction.state.get("warnings", []),
"conversation": interaction.state.get("conversation", []),
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
@@ -217,8 +220,13 @@ async def run_case(
def make_judge(settings: Settings) -> Judge:
"""Jev when a TypeSafe key is set, unless EVAL_JUDGE=gemini asks for the
language-model judge; otherwise the language-model judge."""
from app.agent import open_decider
from app.providers import load_provider
if settings.TYPESAFE_API_KEY and os.environ.get("EVAL_JUDGE") != "gemini":
return DeciderJudge(open_decider(settings))
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
settings.CLOUD_PROVIDER
).models.default.removeprefix(GEMINI_PREFIX)