Files
gcp_solution_architecture_a…/eval/calibrate_judge.py
Jonathan Boniface a48468d477
Some checks failed
validation / verify (push) Failing after 11s
feat: what a request does not say is a question for the customer
A pillar's checklist item the request gives no information about was marked
"not addressed", which reads as a fault in the design. It is missing
information. The checklist review gains a fourth outcome, to confirm with the
customer, which must carry the question to ask. The questions are written
into the design recommendations and the guide, and to customer-questions.md;
the evaluation's judged check counts such an item as accounted for.

Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's
retention, training, context and rate limits (iteration 5 step 1), and the
owner's labels, which showed the judged question was wrong, not the judges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 19:30:12 +01:00

184 lines
6.9 KiB
Python

"""Calibrate the evaluation's judges against a person's labels.
A labelled set is questions about saved design recommendations, each with the
answer a person gave. This asks every judge each question several times and reports:
agreement how often the judge's answer matches the person's
stability how often the judge gives the same answer to the same question
between how often the two judges agree with each other
Stability and agreement between judges need no labels, so they can be measured
before anyone has marked the set. Agreement with a person cannot.
Usage:
.venv/bin/python -m eval.calibrate_judge [--runs 3] [--build]
--build writes eval/datasets/judge/labels.json from the documents in that folder,
with `label` left empty for a person to fill in: 1 or 0 for a yes_no question; 0,
0.33, 0.67 or 1 for a graded one. Costs model usage: run deliberately.
"""
from __future__ import annotations
import asyncio
import json
import random
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from app.agent import open_decider
from app.config import get_settings
from app.providers import load_provider
from app.tools.assessment import checklist_question
from eval.judge import DeciderJudge, GeminiJudge, Judge, JudgeUnavailable
from eval.scoring import QUALITY_RUBRIC, checklist_items
FOLDER = Path(__file__).resolve().parent / "datasets" / "judge"
LABELS = FOLDER / "labels.json"
GEMINI_PREFIX = "gemini/"
PER_PILLAR = 2 # checklist items sampled from each pillar, per document
TOLERANCE = 0.17 # two graded answers within half a grade count as the same
def build(seed: int = 7) -> list[dict[str, Any]]:
"""Twenty-odd questions over the saved documents, labels left for a person."""
bundle = load_provider(get_settings().CLOUD_PROVIDER)
chooser = random.Random(seed)
items: list[dict[str, Any]] = []
for document in sorted(FOLDER.glob("*.md")):
for pillar, skill in bundle.pillar_skills.items():
checklist = checklist_items(bundle.skills[skill].path / "SKILL.md")
for item in chooser.sample(checklist, min(PER_PILLAR, len(checklist))):
items.append(
{
"id": f"{document.stem}.{pillar}.{len(items)}",
"document": document.name,
"kind": "yes_no",
"text": checklist_question(pillar, item),
"label": None,
}
)
for key, text in QUALITY_RUBRIC.items():
items.append(
{
"id": f"{document.stem}.quality.{key}",
"document": document.name,
"kind": "graded",
"text": text,
"label": None,
}
)
return items
def same(a: float, b: float) -> bool:
return abs(a - b) <= TOLERANCE
def share(pairs: Sequence[tuple[float, float]]) -> float | None:
return round(sum(same(a, b) for a, b in pairs) / len(pairs), 2) if pairs else None
async def ask(
judge: Judge, items: Sequence[Mapping[str, Any]], runs: int
) -> dict[str, list[float]]:
"""Every question's answers, one per run, by question id."""
answers: dict[str, list[float]] = {item["id"]: [] for item in items}
documents = sorted({item["document"] for item in items})
for _ in range(runs):
for name in documents:
asked = {
item["id"]: {"kind": item["kind"], "text": item["text"]}
for item in items
if item["document"] == name
}
try:
scores = await judge.answer(
(FOLDER / name).read_text(encoding="utf-8"), asked
)
except JudgeUnavailable:
continue
for key, value in scores.items():
if key in answers:
answers[key].append(float(value))
return answers
def report(
items: Sequence[Mapping[str, Any]], by_judge: Mapping[str, dict[str, list[float]]]
) -> dict[str, Any]:
labelled = {i["id"]: float(i["label"]) for i in items if i["label"] is not None}
result: dict[str, Any] = {
"questions": len(items),
"labelled": len(labelled),
"judges": {},
}
for name, answers in by_judge.items():
first = {key: values[0] for key, values in answers.items() if values}
result["judges"][name] = {
"answered": len(first),
# Same answer on every run, per question.
"stability": share(
[
(values[0], other)
for values in answers.values()
for other in values[1:]
]
),
"agreement_with_labels": share(
[(first[key], label) for key, label in labelled.items() if key in first]
),
"mean_answer": round(sum(first.values()) / len(first), 2)
if first
else None,
}
names = list(by_judge)
if len(names) == 2:
a, b = (by_judge[name] for name in names)
result["between_judges"] = share(
[(a[key][0], b[key][0]) for key in a if a[key] and b.get(key)]
)
return result
async def main(arguments: Sequence[str]) -> int:
if "--build" in arguments:
items = build()
LABELS.write_text(json.dumps(items, indent=2, ensure_ascii=False) + "\n")
print(f"Wrote {len(items)} questions to {LABELS}; fill in each `label`.")
return 0
runs = int(arguments[arguments.index("--runs") + 1]) if "--runs" in arguments else 3
items = json.loads(LABELS.read_text(encoding="utf-8"))
settings = get_settings()
model = load_provider(settings.CLOUD_PROVIDER).models.default
by_judge: dict[str, dict[str, list[float]]] = {
"jev": {item["id"]: [] for item in items},
"gemini": {item["id"]: [] for item in items},
}
for _ in range(runs):
# A new Jev client for each run: the client caches identical requests, and a
# cached answer would make stability look perfect whatever the service does.
decider = open_decider(settings)
judges: dict[str, Judge] = {
"jev": DeciderJudge(decider, scope="judge-calibration"),
"gemini": GeminiJudge(model.removeprefix(GEMINI_PREFIX)),
}
try:
for name, judge in judges.items():
for key, values in (await ask(judge, items, 1)).items():
by_judge[name][key] += values
finally:
aclose = getattr(decider, "aclose", None)
if aclose:
await aclose()
result = {"runs": runs, **report(items, by_judge)}
(FOLDER / "calibration.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))