"""Calibrate the evaluation's judges against a person's labels. A labelled set is questions about saved design recommendations, each with the answer a person gave. This asks every judge each question several times and reports: agreement how often the judge's answer matches the person's stability how often the judge gives the same answer to the same question between how often the two judges agree with each other Stability and agreement between judges need no labels, so they can be measured before anyone has marked the set. Agreement with a person cannot. Usage: .venv/bin/python -m eval.calibrate_judge [--runs 3] [--build] --build writes eval/datasets/judge/labels.json from the documents in that folder, with `label` left empty for a person to fill in: 1 or 0 for a yes_no question; 0, 0.33, 0.67 or 1 for a graded one. Costs model usage: run deliberately. """ from __future__ import annotations import asyncio import json import random import sys from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any from app.agent import open_decider from app.config import get_settings from app.providers import load_provider from app.tools.assessment import checklist_question from eval.judge import DeciderJudge, GeminiJudge, Judge, JudgeUnavailable from eval.scoring import QUALITY_RUBRIC, checklist_items FOLDER = Path(__file__).resolve().parent / "datasets" / "judge" LABELS = FOLDER / "labels.json" GEMINI_PREFIX = "gemini/" PER_PILLAR = 2 # checklist items sampled from each pillar, per document TOLERANCE = 0.17 # two graded answers within half a grade count as the same def build(seed: int = 7) -> list[dict[str, Any]]: """Twenty-odd questions over the saved documents, labels left for a person.""" bundle = load_provider(get_settings().CLOUD_PROVIDER) chooser = random.Random(seed) items: list[dict[str, Any]] = [] for document in sorted(FOLDER.glob("*.md")): for pillar, skill in bundle.pillar_skills.items(): checklist = checklist_items(bundle.skills[skill].path / "SKILL.md") for item in chooser.sample(checklist, min(PER_PILLAR, len(checklist))): items.append( { "id": f"{document.stem}.{pillar}.{len(items)}", "document": document.name, "kind": "yes_no", "text": checklist_question(pillar, item), "label": None, } ) for key, text in QUALITY_RUBRIC.items(): items.append( { "id": f"{document.stem}.quality.{key}", "document": document.name, "kind": "graded", "text": text, "label": None, } ) return items def same(a: float, b: float) -> bool: return abs(a - b) <= TOLERANCE def share(pairs: Sequence[tuple[float, float]]) -> float | None: return round(sum(same(a, b) for a, b in pairs) / len(pairs), 2) if pairs else None async def ask( judge: Judge, items: Sequence[Mapping[str, Any]], runs: int ) -> dict[str, list[float]]: """Every question's answers, one per run, by question id.""" answers: dict[str, list[float]] = {item["id"]: [] for item in items} documents = sorted({item["document"] for item in items}) for _ in range(runs): for name in documents: asked = { item["id"]: {"kind": item["kind"], "text": item["text"]} for item in items if item["document"] == name } try: scores = await judge.answer( (FOLDER / name).read_text(encoding="utf-8"), asked ) except JudgeUnavailable: continue for key, value in scores.items(): if key in answers: answers[key].append(float(value)) return answers def report( items: Sequence[Mapping[str, Any]], by_judge: Mapping[str, dict[str, list[float]]] ) -> dict[str, Any]: labelled = {i["id"]: float(i["label"]) for i in items if i["label"] is not None} result: dict[str, Any] = { "questions": len(items), "labelled": len(labelled), "judges": {}, } for name, answers in by_judge.items(): first = {key: values[0] for key, values in answers.items() if values} result["judges"][name] = { "answered": len(first), # Same answer on every run, per question. "stability": share( [ (values[0], other) for values in answers.values() for other in values[1:] ] ), "agreement_with_labels": share( [(first[key], label) for key, label in labelled.items() if key in first] ), "mean_answer": round(sum(first.values()) / len(first), 2) if first else None, } names = list(by_judge) if len(names) == 2: a, b = (by_judge[name] for name in names) result["between_judges"] = share( [(a[key][0], b[key][0]) for key in a if a[key] and b.get(key)] ) return result async def main(arguments: Sequence[str]) -> int: if "--build" in arguments: items = build() LABELS.write_text(json.dumps(items, indent=2, ensure_ascii=False) + "\n") print(f"Wrote {len(items)} questions to {LABELS}; fill in each `label`.") return 0 runs = int(arguments[arguments.index("--runs") + 1]) if "--runs" in arguments else 3 items = json.loads(LABELS.read_text(encoding="utf-8")) settings = get_settings() model = load_provider(settings.CLOUD_PROVIDER).models.default by_judge: dict[str, dict[str, list[float]]] = { "jev": {item["id"]: [] for item in items}, "gemini": {item["id"]: [] for item in items}, } for _ in range(runs): # A new Jev client for each run: the client caches identical requests, and a # cached answer would make stability look perfect whatever the service does. decider = open_decider(settings) judges: dict[str, Judge] = { "jev": DeciderJudge(decider, scope="judge-calibration"), "gemini": GeminiJudge(model.removeprefix(GEMINI_PREFIX)), } try: for name, judge in judges.items(): for key, values in (await ask(judge, items, 1)).items(): by_judge[name][key] += values finally: aclose = getattr(decider, "aclose", None) if aclose: await aclose() result = {"runs": runs, **report(items, by_judge)} (FOLDER / "calibration.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result, indent=2)) return 0 if __name__ == "__main__": sys.exit(asyncio.run(main(sys.argv[1:])))