Some checks failed
validation / verify (push) Failing after 11s
A pillar's checklist item the request gives no information about was marked "not addressed", which reads as a fault in the design. It is missing information. The checklist review gains a fourth outcome, to confirm with the customer, which must carry the question to ask. The questions are written into the design recommendations and the guide, and to customer-questions.md; the evaluation's judged check counts such an item as accounted for. Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's retention, training, context and rate limits (iteration 5 step 1), and the owner's labels, which showed the judged question was wrong, not the judges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
184 lines
6.9 KiB
Python
184 lines
6.9 KiB
Python
"""Calibrate the evaluation's judges against a person's labels.
|
|
|
|
A labelled set is questions about saved design recommendations, each with the
|
|
answer a person gave. This asks every judge each question several times and reports:
|
|
|
|
agreement how often the judge's answer matches the person's
|
|
stability how often the judge gives the same answer to the same question
|
|
between how often the two judges agree with each other
|
|
|
|
Stability and agreement between judges need no labels, so they can be measured
|
|
before anyone has marked the set. Agreement with a person cannot.
|
|
|
|
Usage:
|
|
.venv/bin/python -m eval.calibrate_judge [--runs 3] [--build]
|
|
|
|
--build writes eval/datasets/judge/labels.json from the documents in that folder,
|
|
with `label` left empty for a person to fill in: 1 or 0 for a yes_no question; 0,
|
|
0.33, 0.67 or 1 for a graded one. Costs model usage: run deliberately.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import random
|
|
import sys
|
|
from collections.abc import Mapping, Sequence
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.agent import open_decider
|
|
from app.config import get_settings
|
|
from app.providers import load_provider
|
|
from app.tools.assessment import checklist_question
|
|
from eval.judge import DeciderJudge, GeminiJudge, Judge, JudgeUnavailable
|
|
from eval.scoring import QUALITY_RUBRIC, checklist_items
|
|
|
|
FOLDER = Path(__file__).resolve().parent / "datasets" / "judge"
|
|
LABELS = FOLDER / "labels.json"
|
|
GEMINI_PREFIX = "gemini/"
|
|
PER_PILLAR = 2 # checklist items sampled from each pillar, per document
|
|
TOLERANCE = 0.17 # two graded answers within half a grade count as the same
|
|
|
|
|
|
def build(seed: int = 7) -> list[dict[str, Any]]:
|
|
"""Twenty-odd questions over the saved documents, labels left for a person."""
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
chooser = random.Random(seed)
|
|
items: list[dict[str, Any]] = []
|
|
for document in sorted(FOLDER.glob("*.md")):
|
|
for pillar, skill in bundle.pillar_skills.items():
|
|
checklist = checklist_items(bundle.skills[skill].path / "SKILL.md")
|
|
for item in chooser.sample(checklist, min(PER_PILLAR, len(checklist))):
|
|
items.append(
|
|
{
|
|
"id": f"{document.stem}.{pillar}.{len(items)}",
|
|
"document": document.name,
|
|
"kind": "yes_no",
|
|
"text": checklist_question(pillar, item),
|
|
"label": None,
|
|
}
|
|
)
|
|
for key, text in QUALITY_RUBRIC.items():
|
|
items.append(
|
|
{
|
|
"id": f"{document.stem}.quality.{key}",
|
|
"document": document.name,
|
|
"kind": "graded",
|
|
"text": text,
|
|
"label": None,
|
|
}
|
|
)
|
|
return items
|
|
|
|
|
|
def same(a: float, b: float) -> bool:
|
|
return abs(a - b) <= TOLERANCE
|
|
|
|
|
|
def share(pairs: Sequence[tuple[float, float]]) -> float | None:
|
|
return round(sum(same(a, b) for a, b in pairs) / len(pairs), 2) if pairs else None
|
|
|
|
|
|
async def ask(
|
|
judge: Judge, items: Sequence[Mapping[str, Any]], runs: int
|
|
) -> dict[str, list[float]]:
|
|
"""Every question's answers, one per run, by question id."""
|
|
answers: dict[str, list[float]] = {item["id"]: [] for item in items}
|
|
documents = sorted({item["document"] for item in items})
|
|
for _ in range(runs):
|
|
for name in documents:
|
|
asked = {
|
|
item["id"]: {"kind": item["kind"], "text": item["text"]}
|
|
for item in items
|
|
if item["document"] == name
|
|
}
|
|
try:
|
|
scores = await judge.answer(
|
|
(FOLDER / name).read_text(encoding="utf-8"), asked
|
|
)
|
|
except JudgeUnavailable:
|
|
continue
|
|
for key, value in scores.items():
|
|
if key in answers:
|
|
answers[key].append(float(value))
|
|
return answers
|
|
|
|
|
|
def report(
|
|
items: Sequence[Mapping[str, Any]], by_judge: Mapping[str, dict[str, list[float]]]
|
|
) -> dict[str, Any]:
|
|
labelled = {i["id"]: float(i["label"]) for i in items if i["label"] is not None}
|
|
result: dict[str, Any] = {
|
|
"questions": len(items),
|
|
"labelled": len(labelled),
|
|
"judges": {},
|
|
}
|
|
for name, answers in by_judge.items():
|
|
first = {key: values[0] for key, values in answers.items() if values}
|
|
result["judges"][name] = {
|
|
"answered": len(first),
|
|
# Same answer on every run, per question.
|
|
"stability": share(
|
|
[
|
|
(values[0], other)
|
|
for values in answers.values()
|
|
for other in values[1:]
|
|
]
|
|
),
|
|
"agreement_with_labels": share(
|
|
[(first[key], label) for key, label in labelled.items() if key in first]
|
|
),
|
|
"mean_answer": round(sum(first.values()) / len(first), 2)
|
|
if first
|
|
else None,
|
|
}
|
|
names = list(by_judge)
|
|
if len(names) == 2:
|
|
a, b = (by_judge[name] for name in names)
|
|
result["between_judges"] = share(
|
|
[(a[key][0], b[key][0]) for key in a if a[key] and b.get(key)]
|
|
)
|
|
return result
|
|
|
|
|
|
async def main(arguments: Sequence[str]) -> int:
|
|
if "--build" in arguments:
|
|
items = build()
|
|
LABELS.write_text(json.dumps(items, indent=2, ensure_ascii=False) + "\n")
|
|
print(f"Wrote {len(items)} questions to {LABELS}; fill in each `label`.")
|
|
return 0
|
|
runs = int(arguments[arguments.index("--runs") + 1]) if "--runs" in arguments else 3
|
|
items = json.loads(LABELS.read_text(encoding="utf-8"))
|
|
settings = get_settings()
|
|
model = load_provider(settings.CLOUD_PROVIDER).models.default
|
|
by_judge: dict[str, dict[str, list[float]]] = {
|
|
"jev": {item["id"]: [] for item in items},
|
|
"gemini": {item["id"]: [] for item in items},
|
|
}
|
|
for _ in range(runs):
|
|
# A new Jev client for each run: the client caches identical requests, and a
|
|
# cached answer would make stability look perfect whatever the service does.
|
|
decider = open_decider(settings)
|
|
judges: dict[str, Judge] = {
|
|
"jev": DeciderJudge(decider, scope="judge-calibration"),
|
|
"gemini": GeminiJudge(model.removeprefix(GEMINI_PREFIX)),
|
|
}
|
|
try:
|
|
for name, judge in judges.items():
|
|
for key, values in (await ask(judge, items, 1)).items():
|
|
by_judge[name][key] += values
|
|
finally:
|
|
aclose = getattr(decider, "aclose", None)
|
|
if aclose:
|
|
await aclose()
|
|
result = {"runs": runs, **report(items, by_judge)}
|
|
(FOLDER / "calibration.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
print(json.dumps(result, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main(sys.argv[1:])))
|