feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
184
eval/calibrate_judge.py
Normal file
184
eval/calibrate_judge.py
Normal file
@@ -0,0 +1,184 @@
|
||||
"""Calibrate the evaluation's judges against a person's labels.
|
||||
|
||||
A labelled set is questions about saved design recommendations, each with the
|
||||
answer a person gave. This asks every judge each question several times and reports:
|
||||
|
||||
agreement how often the judge's answer matches the person's
|
||||
stability how often the judge gives the same answer to the same question
|
||||
between how often the two judges agree with each other
|
||||
|
||||
Stability and agreement between judges need no labels, so they can be measured
|
||||
before anyone has marked the set. Agreement with a person cannot.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python -m eval.calibrate_judge [--runs 3] [--build]
|
||||
|
||||
--build writes eval/datasets/judge/labels.json from the documents in that folder,
|
||||
with `label` left empty for a person to fill in: 1 or 0 for a yes_no question; 0,
|
||||
0.33, 0.67 or 1 for a graded one. Costs model usage: run deliberately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.agent import open_decider
|
||||
from app.config import get_settings
|
||||
from app.providers import load_provider
|
||||
from eval.judge import DeciderJudge, GeminiJudge, Judge, JudgeUnavailable
|
||||
from eval.scoring import QUALITY_RUBRIC, checklist_items
|
||||
|
||||
FOLDER = Path(__file__).resolve().parent / "datasets" / "judge"
|
||||
LABELS = FOLDER / "labels.json"
|
||||
GEMINI_PREFIX = "gemini/"
|
||||
PER_PILLAR = 2 # checklist items sampled from each pillar, per document
|
||||
TOLERANCE = 0.17 # two graded answers within half a grade count as the same
|
||||
|
||||
|
||||
def build(seed: int = 7) -> list[dict[str, Any]]:
|
||||
"""Twenty-odd questions over the saved documents, labels left for a person."""
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
chooser = random.Random(seed)
|
||||
items: list[dict[str, Any]] = []
|
||||
for document in sorted(FOLDER.glob("*.md")):
|
||||
for pillar, skill in bundle.pillar_skills.items():
|
||||
checklist = checklist_items(bundle.skills[skill].path / "SKILL.md")
|
||||
for item in chooser.sample(checklist, min(PER_PILLAR, len(checklist))):
|
||||
items.append(
|
||||
{
|
||||
"id": f"{document.stem}.{pillar}.{len(items)}",
|
||||
"document": document.name,
|
||||
"kind": "yes_no",
|
||||
"text": f"Do the recommendations address this {pillar} "
|
||||
"checklist item, either by recommending it or by "
|
||||
f"explaining why it does not apply? {item}",
|
||||
"label": None,
|
||||
}
|
||||
)
|
||||
for key, text in QUALITY_RUBRIC.items():
|
||||
items.append(
|
||||
{
|
||||
"id": f"{document.stem}.quality.{key}",
|
||||
"document": document.name,
|
||||
"kind": "graded",
|
||||
"text": text,
|
||||
"label": None,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def same(a: float, b: float) -> bool:
|
||||
return abs(a - b) <= TOLERANCE
|
||||
|
||||
|
||||
def share(pairs: Sequence[tuple[float, float]]) -> float | None:
|
||||
return round(sum(same(a, b) for a, b in pairs) / len(pairs), 2) if pairs else None
|
||||
|
||||
|
||||
async def ask(
|
||||
judge: Judge, items: Sequence[Mapping[str, Any]], runs: int
|
||||
) -> dict[str, list[float]]:
|
||||
"""Every question's answers, one per run, by question id."""
|
||||
answers: dict[str, list[float]] = {item["id"]: [] for item in items}
|
||||
documents = sorted({item["document"] for item in items})
|
||||
for _ in range(runs):
|
||||
for name in documents:
|
||||
asked = {
|
||||
item["id"]: {"kind": item["kind"], "text": item["text"]}
|
||||
for item in items
|
||||
if item["document"] == name
|
||||
}
|
||||
try:
|
||||
scores = await judge.answer(
|
||||
(FOLDER / name).read_text(encoding="utf-8"), asked
|
||||
)
|
||||
except JudgeUnavailable:
|
||||
continue
|
||||
for key, value in scores.items():
|
||||
if key in answers:
|
||||
answers[key].append(float(value))
|
||||
return answers
|
||||
|
||||
|
||||
def report(
|
||||
items: Sequence[Mapping[str, Any]], by_judge: Mapping[str, dict[str, list[float]]]
|
||||
) -> dict[str, Any]:
|
||||
labelled = {i["id"]: float(i["label"]) for i in items if i["label"] is not None}
|
||||
result: dict[str, Any] = {
|
||||
"questions": len(items),
|
||||
"labelled": len(labelled),
|
||||
"judges": {},
|
||||
}
|
||||
for name, answers in by_judge.items():
|
||||
first = {key: values[0] for key, values in answers.items() if values}
|
||||
result["judges"][name] = {
|
||||
"answered": len(first),
|
||||
# Same answer on every run, per question.
|
||||
"stability": share(
|
||||
[
|
||||
(values[0], other)
|
||||
for values in answers.values()
|
||||
for other in values[1:]
|
||||
]
|
||||
),
|
||||
"agreement_with_labels": share(
|
||||
[(first[key], label) for key, label in labelled.items() if key in first]
|
||||
),
|
||||
"mean_answer": round(sum(first.values()) / len(first), 2)
|
||||
if first
|
||||
else None,
|
||||
}
|
||||
names = list(by_judge)
|
||||
if len(names) == 2:
|
||||
a, b = (by_judge[name] for name in names)
|
||||
result["between_judges"] = share(
|
||||
[(a[key][0], b[key][0]) for key in a if a[key] and b.get(key)]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
if "--build" in arguments:
|
||||
items = build()
|
||||
LABELS.write_text(json.dumps(items, indent=2, ensure_ascii=False) + "\n")
|
||||
print(f"Wrote {len(items)} questions to {LABELS}; fill in each `label`.")
|
||||
return 0
|
||||
runs = int(arguments[arguments.index("--runs") + 1]) if "--runs" in arguments else 3
|
||||
items = json.loads(LABELS.read_text(encoding="utf-8"))
|
||||
settings = get_settings()
|
||||
model = load_provider(settings.CLOUD_PROVIDER).models.default
|
||||
by_judge: dict[str, dict[str, list[float]]] = {
|
||||
"jev": {item["id"]: [] for item in items},
|
||||
"gemini": {item["id"]: [] for item in items},
|
||||
}
|
||||
for _ in range(runs):
|
||||
# A new Jev client for each run: the client caches identical requests, and a
|
||||
# cached answer would make stability look perfect whatever the service does.
|
||||
decider = open_decider(settings)
|
||||
judges: dict[str, Judge] = {
|
||||
"jev": DeciderJudge(decider, scope="judge-calibration"),
|
||||
"gemini": GeminiJudge(model.removeprefix(GEMINI_PREFIX)),
|
||||
}
|
||||
try:
|
||||
for name, judge in judges.items():
|
||||
for key, values in (await ask(judge, items, 1)).items():
|
||||
by_judge[name][key] += values
|
||||
finally:
|
||||
aclose = getattr(decider, "aclose", None)
|
||||
if aclose:
|
||||
await aclose()
|
||||
result = {"runs": runs, **report(items, by_judge)}
|
||||
(FOLDER / "calibration.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main(sys.argv[1:])))
|
||||
Reference in New Issue
Block a user