feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

184
eval/calibrate_judge.py Normal file
View File

@@ -0,0 +1,184 @@
"""Calibrate the evaluation's judges against a person's labels.
A labelled set is questions about saved design recommendations, each with the
answer a person gave. This asks every judge each question several times and reports:
agreement how often the judge's answer matches the person's
stability how often the judge gives the same answer to the same question
between how often the two judges agree with each other
Stability and agreement between judges need no labels, so they can be measured
before anyone has marked the set. Agreement with a person cannot.
Usage:
.venv/bin/python -m eval.calibrate_judge [--runs 3] [--build]
--build writes eval/datasets/judge/labels.json from the documents in that folder,
with `label` left empty for a person to fill in: 1 or 0 for a yes_no question; 0,
0.33, 0.67 or 1 for a graded one. Costs model usage: run deliberately.
"""
from __future__ import annotations
import asyncio
import json
import random
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from app.agent import open_decider
from app.config import get_settings
from app.providers import load_provider
from eval.judge import DeciderJudge, GeminiJudge, Judge, JudgeUnavailable
from eval.scoring import QUALITY_RUBRIC, checklist_items
FOLDER = Path(__file__).resolve().parent / "datasets" / "judge"
LABELS = FOLDER / "labels.json"
GEMINI_PREFIX = "gemini/"
PER_PILLAR = 2 # checklist items sampled from each pillar, per document
TOLERANCE = 0.17 # two graded answers within half a grade count as the same
def build(seed: int = 7) -> list[dict[str, Any]]:
"""Twenty-odd questions over the saved documents, labels left for a person."""
bundle = load_provider(get_settings().CLOUD_PROVIDER)
chooser = random.Random(seed)
items: list[dict[str, Any]] = []
for document in sorted(FOLDER.glob("*.md")):
for pillar, skill in bundle.pillar_skills.items():
checklist = checklist_items(bundle.skills[skill].path / "SKILL.md")
for item in chooser.sample(checklist, min(PER_PILLAR, len(checklist))):
items.append(
{
"id": f"{document.stem}.{pillar}.{len(items)}",
"document": document.name,
"kind": "yes_no",
"text": f"Do the recommendations address this {pillar} "
"checklist item, either by recommending it or by "
f"explaining why it does not apply? {item}",
"label": None,
}
)
for key, text in QUALITY_RUBRIC.items():
items.append(
{
"id": f"{document.stem}.quality.{key}",
"document": document.name,
"kind": "graded",
"text": text,
"label": None,
}
)
return items
def same(a: float, b: float) -> bool:
return abs(a - b) <= TOLERANCE
def share(pairs: Sequence[tuple[float, float]]) -> float | None:
return round(sum(same(a, b) for a, b in pairs) / len(pairs), 2) if pairs else None
async def ask(
judge: Judge, items: Sequence[Mapping[str, Any]], runs: int
) -> dict[str, list[float]]:
"""Every question's answers, one per run, by question id."""
answers: dict[str, list[float]] = {item["id"]: [] for item in items}
documents = sorted({item["document"] for item in items})
for _ in range(runs):
for name in documents:
asked = {
item["id"]: {"kind": item["kind"], "text": item["text"]}
for item in items
if item["document"] == name
}
try:
scores = await judge.answer(
(FOLDER / name).read_text(encoding="utf-8"), asked
)
except JudgeUnavailable:
continue
for key, value in scores.items():
if key in answers:
answers[key].append(float(value))
return answers
def report(
items: Sequence[Mapping[str, Any]], by_judge: Mapping[str, dict[str, list[float]]]
) -> dict[str, Any]:
labelled = {i["id"]: float(i["label"]) for i in items if i["label"] is not None}
result: dict[str, Any] = {
"questions": len(items),
"labelled": len(labelled),
"judges": {},
}
for name, answers in by_judge.items():
first = {key: values[0] for key, values in answers.items() if values}
result["judges"][name] = {
"answered": len(first),
# Same answer on every run, per question.
"stability": share(
[
(values[0], other)
for values in answers.values()
for other in values[1:]
]
),
"agreement_with_labels": share(
[(first[key], label) for key, label in labelled.items() if key in first]
),
"mean_answer": round(sum(first.values()) / len(first), 2)
if first
else None,
}
names = list(by_judge)
if len(names) == 2:
a, b = (by_judge[name] for name in names)
result["between_judges"] = share(
[(a[key][0], b[key][0]) for key in a if a[key] and b.get(key)]
)
return result
async def main(arguments: Sequence[str]) -> int:
if "--build" in arguments:
items = build()
LABELS.write_text(json.dumps(items, indent=2, ensure_ascii=False) + "\n")
print(f"Wrote {len(items)} questions to {LABELS}; fill in each `label`.")
return 0
runs = int(arguments[arguments.index("--runs") + 1]) if "--runs" in arguments else 3
items = json.loads(LABELS.read_text(encoding="utf-8"))
settings = get_settings()
model = load_provider(settings.CLOUD_PROVIDER).models.default
by_judge: dict[str, dict[str, list[float]]] = {
"jev": {item["id"]: [] for item in items},
"gemini": {item["id"]: [] for item in items},
}
for _ in range(runs):
# A new Jev client for each run: the client caches identical requests, and a
# cached answer would make stability look perfect whatever the service does.
decider = open_decider(settings)
judges: dict[str, Judge] = {
"jev": DeciderJudge(decider, scope="judge-calibration"),
"gemini": GeminiJudge(model.removeprefix(GEMINI_PREFIX)),
}
try:
for name, judge in judges.items():
for key, values in (await ask(judge, items, 1)).items():
by_judge[name][key] += values
finally:
aclose = getattr(decider, "aclose", None)
if aclose:
await aclose()
result = {"runs": runs, **report(items, by_judge)}
(FOLDER / "calibration.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))