118 lines
4.2 KiB
Python
118 lines
4.2 KiB
Python
"""Proving lessons: the evaluation with them against the evaluation without (ADR-0004).
|
|
|
|
A lesson is kept only if quality holds and Revisions, time or tokens fall. Both runs
|
|
are ordinary evaluation reports; one is made with LESSONS_ENABLED=false.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Collection, Mapping, Sequence
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from execution_engine.learning.proposals import Proposal, write
|
|
|
|
METRICS = ("score", "revisions", "seconds", "tokens")
|
|
_CAVEAT = (
|
|
"This compares every published lesson together, on one run each way. Runs of "
|
|
"the same case vary, so one pair suggests a direction and does not settle it."
|
|
)
|
|
|
|
|
|
def _measures(report: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]:
|
|
cases = {}
|
|
for item in report:
|
|
details = item.get("details") or {}
|
|
tokens = ((details.get("tokens") or {}).get("total") or {}).get("total") or 0
|
|
cases[str(item.get("case_id"))] = {
|
|
"score": float(item.get("score") or 0),
|
|
"revisions": float(sum((details.get("revisions") or {}).values())),
|
|
"seconds": float(details.get("seconds") or 0),
|
|
"tokens": float(tokens),
|
|
}
|
|
return cases
|
|
|
|
|
|
def compare(without: Path, with_lessons: Path) -> dict[str, Any]:
|
|
"""The two reports side by side, for the cases both ran, and whether the
|
|
lessons helped: no case scored lower, and Revisions, time or tokens fell."""
|
|
before = _measures(json.loads(without.read_text(encoding="utf-8")))
|
|
after = _measures(json.loads(with_lessons.read_text(encoding="utf-8")))
|
|
cases = sorted(set(before) & set(after))
|
|
totals = {
|
|
metric: (
|
|
sum(before[case][metric] for case in cases),
|
|
sum(after[case][metric] for case in cases),
|
|
)
|
|
for metric in METRICS
|
|
}
|
|
quality_holds = bool(cases) and all(
|
|
after[case]["score"] >= before[case]["score"] for case in cases
|
|
)
|
|
lower = [
|
|
metric
|
|
for metric in ("revisions", "seconds", "tokens")
|
|
if totals[metric][1] < totals[metric][0]
|
|
]
|
|
return {
|
|
"cases": {
|
|
case: {"without": before[case], "with": after[case]} for case in cases
|
|
},
|
|
"totals": {
|
|
metric: {"without": a, "with": b} for metric, (a, b) in totals.items()
|
|
},
|
|
"quality_holds": quality_holds,
|
|
"lower": lower,
|
|
"helps": quality_holds and bool(lower),
|
|
}
|
|
|
|
|
|
def render(result: Mapping[str, Any], without: Path, with_lessons: Path) -> str:
|
|
"""The comparison as the text of a proposal's Proof section."""
|
|
header = (
|
|
f"Evaluation without lessons: `{without.parent.name}`; with: "
|
|
f"`{with_lessons.parent.name}`."
|
|
)
|
|
lines = [
|
|
header,
|
|
"",
|
|
"| Case | Score | Revisions | Minutes | Tokens (M) |",
|
|
"| --- | --- | --- | --- | --- |",
|
|
]
|
|
for case, pair in result["cases"].items():
|
|
a, b = pair["without"], pair["with"]
|
|
lines.append(
|
|
f"| {case} | {a['score']:.2f} → {b['score']:.2f} | "
|
|
f"{a['revisions']:.0f} → {b['revisions']:.0f} | "
|
|
f"{a['seconds'] / 60:.0f} → {b['seconds'] / 60:.0f} | "
|
|
f"{a['tokens'] / 1e6:.2f} → {b['tokens'] / 1e6:.2f} |"
|
|
)
|
|
if not result["cases"]:
|
|
verdict = "The two runs share no case, so nothing is shown."
|
|
elif result["helps"]:
|
|
verdict = f"Quality held and {', '.join(result['lower'])} fell: kept."
|
|
elif not result["quality_holds"]:
|
|
verdict = "A case scored lower with the lessons: not shown to help."
|
|
else:
|
|
verdict = "Quality held, and nothing fell: not shown to help."
|
|
lines += [
|
|
"",
|
|
verdict,
|
|
"",
|
|
_CAVEAT,
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def record(
|
|
proposals: Sequence[Proposal], proof: str, live: Collection[str]
|
|
) -> list[Proposal]:
|
|
"""Write the comparison into the proposal of every lesson in the Skill.
|
|
|
|
`live` is the ids the Skill holds now. A retired lesson was not part of what
|
|
was compared, so its proposal is left as it was.
|
|
"""
|
|
return [write(replace(p, proof=proof)) for p in proposals if p.id in live]
|