Files
orchestration_engine/execution_engine/learning/proof.py

118 lines
4.2 KiB
Python

"""Proving lessons: the evaluation with them against the evaluation without (ADR-0004).
A lesson is kept only if quality holds and Revisions, time or tokens fall. Both runs
are ordinary evaluation reports; one is made with LESSONS_ENABLED=false.
"""
from __future__ import annotations
import json
from collections.abc import Collection, Mapping, Sequence
from dataclasses import replace
from pathlib import Path
from typing import Any
from execution_engine.learning.proposals import Proposal, write
METRICS = ("score", "revisions", "seconds", "tokens")
_CAVEAT = (
"This compares every published lesson together, on one run each way. Runs of "
"the same case vary, so one pair suggests a direction and does not settle it."
)
def _measures(report: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]:
cases = {}
for item in report:
details = item.get("details") or {}
tokens = ((details.get("tokens") or {}).get("total") or {}).get("total") or 0
cases[str(item.get("case_id"))] = {
"score": float(item.get("score") or 0),
"revisions": float(sum((details.get("revisions") or {}).values())),
"seconds": float(details.get("seconds") or 0),
"tokens": float(tokens),
}
return cases
def compare(without: Path, with_lessons: Path) -> dict[str, Any]:
"""The two reports side by side, for the cases both ran, and whether the
lessons helped: no case scored lower, and Revisions, time or tokens fell."""
before = _measures(json.loads(without.read_text(encoding="utf-8")))
after = _measures(json.loads(with_lessons.read_text(encoding="utf-8")))
cases = sorted(set(before) & set(after))
totals = {
metric: (
sum(before[case][metric] for case in cases),
sum(after[case][metric] for case in cases),
)
for metric in METRICS
}
quality_holds = bool(cases) and all(
after[case]["score"] >= before[case]["score"] for case in cases
)
lower = [
metric
for metric in ("revisions", "seconds", "tokens")
if totals[metric][1] < totals[metric][0]
]
return {
"cases": {
case: {"without": before[case], "with": after[case]} for case in cases
},
"totals": {
metric: {"without": a, "with": b} for metric, (a, b) in totals.items()
},
"quality_holds": quality_holds,
"lower": lower,
"helps": quality_holds and bool(lower),
}
def render(result: Mapping[str, Any], without: Path, with_lessons: Path) -> str:
"""The comparison as the text of a proposal's Proof section."""
header = (
f"Evaluation without lessons: `{without.parent.name}`; with: "
f"`{with_lessons.parent.name}`."
)
lines = [
header,
"",
"| Case | Score | Revisions | Minutes | Tokens (M) |",
"| --- | --- | --- | --- | --- |",
]
for case, pair in result["cases"].items():
a, b = pair["without"], pair["with"]
lines.append(
f"| {case} | {a['score']:.2f} → {b['score']:.2f} | "
f"{a['revisions']:.0f} → {b['revisions']:.0f} | "
f"{a['seconds'] / 60:.0f} → {b['seconds'] / 60:.0f} | "
f"{a['tokens'] / 1e6:.2f} → {b['tokens'] / 1e6:.2f} |"
)
if not result["cases"]:
verdict = "The two runs share no case, so nothing is shown."
elif result["helps"]:
verdict = f"Quality held and {', '.join(result['lower'])} fell: kept."
elif not result["quality_holds"]:
verdict = "A case scored lower with the lessons: not shown to help."
else:
verdict = "Quality held, and nothing fell: not shown to help."
lines += [
"",
verdict,
"",
_CAVEAT,
]
return "\n".join(lines)
def record(
proposals: Sequence[Proposal], proof: str, live: Collection[str]
) -> list[Proposal]:
"""Write the comparison into the proposal of every lesson in the Skill.
`live` is the ids the Skill holds now. A retired lesson was not part of what
was compared, so its proposal is left as it was.
"""
return [write(replace(p, proof=proof)) for p in proposals if p.id in live]