"""Proving lessons: the evaluation with them against the evaluation without (ADR-0004). A lesson is kept only if quality holds and Revisions, time or tokens fall. Both runs are ordinary evaluation reports; one is made with LESSONS_ENABLED=false. """ from __future__ import annotations import json from collections.abc import Collection, Mapping, Sequence from dataclasses import replace from pathlib import Path from typing import Any from execution_engine.learning.proposals import Proposal, write METRICS = ("score", "revisions", "seconds", "tokens") _CAVEAT = ( "This compares every published lesson together, on one run each way. Runs of " "the same case vary, so one pair suggests a direction and does not settle it." ) def _measures(report: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]: cases = {} for item in report: details = item.get("details") or {} tokens = ((details.get("tokens") or {}).get("total") or {}).get("total") or 0 cases[str(item.get("case_id"))] = { "score": float(item.get("score") or 0), "revisions": float(sum((details.get("revisions") or {}).values())), "seconds": float(details.get("seconds") or 0), "tokens": float(tokens), } return cases def compare(without: Path, with_lessons: Path) -> dict[str, Any]: """The two reports side by side, for the cases both ran, and whether the lessons helped: no case scored lower, and Revisions, time or tokens fell.""" before = _measures(json.loads(without.read_text(encoding="utf-8"))) after = _measures(json.loads(with_lessons.read_text(encoding="utf-8"))) cases = sorted(set(before) & set(after)) totals = { metric: ( sum(before[case][metric] for case in cases), sum(after[case][metric] for case in cases), ) for metric in METRICS } quality_holds = bool(cases) and all( after[case]["score"] >= before[case]["score"] for case in cases ) lower = [ metric for metric in ("revisions", "seconds", "tokens") if totals[metric][1] < totals[metric][0] ] return { "cases": { case: {"without": before[case], "with": after[case]} for case in cases }, "totals": { metric: {"without": a, "with": b} for metric, (a, b) in totals.items() }, "quality_holds": quality_holds, "lower": lower, "helps": quality_holds and bool(lower), } def render(result: Mapping[str, Any], without: Path, with_lessons: Path) -> str: """The comparison as the text of a proposal's Proof section.""" header = ( f"Evaluation without lessons: `{without.parent.name}`; with: " f"`{with_lessons.parent.name}`." ) lines = [ header, "", "| Case | Score | Revisions | Minutes | Tokens (M) |", "| --- | --- | --- | --- | --- |", ] for case, pair in result["cases"].items(): a, b = pair["without"], pair["with"] lines.append( f"| {case} | {a['score']:.2f} → {b['score']:.2f} | " f"{a['revisions']:.0f} → {b['revisions']:.0f} | " f"{a['seconds'] / 60:.0f} → {b['seconds'] / 60:.0f} | " f"{a['tokens'] / 1e6:.2f} → {b['tokens'] / 1e6:.2f} |" ) if not result["cases"]: verdict = "The two runs share no case, so nothing is shown." elif result["helps"]: verdict = f"Quality held and {', '.join(result['lower'])} fell: kept." elif not result["quality_holds"]: verdict = "A case scored lower with the lessons: not shown to help." else: verdict = "Quality held, and nothing fell: not shown to help." lines += [ "", verdict, "", _CAVEAT, ] return "\n".join(lines) def record( proposals: Sequence[Proposal], proof: str, live: Collection[str] ) -> list[Proposal]: """Write the comparison into the proposal of every lesson in the Skill. `live` is the ids the Skill holds now. A retired lesson was not part of what was compared, so its proposal is left as it was. """ return [write(replace(p, proof=proof)) for p in proposals if p.id in live]