feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
111
app/learning/proof.py
Normal file
111
app/learning/proof.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""Proving lessons: the evaluation with them against the evaluation without (ADR-0004).
|
||||
|
||||
A lesson is kept only if quality holds and Revisions, time or tokens fall. Both runs
|
||||
are ordinary evaluation reports; one is made with LESSONS_ENABLED=false.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.learning.proposals import Proposal, write
|
||||
|
||||
METRICS = ("score", "revisions", "seconds", "tokens")
|
||||
_CAVEAT = (
|
||||
"This compares every published lesson together, on one run each way. Runs of "
|
||||
"the same case vary, so one pair suggests a direction and does not settle it."
|
||||
)
|
||||
|
||||
|
||||
def _measures(report: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]:
|
||||
cases = {}
|
||||
for item in report:
|
||||
details = item.get("details") or {}
|
||||
tokens = ((details.get("tokens") or {}).get("total") or {}).get("total") or 0
|
||||
cases[str(item.get("case_id"))] = {
|
||||
"score": float(item.get("score") or 0),
|
||||
"revisions": float(sum((details.get("revisions") or {}).values())),
|
||||
"seconds": float(details.get("seconds") or 0),
|
||||
"tokens": float(tokens),
|
||||
}
|
||||
return cases
|
||||
|
||||
|
||||
def compare(without: Path, with_lessons: Path) -> dict[str, Any]:
|
||||
"""The two reports side by side, for the cases both ran, and whether the
|
||||
lessons helped: no case scored lower, and Revisions, time or tokens fell."""
|
||||
before = _measures(json.loads(without.read_text(encoding="utf-8")))
|
||||
after = _measures(json.loads(with_lessons.read_text(encoding="utf-8")))
|
||||
cases = sorted(set(before) & set(after))
|
||||
totals = {
|
||||
metric: (
|
||||
sum(before[case][metric] for case in cases),
|
||||
sum(after[case][metric] for case in cases),
|
||||
)
|
||||
for metric in METRICS
|
||||
}
|
||||
quality_holds = bool(cases) and all(
|
||||
after[case]["score"] >= before[case]["score"] for case in cases
|
||||
)
|
||||
lower = [
|
||||
metric
|
||||
for metric in ("revisions", "seconds", "tokens")
|
||||
if totals[metric][1] < totals[metric][0]
|
||||
]
|
||||
return {
|
||||
"cases": {
|
||||
case: {"without": before[case], "with": after[case]} for case in cases
|
||||
},
|
||||
"totals": {
|
||||
metric: {"without": a, "with": b} for metric, (a, b) in totals.items()
|
||||
},
|
||||
"quality_holds": quality_holds,
|
||||
"lower": lower,
|
||||
"helps": quality_holds and bool(lower),
|
||||
}
|
||||
|
||||
|
||||
def render(result: Mapping[str, Any], without: Path, with_lessons: Path) -> str:
|
||||
"""The comparison as the text of a proposal's Proof section."""
|
||||
header = (
|
||||
f"Evaluation without lessons: `{without.parent.name}`; with: "
|
||||
f"`{with_lessons.parent.name}`."
|
||||
)
|
||||
lines = [
|
||||
header,
|
||||
"",
|
||||
"| Case | Score | Revisions | Minutes | Tokens (M) |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for case, pair in result["cases"].items():
|
||||
a, b = pair["without"], pair["with"]
|
||||
lines.append(
|
||||
f"| {case} | {a['score']:.2f} → {b['score']:.2f} | "
|
||||
f"{a['revisions']:.0f} → {b['revisions']:.0f} | "
|
||||
f"{a['seconds'] / 60:.0f} → {b['seconds'] / 60:.0f} | "
|
||||
f"{a['tokens'] / 1e6:.2f} → {b['tokens'] / 1e6:.2f} |"
|
||||
)
|
||||
if not result["cases"]:
|
||||
verdict = "The two runs share no case, so nothing is shown."
|
||||
elif result["helps"]:
|
||||
verdict = f"Quality held and {', '.join(result['lower'])} fell: kept."
|
||||
elif not result["quality_holds"]:
|
||||
verdict = "A case scored lower with the lessons: not shown to help."
|
||||
else:
|
||||
verdict = "Quality held, and nothing fell: not shown to help."
|
||||
lines += [
|
||||
"",
|
||||
verdict,
|
||||
"",
|
||||
_CAVEAT,
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def record(proposals: Sequence[Proposal], proof: str) -> list[Proposal]:
|
||||
"""Write the comparison into every published proposal."""
|
||||
return [write(replace(p, proof=proof)) for p in proposals if p.published]
|
||||
Reference in New Issue
Block a user