feat: the lessons loop, and six lessons approved and published
Capture, distil, review, publish, retire (ADR-0004). Lesson candidates of past Executions are grouped into what recurred; a lesson-curator role, outside any Execution, proposes a lesson per group worth one; each proposal is an issue file a person approves or rejects; an approved lesson is written into the provider's lessons Skill, which a Specialist is given only once it holds lessons for that Specialist's Deliverable. LESSONS_ENABLED off runs without them, for an evaluation with and without. The curator proposed six lessons from 21 Executions. The owner approved all six on 2026-10-09 and they are published: five for the target architect, one for the IaC author. None is proved by an evaluation yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
155
eval/curate_lessons.py
Normal file
155
eval/curate_lessons.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""The lessons loop, from the command line (ADR-0004).
|
||||
|
||||
propose group the lesson candidates of past Executions and have the curator
|
||||
propose a lesson for each recurring problem worth one. Proposals are
|
||||
written to .scratch/lessons/issues/ as `needs-triage`.
|
||||
status list every proposal and where it stands.
|
||||
publish write every approved proposal (Status: ready-for-agent) into the
|
||||
provider's lessons Skill.
|
||||
retire remove a lesson from the Skill, by id.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
|
||||
.venv/bin/python -m eval.curate_lessons status
|
||||
.venv/bin/python -m eval.curate_lessons publish
|
||||
.venv/bin/python -m eval.curate_lessons retire L-03
|
||||
|
||||
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
|
||||
`wontfix` to reject it. Nothing is published without that. After publishing, prove
|
||||
the lesson: run the evaluation with LESSONS_ENABLED=false and again with it on, and
|
||||
record the comparison under the proposal's `## Proof`. Retire a lesson that does
|
||||
not hold quality and lower Revisions, time or tokens.
|
||||
|
||||
`propose` costs model usage. Lesson candidates are client data; they are read
|
||||
where the Executions wrote them and never copied.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import get_settings
|
||||
from app.learning.curator import LESSON_CURATOR, curate
|
||||
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
||||
from app.learning.proposals import ISSUES, load
|
||||
from app.learning.skill import NotApproved, publish, published, retire
|
||||
from app.providers import ProviderBundle, load_provider
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
# Where Executions write their candidates: under each evaluation run, and under the
|
||||
# server's output root.
|
||||
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
|
||||
|
||||
|
||||
def _skill_dir(bundle: ProviderBundle) -> Path:
|
||||
if not bundle.lessons_skill:
|
||||
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
|
||||
return bundle.skills[bundle.lessons_skill].path
|
||||
|
||||
|
||||
def _options(arguments: Sequence[str], name: str) -> list[str]:
|
||||
return [
|
||||
arguments[index + 1]
|
||||
for index, argument in enumerate(arguments[:-1])
|
||||
if argument == name
|
||||
]
|
||||
|
||||
|
||||
async def propose_lessons(arguments: Sequence[str]) -> int:
|
||||
from app.adk.adk_runtime import AdkAgentRuntime
|
||||
from app.adk.roles import load_roles
|
||||
|
||||
settings = get_settings()
|
||||
bundle = load_provider(settings.CLOUD_PROVIDER)
|
||||
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
|
||||
minimum = int((_options(arguments, "--min") or ["2"])[0])
|
||||
records = read_candidates(sources)
|
||||
groups = group_candidates(records, min_executions=minimum)
|
||||
print(
|
||||
f"{len(records)} Executions, {len(groups)} recurring problems "
|
||||
f"(in at least {minimum} Executions each)."
|
||||
)
|
||||
if not groups:
|
||||
return 0
|
||||
roles = load_roles(ROOT / "agents", bundle)
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
||||
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
||||
)
|
||||
proposed = await curate(
|
||||
runtime,
|
||||
roles[LESSON_CURATOR],
|
||||
groups,
|
||||
executions=len(records),
|
||||
identifiers=identifiers(records),
|
||||
folder=ROOT / ISSUES,
|
||||
)
|
||||
for proposal in proposed:
|
||||
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
||||
print(
|
||||
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
|
||||
"Status to ready-for-agent or wontfix."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def show_status() -> int:
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
live = published(_skill_dir(bundle))
|
||||
proposals = load(ROOT / ISSUES)
|
||||
if not proposals:
|
||||
print("No lesson has been proposed.")
|
||||
for proposal in proposals:
|
||||
where = "in the Skill" if proposal.id in live else "not in the Skill"
|
||||
proof = "proved" if proposal.proof else "not proved"
|
||||
print(
|
||||
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
|
||||
f"[{proposal.deliverable}] {proposal.title}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def publish_approved() -> int:
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
skill_dir = _skill_dir(bundle)
|
||||
done = 0
|
||||
for proposal in load(ROOT / ISSUES):
|
||||
try:
|
||||
publish(proposal, skill_dir)
|
||||
except NotApproved:
|
||||
continue
|
||||
done += 1
|
||||
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
||||
print(f"{done} lessons are in {skill_dir.name}.")
|
||||
return 0
|
||||
|
||||
|
||||
def retire_lesson(arguments: Sequence[str]) -> int:
|
||||
if not arguments:
|
||||
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
|
||||
return 2
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
gone = retire(arguments[0], _skill_dir(bundle))
|
||||
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
|
||||
return 0 if gone else 1
|
||||
|
||||
|
||||
def main(arguments: Sequence[str]) -> int:
|
||||
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
|
||||
if command == "propose":
|
||||
return asyncio.run(propose_lessons(rest))
|
||||
if command == "status":
|
||||
return show_status()
|
||||
if command == "publish":
|
||||
return publish_approved()
|
||||
if command == "retire":
|
||||
return retire_lesson(rest)
|
||||
print(__doc__, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user