Capture, distil, review, publish, retire (ADR-0004). Lesson candidates of past Executions are grouped into what recurred; a lesson-curator role, outside any Execution, proposes a lesson per group worth one; each proposal is an issue file a person approves or rejects; an approved lesson is written into the provider's lessons Skill, which a Specialist is given only once it holds lessons for that Specialist's Deliverable. LESSONS_ENABLED off runs without them, for an evaluation with and without. The curator proposed six lessons from 21 Executions. The owner approved all six on 2026-10-09 and they are published: five for the target architect, one for the IaC author. None is proved by an evaluation yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
156 lines
5.6 KiB
Python
156 lines
5.6 KiB
Python
"""The lessons loop, from the command line (ADR-0004).
|
|
|
|
propose group the lesson candidates of past Executions and have the curator
|
|
propose a lesson for each recurring problem worth one. Proposals are
|
|
written to .scratch/lessons/issues/ as `needs-triage`.
|
|
status list every proposal and where it stands.
|
|
publish write every approved proposal (Status: ready-for-agent) into the
|
|
provider's lessons Skill.
|
|
retire remove a lesson from the Skill, by id.
|
|
|
|
Usage:
|
|
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
|
|
.venv/bin/python -m eval.curate_lessons status
|
|
.venv/bin/python -m eval.curate_lessons publish
|
|
.venv/bin/python -m eval.curate_lessons retire L-03
|
|
|
|
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
|
|
`wontfix` to reject it. Nothing is published without that. After publishing, prove
|
|
the lesson: run the evaluation with LESSONS_ENABLED=false and again with it on, and
|
|
record the comparison under the proposal's `## Proof`. Retire a lesson that does
|
|
not hold quality and lower Revisions, time or tokens.
|
|
|
|
`propose` costs model usage. Lesson candidates are client data; they are read
|
|
where the Executions wrote them and never copied.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sys
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
|
|
from app.config import get_settings
|
|
from app.learning.curator import LESSON_CURATOR, curate
|
|
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
|
from app.learning.proposals import ISSUES, load
|
|
from app.learning.skill import NotApproved, publish, published, retire
|
|
from app.providers import ProviderBundle, load_provider
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
# Where Executions write their candidates: under each evaluation run, and under the
|
|
# server's output root.
|
|
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
|
|
|
|
|
|
def _skill_dir(bundle: ProviderBundle) -> Path:
|
|
if not bundle.lessons_skill:
|
|
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
|
|
return bundle.skills[bundle.lessons_skill].path
|
|
|
|
|
|
def _options(arguments: Sequence[str], name: str) -> list[str]:
|
|
return [
|
|
arguments[index + 1]
|
|
for index, argument in enumerate(arguments[:-1])
|
|
if argument == name
|
|
]
|
|
|
|
|
|
async def propose_lessons(arguments: Sequence[str]) -> int:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.adk.roles import load_roles
|
|
|
|
settings = get_settings()
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
|
|
minimum = int((_options(arguments, "--min") or ["2"])[0])
|
|
records = read_candidates(sources)
|
|
groups = group_candidates(records, min_executions=minimum)
|
|
print(
|
|
f"{len(records)} Executions, {len(groups)} recurring problems "
|
|
f"(in at least {minimum} Executions each)."
|
|
)
|
|
if not groups:
|
|
return 0
|
|
roles = load_roles(ROOT / "agents", bundle)
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
proposed = await curate(
|
|
runtime,
|
|
roles[LESSON_CURATOR],
|
|
groups,
|
|
executions=len(records),
|
|
identifiers=identifiers(records),
|
|
folder=ROOT / ISSUES,
|
|
)
|
|
for proposal in proposed:
|
|
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
print(
|
|
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
|
|
"Status to ready-for-agent or wontfix."
|
|
)
|
|
return 0
|
|
|
|
|
|
def show_status() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
live = published(_skill_dir(bundle))
|
|
proposals = load(ROOT / ISSUES)
|
|
if not proposals:
|
|
print("No lesson has been proposed.")
|
|
for proposal in proposals:
|
|
where = "in the Skill" if proposal.id in live else "not in the Skill"
|
|
proof = "proved" if proposal.proof else "not proved"
|
|
print(
|
|
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
|
|
f"[{proposal.deliverable}] {proposal.title}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def publish_approved() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
skill_dir = _skill_dir(bundle)
|
|
done = 0
|
|
for proposal in load(ROOT / ISSUES):
|
|
try:
|
|
publish(proposal, skill_dir)
|
|
except NotApproved:
|
|
continue
|
|
done += 1
|
|
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
print(f"{done} lessons are in {skill_dir.name}.")
|
|
return 0
|
|
|
|
|
|
def retire_lesson(arguments: Sequence[str]) -> int:
|
|
if not arguments:
|
|
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
|
|
return 2
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
gone = retire(arguments[0], _skill_dir(bundle))
|
|
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
|
|
return 0 if gone else 1
|
|
|
|
|
|
def main(arguments: Sequence[str]) -> int:
|
|
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
|
|
if command == "propose":
|
|
return asyncio.run(propose_lessons(rest))
|
|
if command == "status":
|
|
return show_status()
|
|
if command == "publish":
|
|
return publish_approved()
|
|
if command == "retire":
|
|
return retire_lesson(rest)
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|