Files
gcp_solution_architecture_a…/eval/curate_lessons.py
Jonathan Boniface a7bc5310f3 feat: the lessons loop, and six lessons approved and published
Capture, distil, review, publish, retire (ADR-0004). Lesson candidates of
past Executions are grouped into what recurred; a lesson-curator role,
outside any Execution, proposes a lesson per group worth one; each proposal
is an issue file a person approves or rejects; an approved lesson is
written into the provider's lessons Skill, which a Specialist is given only
once it holds lessons for that Specialist's Deliverable. LESSONS_ENABLED
off runs without them, for an evaluation with and without.

The curator proposed six lessons from 21 Executions. The owner approved all
six on 2026-10-09 and they are published: five for the target architect,
one for the IaC author. None is proved by an evaluation yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 15:19:32 +01:00

156 lines
5.6 KiB
Python

"""The lessons loop, from the command line (ADR-0004).
propose group the lesson candidates of past Executions and have the curator
propose a lesson for each recurring problem worth one. Proposals are
written to .scratch/lessons/issues/ as `needs-triage`.
status list every proposal and where it stands.
publish write every approved proposal (Status: ready-for-agent) into the
provider's lessons Skill.
retire remove a lesson from the Skill, by id.
Usage:
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
.venv/bin/python -m eval.curate_lessons status
.venv/bin/python -m eval.curate_lessons publish
.venv/bin/python -m eval.curate_lessons retire L-03
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
`wontfix` to reject it. Nothing is published without that. After publishing, prove
the lesson: run the evaluation with LESSONS_ENABLED=false and again with it on, and
record the comparison under the proposal's `## Proof`. Retire a lesson that does
not hold quality and lower Revisions, time or tokens.
`propose` costs model usage. Lesson candidates are client data; they are read
where the Executions wrote them and never copied.
"""
from __future__ import annotations
import asyncio
import sys
from collections.abc import Sequence
from pathlib import Path
from app.config import get_settings
from app.learning.curator import LESSON_CURATOR, curate
from app.learning.evidence import group_candidates, identifiers, read_candidates
from app.learning.proposals import ISSUES, load
from app.learning.skill import NotApproved, publish, published, retire
from app.providers import ProviderBundle, load_provider
ROOT = Path(__file__).resolve().parent.parent
# Where Executions write their candidates: under each evaluation run, and under the
# server's output root.
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
def _skill_dir(bundle: ProviderBundle) -> Path:
if not bundle.lessons_skill:
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
return bundle.skills[bundle.lessons_skill].path
def _options(arguments: Sequence[str], name: str) -> list[str]:
return [
arguments[index + 1]
for index, argument in enumerate(arguments[:-1])
if argument == name
]
async def propose_lessons(arguments: Sequence[str]) -> int:
from app.adk.adk_runtime import AdkAgentRuntime
from app.adk.roles import load_roles
settings = get_settings()
bundle = load_provider(settings.CLOUD_PROVIDER)
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
minimum = int((_options(arguments, "--min") or ["2"])[0])
records = read_candidates(sources)
groups = group_candidates(records, min_executions=minimum)
print(
f"{len(records)} Executions, {len(groups)} recurring problems "
f"(in at least {minimum} Executions each)."
)
if not groups:
return 0
roles = load_roles(ROOT / "agents", bundle)
runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
proposed = await curate(
runtime,
roles[LESSON_CURATOR],
groups,
executions=len(records),
identifiers=identifiers(records),
folder=ROOT / ISSUES,
)
for proposal in proposed:
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
print(
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
"Status to ready-for-agent or wontfix."
)
return 0
def show_status() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
live = published(_skill_dir(bundle))
proposals = load(ROOT / ISSUES)
if not proposals:
print("No lesson has been proposed.")
for proposal in proposals:
where = "in the Skill" if proposal.id in live else "not in the Skill"
proof = "proved" if proposal.proof else "not proved"
print(
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
f"[{proposal.deliverable}] {proposal.title}"
)
return 0
def publish_approved() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
skill_dir = _skill_dir(bundle)
done = 0
for proposal in load(ROOT / ISSUES):
try:
publish(proposal, skill_dir)
except NotApproved:
continue
done += 1
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
print(f"{done} lessons are in {skill_dir.name}.")
return 0
def retire_lesson(arguments: Sequence[str]) -> int:
if not arguments:
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
return 2
bundle = load_provider(get_settings().CLOUD_PROVIDER)
gone = retire(arguments[0], _skill_dir(bundle))
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
return 0 if gone else 1
def main(arguments: Sequence[str]) -> int:
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
if command == "propose":
return asyncio.run(propose_lessons(rest))
if command == "status":
return show_status()
if command == "publish":
return publish_approved()
if command == "retire":
return retire_lesson(rest)
print(__doc__, file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))