"""The lessons loop, from the command line (ADR-0004). propose group the lesson candidates of past Executions and have the curator propose a lesson for each recurring problem worth one. Proposals are written to .scratch/lessons/issues/ as `needs-triage`. status list every proposal and where it stands. publish write every approved proposal (Status: ready-for-agent) into the provider's lessons Skill. prove compare an evaluation run without lessons with one run with them, and write the comparison into every published proposal. retire remove a lesson from the Skill, by id. reviewed record that the lessons were read against the vendored Skills as they are now. `status` says when those have changed since. Usage: .venv/bin/python -m eval.curate_lessons propose [--from ...] [--min 2] .venv/bin/python -m eval.curate_lessons status .venv/bin/python -m eval.curate_lessons publish .venv/bin/python -m eval.curate_lessons prove --without --with .venv/bin/python -m eval.curate_lessons retire L-03 .venv/bin/python -m eval.curate_lessons reviewed A person approves a proposal by editing its file: set `Status: ready-for-agent`, or `wontfix` to reject it. Nothing is published without that. After publishing, prove the lessons: run the evaluation with LESSONS_ENABLED=false and again with it on, on the same cases, then run `prove` with the two reports. Retire a lesson that does not hold quality and lower Revisions, time or tokens. `propose` costs model usage. Lesson candidates are client data; they are read where the Executions wrote them and never copied. """ from __future__ import annotations import asyncio import sys from collections.abc import Sequence from pathlib import Path from app.agent import open_decider from app.config import get_settings from app.learning.curator import LESSON_CURATOR, curate from app.learning.evidence import group_candidates, identifiers, read_candidates from app.learning.proof import compare, record, render from app.learning.proposals import ISSUES, load from app.learning.skill import ( NotApproved, mark_reviewed, publish, published, retire, upstream_changes, upstream_reviewed, ) from app.providers import ProviderBundle, load_provider from app.skills.agent_skills import skill_version ROOT = Path(__file__).resolve().parent.parent # Where Executions write their candidates: under each evaluation run, and under the # server's output root. DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..") def _skill_dir(bundle: ProviderBundle) -> Path: if not bundle.lessons_skill: raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.") return bundle.skills[bundle.lessons_skill].path def upstream_versions(bundle: ProviderBundle) -> dict[str, str]: """The vendored Skills, each with the version it has now. They are what the lessons correct the use of, and they change when upstream is synced.""" return { name: skill_version(meta.path) for name, meta in bundle.skills.items() if "vendor" in meta.path.parts } def _options(arguments: Sequence[str], name: str) -> list[str]: return [ arguments[index + 1] for index, argument in enumerate(arguments[:-1]) if argument == name ] async def propose_lessons(arguments: Sequence[str]) -> int: from app.adk.adk_runtime import AdkAgentRuntime from app.adk.roles import load_roles settings = get_settings() bundle = load_provider(settings.CLOUD_PROVIDER) sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES) minimum = int((_options(arguments, "--min") or ["2"])[0]) records = read_candidates(sources) groups = group_candidates(records, min_executions=minimum) print( f"{len(records)} Executions, {len(groups)} recurring problems " f"(in at least {minimum} Executions each)." ) if not groups: return 0 roles = load_roles(ROOT / "agents", bundle) runtime = AdkAgentRuntime( skill_dirs={name: meta.path for name, meta in bundle.skills.items()}, model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS, ) decider = open_decider(settings) try: proposed = await curate( runtime, roles[LESSON_CURATOR], groups, executions=len(records), identifiers=identifiers(records), folder=ROOT / ISSUES, decider=decider, ) finally: aclose = getattr(decider, "aclose", None) if aclose: await aclose() for proposal in proposed: print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}") print( f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its " "Status to ready-for-agent or wontfix." ) return 0 def show_status() -> int: bundle = load_provider(get_settings().CLOUD_PROVIDER) live = published(_skill_dir(bundle)) proposals = load(ROOT / ISSUES) if not proposals: print("No lesson has been proposed.") for proposal in proposals: where = ( "in the Skill" if proposal.id in live else "retired" if proposal.published else "not in the Skill" ) # One comparison suggests a direction; it does not prove a lesson. proof = "compared" if proposal.proof else "not compared" print( f"{proposal.id} {proposal.status:16s} {where:16s} {proof:12s} " f"[{proposal.deliverable}] {proposal.title}" ) changed = upstream_changes(_skill_dir(bundle), upstream_versions(bundle)) if changed: print( f"\nVendored Skills changed since the lessons were last reviewed: " f"{', '.join(changed)}.\nRead each lesson in the Skill against them: " "retire one the Skill now says itself or contradicts. Then run " "`reviewed`." ) return 0 def mark_lessons_reviewed() -> int: bundle = load_provider(get_settings().CLOUD_PROVIDER) mark_reviewed(_skill_dir(bundle), upstream_versions(bundle)) print("Recorded: the lessons stand against the vendored Skills as they are now.") return 0 def publish_approved() -> int: bundle = load_provider(get_settings().CLOUD_PROVIDER) skill_dir = _skill_dir(bundle) done = 0 for proposal in load(ROOT / ISSUES): try: publish(proposal, skill_dir) except NotApproved: continue done += 1 print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}") if done and not upstream_reviewed(skill_dir): # The first lessons: what they were written beside is recorded once. After # that only a person's review moves it on. mark_reviewed(skill_dir, upstream_versions(bundle)) print(f"{done} lessons are in {skill_dir.name}.") return 0 def prove_lessons(arguments: Sequence[str]) -> int: without, with_lessons = ( _options(arguments, "--without"), _options(arguments, "--with"), ) if not without or not with_lessons: print("Give both reports: --without --with .") return 2 first, second = Path(without[0]), Path(with_lessons[0]) result = compare(first, second) proof = render(result, first, second) bundle = load_provider(get_settings().CLOUD_PROVIDER) recorded = record(load(ROOT / ISSUES), proof, published(_skill_dir(bundle))) print(proof) print(f"\nRecorded in {len(recorded)} published proposals.") return 0 if result["helps"] else 1 def retire_lesson(arguments: Sequence[str]) -> int: if not arguments: print("Name the lesson to retire, such as L-03.", file=sys.stderr) return 2 bundle = load_provider(get_settings().CLOUD_PROVIDER) gone = retire(arguments[0], _skill_dir(bundle)) print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.") return 0 if gone else 1 def main(arguments: Sequence[str]) -> int: command, rest = (arguments[0], arguments[1:]) if arguments else ("status", []) if command == "propose": return asyncio.run(propose_lessons(rest)) if command == "status": return show_status() if command == "publish": return publish_approved() if command == "prove": return prove_lessons(rest) if command == "retire": return retire_lesson(rest) if command == "reviewed": return mark_lessons_reviewed() print(__doc__, file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))