Files
gcp_solution_architecture_a…/eval/curate_lessons.py
Jonathan Boniface 71da0ed504
Some checks failed
validation / verify (push) Failing after 12s
chore: housekeeping from the runs of 2026-10-09
- An Execution's files are written as each Deliverable is submitted, marked
  in_progress, so a killed process or one waiting for its user leaves them on
  disk. A write removes files an earlier write left that it no longer has.
- A question about delivered work is answered from that Execution's solution
  guide by the reply writer; a question about the service is answered as
  before. Nothing is started for either.
- Lessons: prove writes only for lessons the Skill holds; the Skill records
  the vendored Skill versions the lessons were read against, status names any
  that changed, and a test fails until they are reviewed.
- Four more code patterns: Firestore, BigQuery, Eventarc and GKE.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 11:44:38 +01:00

238 lines
8.7 KiB
Python

"""The lessons loop, from the command line (ADR-0004).
propose group the lesson candidates of past Executions and have the curator
propose a lesson for each recurring problem worth one. Proposals are
written to .scratch/lessons/issues/ as `needs-triage`.
status list every proposal and where it stands.
publish write every approved proposal (Status: ready-for-agent) into the
provider's lessons Skill.
prove compare an evaluation run without lessons with one run with them,
and write the comparison into every published proposal.
retire remove a lesson from the Skill, by id.
reviewed record that the lessons were read against the vendored Skills as
they are now. `status` says when those have changed since.
Usage:
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
.venv/bin/python -m eval.curate_lessons status
.venv/bin/python -m eval.curate_lessons publish
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
.venv/bin/python -m eval.curate_lessons retire L-03
.venv/bin/python -m eval.curate_lessons reviewed
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
`wontfix` to reject it. Nothing is published without that. After publishing, prove
the lessons: run the evaluation with LESSONS_ENABLED=false and again with it on, on
the same cases, then run `prove` with the two reports. Retire a lesson that does
not hold quality and lower Revisions, time or tokens.
`propose` costs model usage. Lesson candidates are client data; they are read
where the Executions wrote them and never copied.
"""
from __future__ import annotations
import asyncio
import sys
from collections.abc import Sequence
from pathlib import Path
from app.agent import open_decider
from app.config import get_settings
from app.learning.curator import LESSON_CURATOR, curate
from app.learning.evidence import group_candidates, identifiers, read_candidates
from app.learning.proof import compare, record, render
from app.learning.proposals import ISSUES, load
from app.learning.skill import (
NotApproved,
mark_reviewed,
publish,
published,
retire,
upstream_changes,
upstream_reviewed,
)
from app.providers import ProviderBundle, load_provider
from app.skills.agent_skills import skill_version
ROOT = Path(__file__).resolve().parent.parent
# Where Executions write their candidates: under each evaluation run, and under the
# server's output root.
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
def _skill_dir(bundle: ProviderBundle) -> Path:
if not bundle.lessons_skill:
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
return bundle.skills[bundle.lessons_skill].path
def upstream_versions(bundle: ProviderBundle) -> dict[str, str]:
"""The vendored Skills, each with the version it has now. They are what the
lessons correct the use of, and they change when upstream is synced."""
return {
name: skill_version(meta.path)
for name, meta in bundle.skills.items()
if "vendor" in meta.path.parts
}
def _options(arguments: Sequence[str], name: str) -> list[str]:
return [
arguments[index + 1]
for index, argument in enumerate(arguments[:-1])
if argument == name
]
async def propose_lessons(arguments: Sequence[str]) -> int:
from app.adk.adk_runtime import AdkAgentRuntime
from app.adk.roles import load_roles
settings = get_settings()
bundle = load_provider(settings.CLOUD_PROVIDER)
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
minimum = int((_options(arguments, "--min") or ["2"])[0])
records = read_candidates(sources)
groups = group_candidates(records, min_executions=minimum)
print(
f"{len(records)} Executions, {len(groups)} recurring problems "
f"(in at least {minimum} Executions each)."
)
if not groups:
return 0
roles = load_roles(ROOT / "agents", bundle)
runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
decider = open_decider(settings)
try:
proposed = await curate(
runtime,
roles[LESSON_CURATOR],
groups,
executions=len(records),
identifiers=identifiers(records),
folder=ROOT / ISSUES,
decider=decider,
)
finally:
aclose = getattr(decider, "aclose", None)
if aclose:
await aclose()
for proposal in proposed:
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
print(
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
"Status to ready-for-agent or wontfix."
)
return 0
def show_status() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
live = published(_skill_dir(bundle))
proposals = load(ROOT / ISSUES)
if not proposals:
print("No lesson has been proposed.")
for proposal in proposals:
where = (
"in the Skill"
if proposal.id in live
else "retired"
if proposal.published
else "not in the Skill"
)
# One comparison suggests a direction; it does not prove a lesson.
proof = "compared" if proposal.proof else "not compared"
print(
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:12s} "
f"[{proposal.deliverable}] {proposal.title}"
)
changed = upstream_changes(_skill_dir(bundle), upstream_versions(bundle))
if changed:
print(
f"\nVendored Skills changed since the lessons were last reviewed: "
f"{', '.join(changed)}.\nRead each lesson in the Skill against them: "
"retire one the Skill now says itself or contradicts. Then run "
"`reviewed`."
)
return 0
def mark_lessons_reviewed() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
mark_reviewed(_skill_dir(bundle), upstream_versions(bundle))
print("Recorded: the lessons stand against the vendored Skills as they are now.")
return 0
def publish_approved() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
skill_dir = _skill_dir(bundle)
done = 0
for proposal in load(ROOT / ISSUES):
try:
publish(proposal, skill_dir)
except NotApproved:
continue
done += 1
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
if done and not upstream_reviewed(skill_dir):
# The first lessons: what they were written beside is recorded once. After
# that only a person's review moves it on.
mark_reviewed(skill_dir, upstream_versions(bundle))
print(f"{done} lessons are in {skill_dir.name}.")
return 0
def prove_lessons(arguments: Sequence[str]) -> int:
without, with_lessons = (
_options(arguments, "--without"),
_options(arguments, "--with"),
)
if not without or not with_lessons:
print("Give both reports: --without <report.json> --with <report.json>.")
return 2
first, second = Path(without[0]), Path(with_lessons[0])
result = compare(first, second)
proof = render(result, first, second)
bundle = load_provider(get_settings().CLOUD_PROVIDER)
recorded = record(load(ROOT / ISSUES), proof, published(_skill_dir(bundle)))
print(proof)
print(f"\nRecorded in {len(recorded)} published proposals.")
return 0 if result["helps"] else 1
def retire_lesson(arguments: Sequence[str]) -> int:
if not arguments:
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
return 2
bundle = load_provider(get_settings().CLOUD_PROVIDER)
gone = retire(arguments[0], _skill_dir(bundle))
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
return 0 if gone else 1
def main(arguments: Sequence[str]) -> int:
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
if command == "propose":
return asyncio.run(propose_lessons(rest))
if command == "status":
return show_status()
if command == "publish":
return publish_approved()
if command == "prove":
return prove_lessons(rest)
if command == "retire":
return retire_lesson(rest)
if command == "reviewed":
return mark_lessons_reviewed()
print(__doc__, file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))