Some checks failed
validation / verify (push) Failing after 10s
Evidence was grouped only by the kind of problem, and a group gets one lesson, so what kept recurring inside "validation proposed a Check" could not be seen. A proposed or unimplementable Check is now also counted under each provider setting or resource type it names; a subject seen in two or more Executions is its own group and reaches the curator past triage. Over 32 Executions that gave five proposals. The owner approved three, now published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09 backend service capabilities. L-10 is held for more evidence and L-11 rejected. Also: retiring a lesson marks its proposal, so publish no longer brings a retired lesson back. L-04 had returned to the Skill that way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
244 lines
9.0 KiB
Python
244 lines
9.0 KiB
Python
"""The lessons loop, from the command line (ADR-0004).
|
|
|
|
propose group the lesson candidates of past Executions and have the curator
|
|
propose a lesson for each recurring problem worth one. Proposals are
|
|
written to .scratch/lessons/issues/ as `needs-triage`.
|
|
status list every proposal and where it stands.
|
|
publish write every approved proposal (Status: ready-for-agent) into the
|
|
provider's lessons Skill.
|
|
prove compare an evaluation run without lessons with one run with them,
|
|
and write the comparison into every published proposal.
|
|
retire remove a lesson from the Skill, by id.
|
|
reviewed record that the lessons were read against the vendored Skills as
|
|
they are now. `status` says when those have changed since.
|
|
|
|
Usage:
|
|
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
|
|
.venv/bin/python -m eval.curate_lessons status
|
|
.venv/bin/python -m eval.curate_lessons publish
|
|
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
|
|
.venv/bin/python -m eval.curate_lessons retire L-03
|
|
.venv/bin/python -m eval.curate_lessons reviewed
|
|
|
|
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
|
|
`wontfix` to reject it. Nothing is published without that. After publishing, prove
|
|
the lessons: run the evaluation with LESSONS_ENABLED=false and again with it on, on
|
|
the same cases, then run `prove` with the two reports. Retire a lesson that does
|
|
not hold quality and lower Revisions, time or tokens.
|
|
|
|
`propose` costs model usage. Lesson candidates are client data; they are read
|
|
where the Executions wrote them and never copied.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sys
|
|
from collections.abc import Sequence
|
|
from dataclasses import replace
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from app.agent import open_decider
|
|
from app.config import get_settings
|
|
from app.learning.curator import LESSON_CURATOR, curate
|
|
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
|
from app.learning.proof import compare, record, render
|
|
from app.learning.proposals import ISSUES, load, write
|
|
from app.learning.skill import (
|
|
NotApproved,
|
|
mark_reviewed,
|
|
publish,
|
|
published,
|
|
retire,
|
|
upstream_changes,
|
|
upstream_reviewed,
|
|
)
|
|
from app.providers import ProviderBundle, load_provider
|
|
from app.skills.agent_skills import skill_version
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
# Where Executions write their candidates: under each evaluation run, and under the
|
|
# server's output root.
|
|
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
|
|
|
|
|
|
def _skill_dir(bundle: ProviderBundle) -> Path:
|
|
if not bundle.lessons_skill:
|
|
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
|
|
return bundle.skills[bundle.lessons_skill].path
|
|
|
|
|
|
def upstream_versions(bundle: ProviderBundle) -> dict[str, str]:
|
|
"""The vendored Skills, each with the version it has now. They are what the
|
|
lessons correct the use of, and they change when upstream is synced."""
|
|
return {
|
|
name: skill_version(meta.path)
|
|
for name, meta in bundle.skills.items()
|
|
if "vendor" in meta.path.parts
|
|
}
|
|
|
|
|
|
def _options(arguments: Sequence[str], name: str) -> list[str]:
|
|
return [
|
|
arguments[index + 1]
|
|
for index, argument in enumerate(arguments[:-1])
|
|
if argument == name
|
|
]
|
|
|
|
|
|
async def propose_lessons(arguments: Sequence[str]) -> int:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.adk.roles import load_roles
|
|
|
|
settings = get_settings()
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
|
|
minimum = int((_options(arguments, "--min") or ["2"])[0])
|
|
records = read_candidates(sources)
|
|
groups = group_candidates(records, min_executions=minimum)
|
|
print(
|
|
f"{len(records)} Executions, {len(groups)} recurring problems "
|
|
f"(in at least {minimum} Executions each)."
|
|
)
|
|
if not groups:
|
|
return 0
|
|
roles = load_roles(ROOT / "agents", bundle)
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
decider = open_decider(settings)
|
|
try:
|
|
proposed = await curate(
|
|
runtime,
|
|
roles[LESSON_CURATOR],
|
|
groups,
|
|
executions=len(records),
|
|
identifiers=identifiers(records),
|
|
folder=ROOT / ISSUES,
|
|
decider=decider,
|
|
)
|
|
finally:
|
|
aclose = getattr(decider, "aclose", None)
|
|
if aclose:
|
|
await aclose()
|
|
for proposal in proposed:
|
|
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
print(
|
|
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
|
|
"Status to ready-for-agent or wontfix."
|
|
)
|
|
return 0
|
|
|
|
|
|
def show_status() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
live = published(_skill_dir(bundle))
|
|
proposals = load(ROOT / ISSUES)
|
|
if not proposals:
|
|
print("No lesson has been proposed.")
|
|
for proposal in proposals:
|
|
where = (
|
|
"in the Skill"
|
|
if proposal.id in live
|
|
else "retired"
|
|
if proposal.retired
|
|
else "not in the Skill"
|
|
)
|
|
# One comparison suggests a direction; it does not prove a lesson.
|
|
proof = "compared" if proposal.proof else "not compared"
|
|
print(
|
|
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:12s} "
|
|
f"[{proposal.deliverable}] {proposal.title}"
|
|
)
|
|
changed = upstream_changes(_skill_dir(bundle), upstream_versions(bundle))
|
|
if changed:
|
|
print(
|
|
f"\nVendored Skills changed since the lessons were last reviewed: "
|
|
f"{', '.join(changed)}.\nRead each lesson in the Skill against them: "
|
|
"retire one the Skill now says itself or contradicts. Then run "
|
|
"`reviewed`."
|
|
)
|
|
return 0
|
|
|
|
|
|
def mark_lessons_reviewed() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
mark_reviewed(_skill_dir(bundle), upstream_versions(bundle))
|
|
print("Recorded: the lessons stand against the vendored Skills as they are now.")
|
|
return 0
|
|
|
|
|
|
def publish_approved() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
skill_dir = _skill_dir(bundle)
|
|
done = 0
|
|
for proposal in load(ROOT / ISSUES):
|
|
try:
|
|
publish(proposal, skill_dir)
|
|
except NotApproved:
|
|
continue
|
|
done += 1
|
|
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
if done and not upstream_reviewed(skill_dir):
|
|
# The first lessons: what they were written beside is recorded once. After
|
|
# that only a person's review moves it on.
|
|
mark_reviewed(skill_dir, upstream_versions(bundle))
|
|
print(f"{done} lessons are in {skill_dir.name}.")
|
|
return 0
|
|
|
|
|
|
def prove_lessons(arguments: Sequence[str]) -> int:
|
|
without, with_lessons = (
|
|
_options(arguments, "--without"),
|
|
_options(arguments, "--with"),
|
|
)
|
|
if not without or not with_lessons:
|
|
print("Give both reports: --without <report.json> --with <report.json>.")
|
|
return 2
|
|
first, second = Path(without[0]), Path(with_lessons[0])
|
|
result = compare(first, second)
|
|
proof = render(result, first, second)
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
recorded = record(load(ROOT / ISSUES), proof, published(_skill_dir(bundle)))
|
|
print(proof)
|
|
print(f"\nRecorded in {len(recorded)} published proposals.")
|
|
return 0 if result["helps"] else 1
|
|
|
|
|
|
def retire_lesson(arguments: Sequence[str]) -> int:
|
|
if not arguments:
|
|
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
|
|
return 2
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
gone = retire(arguments[0], _skill_dir(bundle))
|
|
for proposal in load(ROOT / ISSUES):
|
|
if proposal.id == arguments[0] and not proposal.retired:
|
|
# Marked on its proposal, so the next publish does not bring it back.
|
|
write(replace(proposal, retired=datetime.now(UTC).date().isoformat()))
|
|
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
|
|
return 0 if gone else 1
|
|
|
|
|
|
def main(arguments: Sequence[str]) -> int:
|
|
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
|
|
if command == "propose":
|
|
return asyncio.run(propose_lessons(rest))
|
|
if command == "status":
|
|
return show_status()
|
|
if command == "publish":
|
|
return publish_approved()
|
|
if command == "prove":
|
|
return prove_lessons(rest)
|
|
if command == "retire":
|
|
return retire_lesson(rest)
|
|
if command == "reviewed":
|
|
return mark_lessons_reviewed()
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|