Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
187 lines
6.8 KiB
Python
187 lines
6.8 KiB
Python
"""The lessons loop, from the command line (ADR-0004).
|
|
|
|
propose group the lesson candidates of past Executions and have the curator
|
|
propose a lesson for each recurring problem worth one. Proposals are
|
|
written to .scratch/lessons/issues/ as `needs-triage`.
|
|
status list every proposal and where it stands.
|
|
publish write every approved proposal (Status: ready-for-agent) into the
|
|
provider's lessons Skill.
|
|
prove compare an evaluation run without lessons with one run with them,
|
|
and write the comparison into every published proposal.
|
|
retire remove a lesson from the Skill, by id.
|
|
|
|
Usage:
|
|
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
|
|
.venv/bin/python -m eval.curate_lessons status
|
|
.venv/bin/python -m eval.curate_lessons publish
|
|
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
|
|
.venv/bin/python -m eval.curate_lessons retire L-03
|
|
|
|
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
|
|
`wontfix` to reject it. Nothing is published without that. After publishing, prove
|
|
the lessons: run the evaluation with LESSONS_ENABLED=false and again with it on, on
|
|
the same cases, then run `prove` with the two reports. Retire a lesson that does
|
|
not hold quality and lower Revisions, time or tokens.
|
|
|
|
`propose` costs model usage. Lesson candidates are client data; they are read
|
|
where the Executions wrote them and never copied.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sys
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
|
|
from app.agent import open_decider
|
|
from app.config import get_settings
|
|
from app.learning.curator import LESSON_CURATOR, curate
|
|
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
|
from app.learning.proof import compare, record, render
|
|
from app.learning.proposals import ISSUES, load
|
|
from app.learning.skill import NotApproved, publish, published, retire
|
|
from app.providers import ProviderBundle, load_provider
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
# Where Executions write their candidates: under each evaluation run, and under the
|
|
# server's output root.
|
|
DEFAULT_SOURCES = (ROOT / "eval" / "results", ROOT / "lessons" / "candidates" / "..")
|
|
|
|
|
|
def _skill_dir(bundle: ProviderBundle) -> Path:
|
|
if not bundle.lessons_skill:
|
|
raise SystemExit(f"The {bundle.name} provider bundle names no lessons_skill.")
|
|
return bundle.skills[bundle.lessons_skill].path
|
|
|
|
|
|
def _options(arguments: Sequence[str], name: str) -> list[str]:
|
|
return [
|
|
arguments[index + 1]
|
|
for index, argument in enumerate(arguments[:-1])
|
|
if argument == name
|
|
]
|
|
|
|
|
|
async def propose_lessons(arguments: Sequence[str]) -> int:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
from app.adk.roles import load_roles
|
|
|
|
settings = get_settings()
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
sources = [Path(p) for p in _options(arguments, "--from")] or list(DEFAULT_SOURCES)
|
|
minimum = int((_options(arguments, "--min") or ["2"])[0])
|
|
records = read_candidates(sources)
|
|
groups = group_candidates(records, min_executions=minimum)
|
|
print(
|
|
f"{len(records)} Executions, {len(groups)} recurring problems "
|
|
f"(in at least {minimum} Executions each)."
|
|
)
|
|
if not groups:
|
|
return 0
|
|
roles = load_roles(ROOT / "agents", bundle)
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
decider = open_decider(settings)
|
|
try:
|
|
proposed = await curate(
|
|
runtime,
|
|
roles[LESSON_CURATOR],
|
|
groups,
|
|
executions=len(records),
|
|
identifiers=identifiers(records),
|
|
folder=ROOT / ISSUES,
|
|
decider=decider,
|
|
)
|
|
finally:
|
|
aclose = getattr(decider, "aclose", None)
|
|
if aclose:
|
|
await aclose()
|
|
for proposal in proposed:
|
|
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
print(
|
|
f"{len(proposed)} proposed in {ROOT / ISSUES}. Review each one and set its "
|
|
"Status to ready-for-agent or wontfix."
|
|
)
|
|
return 0
|
|
|
|
|
|
def show_status() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
live = published(_skill_dir(bundle))
|
|
proposals = load(ROOT / ISSUES)
|
|
if not proposals:
|
|
print("No lesson has been proposed.")
|
|
for proposal in proposals:
|
|
where = "in the Skill" if proposal.id in live else "not in the Skill"
|
|
proof = "proved" if proposal.proof else "not proved"
|
|
print(
|
|
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
|
|
f"[{proposal.deliverable}] {proposal.title}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def publish_approved() -> int:
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
skill_dir = _skill_dir(bundle)
|
|
done = 0
|
|
for proposal in load(ROOT / ISSUES):
|
|
try:
|
|
publish(proposal, skill_dir)
|
|
except NotApproved:
|
|
continue
|
|
done += 1
|
|
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
|
print(f"{done} lessons are in {skill_dir.name}.")
|
|
return 0
|
|
|
|
|
|
def prove_lessons(arguments: Sequence[str]) -> int:
|
|
without, with_lessons = (
|
|
_options(arguments, "--without"),
|
|
_options(arguments, "--with"),
|
|
)
|
|
if not without or not with_lessons:
|
|
print("Give both reports: --without <report.json> --with <report.json>.")
|
|
return 2
|
|
first, second = Path(without[0]), Path(with_lessons[0])
|
|
result = compare(first, second)
|
|
proof = render(result, first, second)
|
|
recorded = record(load(ROOT / ISSUES), proof)
|
|
print(proof)
|
|
print(f"\nRecorded in {len(recorded)} published proposals.")
|
|
return 0 if result["helps"] else 1
|
|
|
|
|
|
def retire_lesson(arguments: Sequence[str]) -> int:
|
|
if not arguments:
|
|
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
|
|
return 2
|
|
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
|
gone = retire(arguments[0], _skill_dir(bundle))
|
|
print(f"Retired {arguments[0]}." if gone else f"{arguments[0]} is not published.")
|
|
return 0 if gone else 1
|
|
|
|
|
|
def main(arguments: Sequence[str]) -> int:
|
|
command, rest = (arguments[0], arguments[1:]) if arguments else ("status", [])
|
|
if command == "propose":
|
|
return asyncio.run(propose_lessons(rest))
|
|
if command == "status":
|
|
return show_status()
|
|
if command == "publish":
|
|
return publish_approved()
|
|
if command == "prove":
|
|
return prove_lessons(rest)
|
|
if command == "retire":
|
|
return retire_lesson(rest)
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|