feat: the lessons loop, and six lessons approved and published

Capture, distil, review, publish, retire (ADR-0004). Lesson candidates of
past Executions are grouped into what recurred; a lesson-curator role,
outside any Execution, proposes a lesson per group worth one; each proposal
is an issue file a person approves or rejects; an approved lesson is
written into the provider's lessons Skill, which a Specialist is given only
once it holds lessons for that Specialist's Deliverable. LESSONS_ENABLED
off runs without them, for an evaluation with and without.

The curator proposed six lessons from 21 Executions. The owner approved all
six on 2026-10-09 and they are published: five for the target architect,
one for the IaC author. None is proved by an evaluation yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 15:19:32 +01:00
parent 59a641fded
commit a7bc5310f3
29 changed files with 1560 additions and 6 deletions

View File

@@ -21,13 +21,15 @@ def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
assert set(roles) == expected
# The lesson curator runs outside any Execution; the engine does not need it.
assert set(roles) == expected | {"lesson-curator"}
for role in roles.values():
assert role.instructions.strip()
def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
roles = load_roles(AGENTS_DIR, bundle)
# Without lessons: what the bundle's role_skills assigns, whatever has been learned.
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
assert roles["orchestrator"].skills == ("google-cloud-solution-architecture",)
assert roles[PILLAR_ADVISOR].skills == ()

401
tests/test_lessons_loop.py Normal file
View File

@@ -0,0 +1,401 @@
"""The lessons loop (ADR-0004): capture, distil, review, publish, retire.
No lesson reaches a Skill, and so an agent, without an issue file a person approved.
"""
import json
import re
from datetime import date
from pathlib import Path
import pytest
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.roles import load_roles
from app.adk.runtime import RoleSpec
from app.learning.curator import LESSON_CURATOR, curate
from app.learning.evidence import group_candidates, identifiers, read_candidates
from app.learning.proposals import (
APPROVED,
NEEDS_TRIAGE,
REJECTED,
load,
parse,
propose,
write,
)
from app.learning.skill import (
NotApproved,
deliverables_with_lessons,
publish,
published,
retire,
)
from app.providers import load_provider
AGENTS_DIR = Path(__file__).parents[1] / "agents"
CURATOR = RoleSpec(LESSON_CURATOR, "Propose lessons.", budget=30)
DAY = date(2026, 10, 9)
def record(execution: str, *candidates: dict, case: str = "acme-webhooks") -> dict:
return {
"execution_id": execution,
"customer_case": case,
"candidates": list(candidates),
}
def refused(message: str, deliverable="target_architecture") -> dict:
return {
"type": "refusal",
"deliverable": deliverable,
"tool": "submit_deliverable",
"message": message,
}
NO_CHECK = "Not submitted. The Deliverable is missing: Element E-{n} has no Check"
RECORDS = [
record("run-1", refused(NO_CHECK.format(n=3)), refused(NO_CHECK.format(n=9))),
record("run-2", refused(NO_CHECK.format(n=14))),
record(
"run-3",
refused(NO_CHECK.format(n=2)),
refused("Refused"), # a cloud read the Execution was not permitted
refused("Not submitted. Something that happened once."),
),
]
# --- distil ----------------------------------------------------------------------------
def test_what_recurs_across_executions_is_grouped_whatever_its_ids():
[group] = group_candidates(RECORDS, min_executions=2)
assert group.deliverable == "target_architecture" and group.kind == "refusal"
assert group.executions == {"run-1", "run-2", "run-3"}
assert group.occurrences == 4
assert "E-3" not in group.description and "has no Check" in group.description
assert len(group.examples) == 3
def test_a_one_off_and_a_refusal_by_policy_are_not_evidence():
groups = group_candidates(RECORDS, min_executions=1)
descriptions = [group.description for group in groups]
assert len(groups) == 2
assert not any(
description.endswith("refused: Refused") for description in descriptions
)
assert group_candidates(RECORDS[:1], min_executions=2) == []
def test_the_same_evidence_has_the_same_key_on_every_run():
first = group_candidates(RECORDS)[0].key
again = group_candidates(list(reversed(RECORDS)))[0].key
assert first == again
def test_candidates_are_read_where_the_executions_wrote_them(tmp_path):
folder = tmp_path / "run" / "lessons" / "candidates" / "acme-webhooks"
folder.mkdir(parents=True)
(folder / "run-1.json").write_text(json.dumps(RECORDS[0]))
(folder / "broken.json").write_text("not json")
# A folder given twice, or inside another that is given, is read once.
records = read_candidates([tmp_path, tmp_path / "run"])
assert [r["execution_id"] for r in records] == ["run-1"]
assert identifiers(records) == {"run-1", "acme-webhooks"}
# --- review: proposals are issue files ---------------------------------------------------
def proposal(folder: Path, **overrides):
values = {
"title": "Give every Element a Check",
"deliverable": "target_architecture",
"rule": "Before you submit, give every Element at least one Check.",
"reason": "A graph with an unchecked Element is refused, which costs a turn.",
"evidence": "Seen in 3 of 3 Executions, 4 times in all.",
"evidence_key": "abc1234567",
"folder": folder,
"today": DAY,
}
return propose(**{**values, **overrides})
def test_a_proposal_is_an_issue_file_awaiting_a_persons_decision(tmp_path):
made = proposal(tmp_path)
assert made.id == "L-01" and made.status == NEEDS_TRIAGE
assert made.path.name == "01-give-every-element-a-check.md"
text = made.path.read_text()
assert "Status: needs-triage" in text and "## Rule" in text and "## Proof" in text
assert parse(made.path) == made
assert proposal(tmp_path, title="Another").id == "L-02"
def test_a_persons_edits_to_the_file_are_what_is_read(tmp_path):
made = proposal(tmp_path)
text = made.path.read_text().replace("Status: needs-triage", "Status: wontfix")
made.path.write_text(text + "\nNot worth the tokens.\n")
[read] = load(tmp_path)
assert read.status == REJECTED and not read.approved
assert read.comments == "Not worth the tokens."
def test_an_unknown_status_is_an_error_not_an_approval(tmp_path):
made = proposal(tmp_path)
made.path.write_text(made.path.read_text().replace("needs-triage", "approved"))
with pytest.raises(ValueError, match="Status must be one of"):
load(tmp_path)
# --- publish and retire ------------------------------------------------------------------
def lessons_skill(tmp_path: Path) -> Path:
source = Path(__file__).parents[1] / "skills" / "google-cloud-lessons" / "SKILL.md"
skill = tmp_path / "google-cloud-lessons"
skill.mkdir()
# The real Skill's text, with whatever has been published since taken out.
empty = re.sub(
r"<!-- lessons:begin -->.*<!-- lessons:end -->",
"<!-- lessons:begin -->\nNo lessons have been published yet.\n"
"<!-- lessons:end -->",
source.read_text(),
flags=re.DOTALL,
)
(skill / "SKILL.md").write_text(empty)
return skill
def approve(made, folder: Path):
from dataclasses import replace
return write(replace(made, status=APPROVED), folder)
def test_no_lesson_reaches_the_skill_without_an_approved_proposal(tmp_path):
skill = lessons_skill(tmp_path)
made = proposal(tmp_path / "issues")
with pytest.raises(NotApproved, match="L-01 is needs-triage"):
publish(made, skill)
assert published(skill) == {}
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
def test_an_approved_lesson_is_written_for_its_deliverable_and_indexed(tmp_path):
skill = lessons_skill(tmp_path)
made = approve(proposal(tmp_path / "issues"), tmp_path / "issues")
done = publish(made, skill, today=DAY)
reference = (skill / "references" / "target_architecture.md").read_text()
assert "## L-01: Give every Element a Check" in reference
assert "**Rule.** Before you submit, give every Element at least one Check." in (
reference
)
assert "Published 2026-10-09, from lesson proposal 01." in reference
index = (skill / "SKILL.md").read_text()
assert "| target architecture | `references/target_architecture.md` | 1 |" in index
assert "No lessons have been published yet." not in index
assert done.published == "2026-10-09"
assert "Published: 2026-10-09" in done.path.read_text()
def test_publishing_again_replaces_a_lesson_and_never_duplicates_it(tmp_path):
from dataclasses import replace
skill = lessons_skill(tmp_path)
issues = tmp_path / "issues"
first = publish(approve(proposal(issues), issues), skill, today=DAY)
second = approve(
proposal(issues, title="Name settings exactly", evidence_key="def"), issues
)
publish(second, skill, today=DAY)
publish(replace(first, rule="Give each Element two Checks."), skill, today=DAY)
reference = (skill / "references" / "target_architecture.md").read_text()
assert reference.count("## L-01:") == 1 and reference.count("## L-02:") == 1
assert "Give each Element two Checks." in reference
assert published(skill) == {
"L-01": "target_architecture",
"L-02": "target_architecture",
}
def test_a_retired_lesson_is_removed_and_an_empty_reference_goes_with_it(tmp_path):
skill = lessons_skill(tmp_path)
issues = tmp_path / "issues"
publish(approve(proposal(issues), issues), skill, today=DAY)
assert retire("L-01", skill) is True
assert retire("L-01", skill) is False
assert deliverables_with_lessons(skill) == []
assert not (skill / "references" / "target_architecture.md").exists()
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
# --- the curator only proposes -----------------------------------------------------------
def curator_calls(*calls: Call) -> FakeAgentRuntime:
return FakeAgentRuntime({LESSON_CURATOR: [[*calls, Call("finish")]]})
LESSON = {
"deliverable": "target_architecture",
"title": "Give every Element a Check",
"rule": "Before you submit, give every Element at least one Check.",
"reason": "An unchecked Element is refused, which costs a turn.",
}
async def test_the_curator_is_given_the_evidence_and_its_proposals_await_review(
tmp_path,
):
groups = group_candidates(RECORDS)
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
[made] = proposed
assert made.status == NEEDS_TRIAGE and made.evidence_key == groups[0].key
assert made.evidence.startswith("Seen in 3 of 3 Executions, 4 times in all:")
# The issue file holds counts and a general description, never the examples.
assert "E-3" not in made.path.read_text()
context = runtime.runs[0].task.context
assert context["executions"] == 3 and context["groups"][0]["occurrences"] == 4
assert len(context["groups"][0]["examples"]) == 3
async def test_a_lesson_that_names_a_client_or_an_execution_is_refused(tmp_path):
groups = group_candidates(RECORDS)
leaky = {**LESSON, "reason": "It happened to acme-webhooks in run-2."}
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **leaky}))
proposed = await curate(
runtime,
CURATOR,
groups,
executions=3,
identifiers=identifiers(RECORDS),
folder=tmp_path,
)
assert proposed == [] and load(tmp_path) == []
refusal = runtime.runs[0].results[0][1]
assert refusal.startswith("Refused: a lesson carries no client or Execution")
assert "acme-webhooks" in refusal and "run-2" in refusal
async def test_evidence_already_decided_is_not_put_to_the_curator_again(tmp_path):
groups = group_candidates(RECORDS)
first = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
await curate(first, CURATOR, groups, executions=3, folder=tmp_path)
again = FakeAgentRuntime({})
proposed = await curate(again, CURATOR, groups, executions=3, folder=tmp_path)
assert proposed == [] and again.runs == [] # nothing new: the curator is not run
async def test_a_group_the_curator_passes_over_makes_no_proposal(tmp_path):
groups = group_candidates(RECORDS)
runtime = curator_calls(
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."}),
Call("propose_lesson", {"group": "nope", **LESSON}),
)
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
assert proposed == []
assert runtime.runs[0].results[1][1].startswith("Refused: nope is not one")
# --- a Specialist is given the lessons for its own Deliverable ---------------------------
def bundle_with_lessons(tmp_path: Path, *lessons_for: str):
"""The real bundle, with its lessons Skill replaced by one that has lessons."""
import dataclasses
skill = lessons_skill(tmp_path)
issues = tmp_path / "issues"
for number, kind in enumerate(lessons_for):
made = approve(
proposal(issues, deliverable=kind, evidence_key=f"k{number}"), issues
)
publish(made, skill, today=DAY)
bundle = load_provider("gcp")
name = bundle.lessons_skill
assert name is not None
skills = {
**bundle.skills,
name: dataclasses.replace(bundle.skills[name], path=skill),
}
return dataclasses.replace(bundle, skills=skills)
def test_a_specialist_is_given_the_lessons_skill_once_it_has_lessons_for_it(tmp_path):
bundle = bundle_with_lessons(tmp_path, "target_architecture")
roles = load_roles(AGENTS_DIR, bundle)
assert roles["target-architect"].skills[-1] == "google-cloud-lessons"
# A Specialist with nothing to read is not sent to load it.
assert "google-cloud-lessons" not in roles["iac-author"].skills
assert "google-cloud-lessons" not in roles["orchestrator"].skills
def test_lessons_can_be_switched_off_to_prove_what_they_change(tmp_path):
bundle = bundle_with_lessons(tmp_path, "target_architecture")
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
assert all("google-cloud-lessons" not in role.skills for role in roles.values())
def test_the_real_bundle_names_its_lessons_skill_and_gives_it_only_where_it_has_lessons():
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
assert bundle.lessons_skill == "google-cloud-lessons"
learned = set(deliverables_with_lessons(bundle.skills["google-cloud-lessons"].path))
given = {
name for name, role in roles.items() if "google-cloud-lessons" in role.skills
}
from app.workflows.deliverables import DEPENDENCY_GRAPH
assert given == {
spec.specialist
for kind, spec in DEPENDENCY_GRAPH.items()
if kind.value in learned
}
def test_every_published_lesson_has_an_approved_proposal():
"""The real Skill and the real proposals: nothing is in one without the other."""
from app.learning.proposals import ISSUES
root = Path(__file__).parents[1]
live = published(root / "skills" / "google-cloud-lessons")
approved = {p.id: p for p in load(root / ISSUES) if p.approved}
assert set(live) <= set(approved)
for lesson, deliverable in live.items():
assert approved[lesson].deliverable == deliverable
assert approved[lesson].published