"""The lessons loop (ADR-0004): capture, distil, review, publish, retire. No lesson reaches a Skill, and so an agent, without an issue file a person approved. """ import json import re from datetime import date from pathlib import Path import pytest from app.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider from app.adk.roles import load_roles from app.adk.runtime import Answer, RoleSpec from app.learning.curator import LESSON_CURATOR, curate from app.learning.evidence import group_candidates, identifiers, read_candidates from app.learning.proposals import ( APPROVED, NEEDS_TRIAGE, REJECTED, load, parse, propose, write, ) from app.learning.skill import ( NotApproved, deliverables_with_lessons, publish, published, retire, ) from app.providers import load_provider AGENTS_DIR = Path(__file__).parents[1] / "agents" CURATOR = RoleSpec(LESSON_CURATOR, "Propose lessons.", budget=30) DAY = date(2026, 10, 9) def record(execution: str, *candidates: dict, case: str = "acme-webhooks") -> dict: return { "execution_id": execution, "customer_case": case, "candidates": list(candidates), } def refused(message: str, deliverable="target_architecture") -> dict: return { "type": "refusal", "deliverable": deliverable, "tool": "submit_deliverable", "message": message, } NO_CHECK = "Not submitted. The Deliverable is missing: Element E-{n} has no Check" RECORDS = [ record("run-1", refused(NO_CHECK.format(n=3)), refused(NO_CHECK.format(n=9))), record("run-2", refused(NO_CHECK.format(n=14))), record( "run-3", refused(NO_CHECK.format(n=2)), refused("Refused"), # a cloud read the Execution was not permitted refused("Not submitted. Something that happened once."), ), ] # --- distil ---------------------------------------------------------------------------- def test_what_recurs_across_executions_is_grouped_whatever_its_ids(): [group] = group_candidates(RECORDS, min_executions=2) assert group.deliverable == "target_architecture" and group.kind == "refusal" assert group.executions == {"run-1", "run-2", "run-3"} assert group.occurrences == 4 assert "E-3" not in group.description and "has no Check" in group.description assert len(group.examples) == 3 def test_a_one_off_and_a_refusal_by_policy_are_not_evidence(): groups = group_candidates(RECORDS, min_executions=1) descriptions = [group.description for group in groups] assert len(groups) == 2 assert not any( description.endswith("refused: Refused") for description in descriptions ) assert group_candidates(RECORDS[:1], min_executions=2) == [] def test_the_same_evidence_has_the_same_key_on_every_run(): first = group_candidates(RECORDS)[0].key again = group_candidates(list(reversed(RECORDS)))[0].key assert first == again def test_candidates_are_read_where_the_executions_wrote_them(tmp_path): folder = tmp_path / "run" / "lessons" / "candidates" / "acme-webhooks" folder.mkdir(parents=True) (folder / "run-1.json").write_text(json.dumps(RECORDS[0])) (folder / "broken.json").write_text("not json") # A folder given twice, or inside another that is given, is read once. records = read_candidates([tmp_path, tmp_path / "run"]) assert [r["execution_id"] for r in records] == ["run-1"] assert identifiers(records) == {"run-1", "acme-webhooks"} # --- review: proposals are issue files --------------------------------------------------- def proposal(folder: Path, **overrides): values = { "title": "Give every Element a Check", "deliverable": "target_architecture", "rule": "Before you submit, give every Element at least one Check.", "reason": "A graph with an unchecked Element is refused, which costs a turn.", "evidence": "Seen in 3 of 3 Executions, 4 times in all.", "evidence_key": "abc1234567", "folder": folder, "today": DAY, } return propose(**{**values, **overrides}) def test_a_proposal_is_an_issue_file_awaiting_a_persons_decision(tmp_path): made = proposal(tmp_path) assert made.id == "L-01" and made.status == NEEDS_TRIAGE assert made.path.name == "01-give-every-element-a-check.md" text = made.path.read_text() assert "Status: needs-triage" in text and "## Rule" in text and "## Proof" in text assert parse(made.path) == made assert proposal(tmp_path, title="Another").id == "L-02" def test_a_persons_edits_to_the_file_are_what_is_read(tmp_path): made = proposal(tmp_path) text = made.path.read_text().replace("Status: needs-triage", "Status: wontfix") made.path.write_text(text + "\nNot worth the tokens.\n") [read] = load(tmp_path) assert read.status == REJECTED and not read.approved assert read.comments == "Not worth the tokens." def test_an_unknown_status_is_an_error_not_an_approval(tmp_path): made = proposal(tmp_path) made.path.write_text(made.path.read_text().replace("needs-triage", "approved")) with pytest.raises(ValueError, match="Status must be one of"): load(tmp_path) # --- publish and retire ------------------------------------------------------------------ def lessons_skill(tmp_path: Path) -> Path: source = Path(__file__).parents[1] / "skills" / "google-cloud-lessons" / "SKILL.md" skill = tmp_path / "google-cloud-lessons" skill.mkdir() # The real Skill's text, with whatever has been published since taken out. empty = re.sub( r".*", "\nNo lessons have been published yet.\n" "", source.read_text(), flags=re.DOTALL, ) (skill / "SKILL.md").write_text(empty) return skill def approve(made, folder: Path): from dataclasses import replace return write(replace(made, status=APPROVED), folder) def test_no_lesson_reaches_the_skill_without_an_approved_proposal(tmp_path): skill = lessons_skill(tmp_path) made = proposal(tmp_path / "issues") with pytest.raises(NotApproved, match="L-01 is needs-triage"): publish(made, skill) assert published(skill) == {} assert "No lessons have been published yet." in (skill / "SKILL.md").read_text() def test_an_approved_lesson_is_written_for_its_deliverable_and_indexed(tmp_path): skill = lessons_skill(tmp_path) made = approve(proposal(tmp_path / "issues"), tmp_path / "issues") done = publish(made, skill, today=DAY) reference = (skill / "references" / "target_architecture.md").read_text() assert "## L-01: Give every Element a Check" in reference assert "**Rule.** Before you submit, give every Element at least one Check." in ( reference ) assert "Published 2026-10-09, from lesson proposal 01." in reference index = (skill / "SKILL.md").read_text() assert "| target architecture | `references/target_architecture.md` | 1 |" in index assert "No lessons have been published yet." not in index assert done.published == "2026-10-09" assert "Published: 2026-10-09" in done.path.read_text() def test_publishing_again_replaces_a_lesson_and_never_duplicates_it(tmp_path): from dataclasses import replace skill = lessons_skill(tmp_path) issues = tmp_path / "issues" first = publish(approve(proposal(issues), issues), skill, today=DAY) second = approve( proposal(issues, title="Name settings exactly", evidence_key="def"), issues ) publish(second, skill, today=DAY) publish(replace(first, rule="Give each Element two Checks."), skill, today=DAY) reference = (skill / "references" / "target_architecture.md").read_text() assert reference.count("## L-01:") == 1 and reference.count("## L-02:") == 1 assert "Give each Element two Checks." in reference assert published(skill) == { "L-01": "target_architecture", "L-02": "target_architecture", } def test_a_retired_lesson_is_removed_and_an_empty_reference_goes_with_it(tmp_path): skill = lessons_skill(tmp_path) issues = tmp_path / "issues" publish(approve(proposal(issues), issues), skill, today=DAY) assert retire("L-01", skill) is True assert retire("L-01", skill) is False assert deliverables_with_lessons(skill) == [] assert not (skill / "references" / "target_architecture.md").exists() assert "No lessons have been published yet." in (skill / "SKILL.md").read_text() # --- the curator only proposes ----------------------------------------------------------- def curator_calls(*calls: Call) -> FakeAgentRuntime: return FakeAgentRuntime({LESSON_CURATOR: [[*calls, Call("finish")]]}) LESSON = { "deliverable": "target_architecture", "title": "Give every Element a Check", "rule": "Before you submit, give every Element at least one Check.", "reason": "An unchecked Element is refused, which costs a turn.", } async def test_the_curator_is_given_the_evidence_and_its_proposals_await_review( tmp_path, ): groups = group_candidates(RECORDS) runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON})) proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path) [made] = proposed assert made.status == NEEDS_TRIAGE and made.evidence_key == groups[0].key assert made.evidence.startswith("Seen in 3 of 3 Executions, 4 times in all:") # The issue file holds counts and a general description, never the examples. assert "E-3" not in made.path.read_text() context = runtime.runs[0].task.context assert context["executions"] == 3 and context["groups"][0]["occurrences"] == 4 assert len(context["groups"][0]["examples"]) == 3 async def test_a_lesson_that_names_a_client_or_an_execution_is_refused(tmp_path): groups = group_candidates(RECORDS) leaky = {**LESSON, "reason": "It happened to acme-webhooks in run-2."} runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **leaky})) proposed = await curate( runtime, CURATOR, groups, executions=3, identifiers=identifiers(RECORDS), folder=tmp_path, ) assert proposed == [] and load(tmp_path) == [] refusal = runtime.runs[0].results[0][1] assert refusal.startswith("Refused: a lesson carries no client or Execution") assert "acme-webhooks" in refusal and "run-2" in refusal async def test_evidence_already_decided_is_not_put_to_the_curator_again(tmp_path): groups = group_candidates(RECORDS) first = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON})) await curate(first, CURATOR, groups, executions=3, folder=tmp_path) again = FakeAgentRuntime({}) proposed = await curate(again, CURATOR, groups, executions=3, folder=tmp_path) assert proposed == [] and again.runs == [] # nothing new: the curator is not run async def test_a_group_the_curator_passes_over_makes_no_proposal(tmp_path): groups = group_candidates(RECORDS) runtime = curator_calls( Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."}), Call("propose_lesson", {"group": "nope", **LESSON}), ) proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path) assert proposed == [] assert runtime.runs[0].results[1][1].startswith("Refused: nope is not one") # --- a Specialist is given the lessons for its own Deliverable --------------------------- def bundle_with_lessons(tmp_path: Path, *lessons_for: str): """The real bundle, with its lessons Skill replaced by one that has lessons.""" import dataclasses skill = lessons_skill(tmp_path) issues = tmp_path / "issues" for number, kind in enumerate(lessons_for): made = approve( proposal(issues, deliverable=kind, evidence_key=f"k{number}"), issues ) publish(made, skill, today=DAY) bundle = load_provider("gcp") name = bundle.lessons_skill assert name is not None skills = { **bundle.skills, name: dataclasses.replace(bundle.skills[name], path=skill), } return dataclasses.replace(bundle, skills=skills) def test_a_specialist_is_given_the_lessons_skill_once_it_has_lessons_for_it(tmp_path): bundle = bundle_with_lessons(tmp_path, "target_architecture") roles = load_roles(AGENTS_DIR, bundle) assert roles["target-architect"].skills[-1] == "google-cloud-lessons" # A Specialist with nothing to read is not sent to load it. assert "google-cloud-lessons" not in roles["iac-author"].skills assert "google-cloud-lessons" not in roles["orchestrator"].skills def test_lessons_can_be_switched_off_to_prove_what_they_change(tmp_path): bundle = bundle_with_lessons(tmp_path, "target_architecture") roles = load_roles(AGENTS_DIR, bundle, lessons=False) assert all("google-cloud-lessons" not in role.skills for role in roles.values()) def test_the_real_bundle_names_its_lessons_skill_and_gives_it_only_where_it_has_lessons(): bundle = load_provider("gcp") roles = load_roles(AGENTS_DIR, bundle) assert bundle.lessons_skill == "google-cloud-lessons" learned = set(deliverables_with_lessons(bundle.skills["google-cloud-lessons"].path)) given = { name for name, role in roles.items() if "google-cloud-lessons" in role.skills } from app.workflows.deliverables import DEPENDENCY_GRAPH assert given == { spec.specialist for kind, spec in DEPENDENCY_GRAPH.items() if kind.value in learned } def test_every_published_lesson_has_an_approved_proposal(): """The real Skill and the real proposals: nothing is in one without the other.""" from app.learning.proposals import ISSUES root = Path(__file__).parents[1] live = published(root / "skills" / "google-cloud-lessons") approved = {p.id: p for p in load(root / ISSUES) if p.approved} assert set(live) <= set(approved) for lesson, deliverable in live.items(): assert approved[lesson].deliverable == deliverable assert approved[lesson].published # --- triage: evidence a lesson already covers --------------------------------------------- async def test_a_problem_an_existing_lesson_covers_is_not_put_to_the_curator(tmp_path): from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer proposal(tmp_path, evidence_key="older-evidence") # L-01 exists groups = group_candidates(RECORDS) decider = FakeDecider({groups[0].key: Answer("L-01", 0.9, {"L-01": 0.9})}) runtime = FakeAgentRuntime({}) proposed = await curate( runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider ) assert proposed == [] and runtime.runs == [] state, questions, _ = decider.asked[0] assert "- L-01: Before you submit, give every Element" in state assert questions[groups[0].key].options == ("L-01", "none") async def test_a_problem_no_lesson_covers_still_reaches_the_curator(tmp_path): from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer proposal(tmp_path, evidence_key="older-evidence") groups = group_candidates(RECORDS) decider = FakeDecider({groups[0].key: Answer("none", 0.95, {"none": 0.95})}) runtime = curator_calls( Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."}) ) await curate( runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider ) assert len(runtime.runs) == 1 # --- prove: with against without ----------------------------------------------------------- def report(tmp_path: Path, name: str, **cases) -> Path: folder = tmp_path / name folder.mkdir() path = folder / "report.json" path.write_text( json.dumps( [ { "case_id": case, "score": score, "details": { "revisions": {"target_architecture": revisions}, "seconds": minutes * 60, "tokens": {"total": {"total": tokens}}, }, } for case, (score, revisions, minutes, tokens) in cases.items() ] ) ) return path def test_lessons_help_when_quality_holds_and_something_falls(tmp_path): from app.learning.proof import compare, render without = report( tmp_path, "run-a", case_02=(1.0, 2, 40, 5_000_000), other=(1, 0, 1, 1) ) with_lessons = report(tmp_path, "run-b", case_02=(1.0, 0, 25, 3_000_000)) result = compare(without, with_lessons) assert result["helps"] and result["lower"] == ["revisions", "seconds", "tokens"] assert list(result["cases"]) == ["case_02"] # only what both ran text = render(result, without, with_lessons) assert "| case_02 | 1.00 → 1.00 | 2 → 0 | 40 → 25 | 5.00 → 3.00 |" in text assert "Quality held and revisions, seconds, tokens fell: kept." in text assert "`run-a`" in text and "`run-b`" in text def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_path): from app.learning.proof import compare, render base = report(tmp_path, "a", case_02=(1.0, 1, 30, 4_000_000)) worse = report(tmp_path, "b", case_02=(0.9, 0, 20, 3_000_000)) same = report(tmp_path, "c", case_02=(1.0, 1, 31, 4_100_000)) assert not compare(base, worse)["helps"] assert "scored lower" in render(compare(base, worse), base, worse) assert not compare(base, same)["helps"] assert "nothing fell" in render(compare(base, same), base, same) def test_the_comparison_is_written_only_for_lessons_in_the_skill(tmp_path): from app.learning.proof import record skill = lessons_skill(tmp_path) issues = tmp_path / "issues" live = publish(approve(proposal(issues), issues), skill, today=DAY) waiting = proposal(issues, title="Another", evidence_key="zzz") gone = publish( approve(proposal(issues, title="Retired", evidence_key="yyy"), issues), skill, today=DAY, ) retire(gone.id, skill) recorded = record([live, waiting, gone], "| a table |", published(skill)) assert [p.id for p in recorded] == ["L-01"] assert parse(live.path).proof == "| a table |" assert parse(waiting.path).proof == "" # A retired lesson was not part of what was compared. assert parse(gone.path).proof == "" # --- lessons are read again when the Skills beside them change --------------------------- def test_a_changed_upstream_skill_is_flagged_until_the_lessons_are_reviewed(tmp_path): from app.learning.skill import mark_reviewed, upstream_changes skill = lessons_skill(tmp_path) issues = tmp_path / "issues" publish(approve(proposal(issues), issues), skill, today=DAY) then = {"architecture": "aaa", "security": "bbb"} mark_reviewed(skill, then) assert upstream_changes(skill, then) == [] now = {"architecture": "aaa", "security": "ccc", "cost": "ddd"} assert upstream_changes(skill, now) == ["cost", "security"] mark_reviewed(skill, now) assert upstream_changes(skill, now) == [] def test_nothing_is_flagged_with_no_lessons_or_nothing_recorded(tmp_path): from app.learning.skill import mark_reviewed, upstream_changes skill = lessons_skill(tmp_path) assert upstream_changes(skill, {"architecture": "aaa"}) == [] mark_reviewed(skill, {"architecture": "aaa"}) # No lesson is published, so there is nothing to review. assert upstream_changes(skill, {"architecture": "zzz"}) == [] def test_the_published_lessons_were_reviewed_against_todays_vendored_skills(): """Syncing the vendored Skills changes what the lessons sit beside. This fails until a person has read each lesson again and run `curate_lessons reviewed`.""" from app.learning.skill import upstream_changes, upstream_reviewed from app.providers import load_provider from eval.curate_lessons import upstream_versions bundle = load_provider("gcp") skill = bundle.skills[bundle.lessons_skill].path assert upstream_reviewed(skill), "run: python -m eval.curate_lessons reviewed" assert upstream_changes(skill, upstream_versions(bundle)) == [] # --- what recurs inside a kind of problem ------------------------------------------------ def addition(statement: str) -> dict: return {"type": "proposed_addition", "statement": statement, "serves": "NFR-1"} def test_the_same_setting_missed_in_several_executions_is_its_own_group(): records = [ record("run-1", addition("`deletion_protection` is true on the database")), record("run-2", addition("deletion_protection is enabled"), addition("x y")), record("run-3", addition("`min_tls_version` is TLS_1_2")), ] groups = {group.subject: group for group in group_candidates(records)} # Every proposed Check is one broad group; the setting missed twice is another. assert len(groups[None].executions) == 3 narrow = groups["deletion_protection"] assert len(narrow.executions) == 2 and narrow.kind == "proposed_addition" assert narrow.description == ( "validation proposed a Check about `deletion_protection` that the design lacked" ) assert narrow.key != groups[None].key # Seen in one Execution only, a setting is not a pattern. assert "min_tls_version" not in groups def test_a_resource_that_checks_keep_asking_too_much_of_is_its_own_group(): def cannot(why: str) -> dict: return { "type": "revision", "deliverable": "target_architecture", "reason": f"Check K-1 cannot be implemented as written ({why}).", } records = [ record("run-1", cannot("example_database has no backup key argument")), record("run-2", cannot("example_database has no query access flag")), ] subjects = {group.subject for group in group_candidates(records)} assert subjects == {None, "example_database"} async def test_a_group_about_one_setting_reaches_the_curator_past_triage(tmp_path): issues = tmp_path / "issues" approve(proposal(issues), issues) # a general lesson already exists records = [ record("run-1", addition("`deletion_protection` is true")), record("run-2", addition("`deletion_protection` is true")), ] groups = group_candidates(records) # Triage says the general lesson covers everything it is asked about. decider = FakeDecider( {group.key: Answer("L-01", 0.95) for group in groups if group.subject is None} ) seen: list[str] = [] def turn(task): seen.extend(group["what_happened"] for group in task.context["groups"]) return [Call("finish")] runtime = FakeAgentRuntime({LESSON_CURATOR: [turn]}) await curate( runtime, CURATOR, groups, executions=2, folder=issues, decider=decider, ) assert seen == [ "validation proposed a Check about `deletion_protection` that the design lacked" ] def test_a_retired_lesson_is_not_published_again(tmp_path): """Retiring left the proposal approved, so the next publish put the lesson back: L-04 returned to the Skill that way on 2026-10-10.""" from dataclasses import replace skill = lessons_skill(tmp_path) issues = tmp_path / "issues" made = publish(approve(proposal(issues), issues), skill, today=DAY) retire(made.id, skill) retired = write(replace(parse(made.path), retired="2026-10-09"), issues) assert parse(retired.path).retired == "2026-10-09" assert not retired.approved with pytest.raises(NotApproved, match="was retired on 2026-10-09"): publish(retired, skill) assert published(skill) == {}