Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
514 lines
18 KiB
Python
514 lines
18 KiB
Python
"""The lessons loop (ADR-0004): capture, distil, review, publish, retire.
|
|
|
|
No lesson reaches a Skill, and so an agent, without an issue file a person approved.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from app.adk.roles import load_roles
|
|
from app.adk.runtime import RoleSpec
|
|
from app.learning.curator import LESSON_CURATOR, curate
|
|
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
|
from app.learning.proposals import (
|
|
APPROVED,
|
|
NEEDS_TRIAGE,
|
|
REJECTED,
|
|
load,
|
|
parse,
|
|
propose,
|
|
write,
|
|
)
|
|
from app.learning.skill import (
|
|
NotApproved,
|
|
deliverables_with_lessons,
|
|
publish,
|
|
published,
|
|
retire,
|
|
)
|
|
from app.providers import load_provider
|
|
|
|
AGENTS_DIR = Path(__file__).parents[1] / "agents"
|
|
CURATOR = RoleSpec(LESSON_CURATOR, "Propose lessons.", budget=30)
|
|
DAY = date(2026, 10, 9)
|
|
|
|
|
|
def record(execution: str, *candidates: dict, case: str = "acme-webhooks") -> dict:
|
|
return {
|
|
"execution_id": execution,
|
|
"customer_case": case,
|
|
"candidates": list(candidates),
|
|
}
|
|
|
|
|
|
def refused(message: str, deliverable="target_architecture") -> dict:
|
|
return {
|
|
"type": "refusal",
|
|
"deliverable": deliverable,
|
|
"tool": "submit_deliverable",
|
|
"message": message,
|
|
}
|
|
|
|
|
|
NO_CHECK = "Not submitted. The Deliverable is missing: Element E-{n} has no Check"
|
|
RECORDS = [
|
|
record("run-1", refused(NO_CHECK.format(n=3)), refused(NO_CHECK.format(n=9))),
|
|
record("run-2", refused(NO_CHECK.format(n=14))),
|
|
record(
|
|
"run-3",
|
|
refused(NO_CHECK.format(n=2)),
|
|
refused("Refused"), # a cloud read the Execution was not permitted
|
|
refused("Not submitted. Something that happened once."),
|
|
),
|
|
]
|
|
|
|
|
|
# --- distil ----------------------------------------------------------------------------
|
|
|
|
|
|
def test_what_recurs_across_executions_is_grouped_whatever_its_ids():
|
|
[group] = group_candidates(RECORDS, min_executions=2)
|
|
|
|
assert group.deliverable == "target_architecture" and group.kind == "refusal"
|
|
assert group.executions == {"run-1", "run-2", "run-3"}
|
|
assert group.occurrences == 4
|
|
assert "E-3" not in group.description and "has no Check" in group.description
|
|
assert len(group.examples) == 3
|
|
|
|
|
|
def test_a_one_off_and_a_refusal_by_policy_are_not_evidence():
|
|
groups = group_candidates(RECORDS, min_executions=1)
|
|
|
|
descriptions = [group.description for group in groups]
|
|
assert len(groups) == 2
|
|
assert not any(
|
|
description.endswith("refused: Refused") for description in descriptions
|
|
)
|
|
assert group_candidates(RECORDS[:1], min_executions=2) == []
|
|
|
|
|
|
def test_the_same_evidence_has_the_same_key_on_every_run():
|
|
first = group_candidates(RECORDS)[0].key
|
|
again = group_candidates(list(reversed(RECORDS)))[0].key
|
|
|
|
assert first == again
|
|
|
|
|
|
def test_candidates_are_read_where_the_executions_wrote_them(tmp_path):
|
|
folder = tmp_path / "run" / "lessons" / "candidates" / "acme-webhooks"
|
|
folder.mkdir(parents=True)
|
|
(folder / "run-1.json").write_text(json.dumps(RECORDS[0]))
|
|
(folder / "broken.json").write_text("not json")
|
|
|
|
# A folder given twice, or inside another that is given, is read once.
|
|
records = read_candidates([tmp_path, tmp_path / "run"])
|
|
|
|
assert [r["execution_id"] for r in records] == ["run-1"]
|
|
assert identifiers(records) == {"run-1", "acme-webhooks"}
|
|
|
|
|
|
# --- review: proposals are issue files ---------------------------------------------------
|
|
|
|
|
|
def proposal(folder: Path, **overrides):
|
|
values = {
|
|
"title": "Give every Element a Check",
|
|
"deliverable": "target_architecture",
|
|
"rule": "Before you submit, give every Element at least one Check.",
|
|
"reason": "A graph with an unchecked Element is refused, which costs a turn.",
|
|
"evidence": "Seen in 3 of 3 Executions, 4 times in all.",
|
|
"evidence_key": "abc1234567",
|
|
"folder": folder,
|
|
"today": DAY,
|
|
}
|
|
return propose(**{**values, **overrides})
|
|
|
|
|
|
def test_a_proposal_is_an_issue_file_awaiting_a_persons_decision(tmp_path):
|
|
made = proposal(tmp_path)
|
|
|
|
assert made.id == "L-01" and made.status == NEEDS_TRIAGE
|
|
assert made.path.name == "01-give-every-element-a-check.md"
|
|
text = made.path.read_text()
|
|
assert "Status: needs-triage" in text and "## Rule" in text and "## Proof" in text
|
|
assert parse(made.path) == made
|
|
assert proposal(tmp_path, title="Another").id == "L-02"
|
|
|
|
|
|
def test_a_persons_edits_to_the_file_are_what_is_read(tmp_path):
|
|
made = proposal(tmp_path)
|
|
text = made.path.read_text().replace("Status: needs-triage", "Status: wontfix")
|
|
made.path.write_text(text + "\nNot worth the tokens.\n")
|
|
|
|
[read] = load(tmp_path)
|
|
|
|
assert read.status == REJECTED and not read.approved
|
|
assert read.comments == "Not worth the tokens."
|
|
|
|
|
|
def test_an_unknown_status_is_an_error_not_an_approval(tmp_path):
|
|
made = proposal(tmp_path)
|
|
made.path.write_text(made.path.read_text().replace("needs-triage", "approved"))
|
|
|
|
with pytest.raises(ValueError, match="Status must be one of"):
|
|
load(tmp_path)
|
|
|
|
|
|
# --- publish and retire ------------------------------------------------------------------
|
|
|
|
|
|
def lessons_skill(tmp_path: Path) -> Path:
|
|
source = Path(__file__).parents[1] / "skills" / "google-cloud-lessons" / "SKILL.md"
|
|
skill = tmp_path / "google-cloud-lessons"
|
|
skill.mkdir()
|
|
# The real Skill's text, with whatever has been published since taken out.
|
|
empty = re.sub(
|
|
r"<!-- lessons:begin -->.*<!-- lessons:end -->",
|
|
"<!-- lessons:begin -->\nNo lessons have been published yet.\n"
|
|
"<!-- lessons:end -->",
|
|
source.read_text(),
|
|
flags=re.DOTALL,
|
|
)
|
|
(skill / "SKILL.md").write_text(empty)
|
|
return skill
|
|
|
|
|
|
def approve(made, folder: Path):
|
|
from dataclasses import replace
|
|
|
|
return write(replace(made, status=APPROVED), folder)
|
|
|
|
|
|
def test_no_lesson_reaches_the_skill_without_an_approved_proposal(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
made = proposal(tmp_path / "issues")
|
|
|
|
with pytest.raises(NotApproved, match="L-01 is needs-triage"):
|
|
publish(made, skill)
|
|
|
|
assert published(skill) == {}
|
|
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
|
|
|
|
|
|
def test_an_approved_lesson_is_written_for_its_deliverable_and_indexed(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
made = approve(proposal(tmp_path / "issues"), tmp_path / "issues")
|
|
|
|
done = publish(made, skill, today=DAY)
|
|
|
|
reference = (skill / "references" / "target_architecture.md").read_text()
|
|
assert "## L-01: Give every Element a Check" in reference
|
|
assert "**Rule.** Before you submit, give every Element at least one Check." in (
|
|
reference
|
|
)
|
|
assert "Published 2026-10-09, from lesson proposal 01." in reference
|
|
index = (skill / "SKILL.md").read_text()
|
|
assert "| target architecture | `references/target_architecture.md` | 1 |" in index
|
|
assert "No lessons have been published yet." not in index
|
|
assert done.published == "2026-10-09"
|
|
assert "Published: 2026-10-09" in done.path.read_text()
|
|
|
|
|
|
def test_publishing_again_replaces_a_lesson_and_never_duplicates_it(tmp_path):
|
|
from dataclasses import replace
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
first = publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
second = approve(
|
|
proposal(issues, title="Name settings exactly", evidence_key="def"), issues
|
|
)
|
|
publish(second, skill, today=DAY)
|
|
|
|
publish(replace(first, rule="Give each Element two Checks."), skill, today=DAY)
|
|
|
|
reference = (skill / "references" / "target_architecture.md").read_text()
|
|
assert reference.count("## L-01:") == 1 and reference.count("## L-02:") == 1
|
|
assert "Give each Element two Checks." in reference
|
|
assert published(skill) == {
|
|
"L-01": "target_architecture",
|
|
"L-02": "target_architecture",
|
|
}
|
|
|
|
|
|
def test_a_retired_lesson_is_removed_and_an_empty_reference_goes_with_it(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
|
|
assert retire("L-01", skill) is True
|
|
assert retire("L-01", skill) is False
|
|
|
|
assert deliverables_with_lessons(skill) == []
|
|
assert not (skill / "references" / "target_architecture.md").exists()
|
|
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
|
|
|
|
|
|
# --- the curator only proposes -----------------------------------------------------------
|
|
|
|
|
|
def curator_calls(*calls: Call) -> FakeAgentRuntime:
|
|
return FakeAgentRuntime({LESSON_CURATOR: [[*calls, Call("finish")]]})
|
|
|
|
|
|
LESSON = {
|
|
"deliverable": "target_architecture",
|
|
"title": "Give every Element a Check",
|
|
"rule": "Before you submit, give every Element at least one Check.",
|
|
"reason": "An unchecked Element is refused, which costs a turn.",
|
|
}
|
|
|
|
|
|
async def test_the_curator_is_given_the_evidence_and_its_proposals_await_review(
|
|
tmp_path,
|
|
):
|
|
groups = group_candidates(RECORDS)
|
|
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
|
|
|
|
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
[made] = proposed
|
|
assert made.status == NEEDS_TRIAGE and made.evidence_key == groups[0].key
|
|
assert made.evidence.startswith("Seen in 3 of 3 Executions, 4 times in all:")
|
|
# The issue file holds counts and a general description, never the examples.
|
|
assert "E-3" not in made.path.read_text()
|
|
context = runtime.runs[0].task.context
|
|
assert context["executions"] == 3 and context["groups"][0]["occurrences"] == 4
|
|
assert len(context["groups"][0]["examples"]) == 3
|
|
|
|
|
|
async def test_a_lesson_that_names_a_client_or_an_execution_is_refused(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
leaky = {**LESSON, "reason": "It happened to acme-webhooks in run-2."}
|
|
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **leaky}))
|
|
|
|
proposed = await curate(
|
|
runtime,
|
|
CURATOR,
|
|
groups,
|
|
executions=3,
|
|
identifiers=identifiers(RECORDS),
|
|
folder=tmp_path,
|
|
)
|
|
|
|
assert proposed == [] and load(tmp_path) == []
|
|
refusal = runtime.runs[0].results[0][1]
|
|
assert refusal.startswith("Refused: a lesson carries no client or Execution")
|
|
assert "acme-webhooks" in refusal and "run-2" in refusal
|
|
|
|
|
|
async def test_evidence_already_decided_is_not_put_to_the_curator_again(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
first = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
|
|
await curate(first, CURATOR, groups, executions=3, folder=tmp_path)
|
|
again = FakeAgentRuntime({})
|
|
|
|
proposed = await curate(again, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
assert proposed == [] and again.runs == [] # nothing new: the curator is not run
|
|
|
|
|
|
async def test_a_group_the_curator_passes_over_makes_no_proposal(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
runtime = curator_calls(
|
|
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."}),
|
|
Call("propose_lesson", {"group": "nope", **LESSON}),
|
|
)
|
|
|
|
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
assert proposed == []
|
|
assert runtime.runs[0].results[1][1].startswith("Refused: nope is not one")
|
|
|
|
|
|
# --- a Specialist is given the lessons for its own Deliverable ---------------------------
|
|
|
|
|
|
def bundle_with_lessons(tmp_path: Path, *lessons_for: str):
|
|
"""The real bundle, with its lessons Skill replaced by one that has lessons."""
|
|
import dataclasses
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
for number, kind in enumerate(lessons_for):
|
|
made = approve(
|
|
proposal(issues, deliverable=kind, evidence_key=f"k{number}"), issues
|
|
)
|
|
publish(made, skill, today=DAY)
|
|
bundle = load_provider("gcp")
|
|
name = bundle.lessons_skill
|
|
assert name is not None
|
|
skills = {
|
|
**bundle.skills,
|
|
name: dataclasses.replace(bundle.skills[name], path=skill),
|
|
}
|
|
return dataclasses.replace(bundle, skills=skills)
|
|
|
|
|
|
def test_a_specialist_is_given_the_lessons_skill_once_it_has_lessons_for_it(tmp_path):
|
|
bundle = bundle_with_lessons(tmp_path, "target_architecture")
|
|
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert roles["target-architect"].skills[-1] == "google-cloud-lessons"
|
|
# A Specialist with nothing to read is not sent to load it.
|
|
assert "google-cloud-lessons" not in roles["iac-author"].skills
|
|
assert "google-cloud-lessons" not in roles["orchestrator"].skills
|
|
|
|
|
|
def test_lessons_can_be_switched_off_to_prove_what_they_change(tmp_path):
|
|
bundle = bundle_with_lessons(tmp_path, "target_architecture")
|
|
|
|
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
|
|
|
|
assert all("google-cloud-lessons" not in role.skills for role in roles.values())
|
|
|
|
|
|
def test_the_real_bundle_names_its_lessons_skill_and_gives_it_only_where_it_has_lessons():
|
|
bundle = load_provider("gcp")
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert bundle.lessons_skill == "google-cloud-lessons"
|
|
learned = set(deliverables_with_lessons(bundle.skills["google-cloud-lessons"].path))
|
|
given = {
|
|
name for name, role in roles.items() if "google-cloud-lessons" in role.skills
|
|
}
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
|
|
|
assert given == {
|
|
spec.specialist
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
if kind.value in learned
|
|
}
|
|
|
|
|
|
def test_every_published_lesson_has_an_approved_proposal():
|
|
"""The real Skill and the real proposals: nothing is in one without the other."""
|
|
from app.learning.proposals import ISSUES
|
|
|
|
root = Path(__file__).parents[1]
|
|
live = published(root / "skills" / "google-cloud-lessons")
|
|
approved = {p.id: p for p in load(root / ISSUES) if p.approved}
|
|
|
|
assert set(live) <= set(approved)
|
|
for lesson, deliverable in live.items():
|
|
assert approved[lesson].deliverable == deliverable
|
|
assert approved[lesson].published
|
|
|
|
|
|
# --- triage: evidence a lesson already covers ---------------------------------------------
|
|
|
|
|
|
async def test_a_problem_an_existing_lesson_covers_is_not_put_to_the_curator(tmp_path):
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
|
|
proposal(tmp_path, evidence_key="older-evidence") # L-01 exists
|
|
groups = group_candidates(RECORDS)
|
|
decider = FakeDecider({groups[0].key: Answer("L-01", 0.9, {"L-01": 0.9})})
|
|
runtime = FakeAgentRuntime({})
|
|
|
|
proposed = await curate(
|
|
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
|
)
|
|
|
|
assert proposed == [] and runtime.runs == []
|
|
state, questions, _ = decider.asked[0]
|
|
assert "- L-01: Before you submit, give every Element" in state
|
|
assert questions[groups[0].key].options == ("L-01", "none")
|
|
|
|
|
|
async def test_a_problem_no_lesson_covers_still_reaches_the_curator(tmp_path):
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
|
|
proposal(tmp_path, evidence_key="older-evidence")
|
|
groups = group_candidates(RECORDS)
|
|
decider = FakeDecider({groups[0].key: Answer("none", 0.95, {"none": 0.95})})
|
|
runtime = curator_calls(
|
|
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."})
|
|
)
|
|
|
|
await curate(
|
|
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
|
)
|
|
|
|
assert len(runtime.runs) == 1
|
|
|
|
|
|
# --- prove: with against without -----------------------------------------------------------
|
|
|
|
|
|
def report(tmp_path: Path, name: str, **cases) -> Path:
|
|
folder = tmp_path / name
|
|
folder.mkdir()
|
|
path = folder / "report.json"
|
|
path.write_text(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"case_id": case,
|
|
"score": score,
|
|
"details": {
|
|
"revisions": {"target_architecture": revisions},
|
|
"seconds": minutes * 60,
|
|
"tokens": {"total": {"total": tokens}},
|
|
},
|
|
}
|
|
for case, (score, revisions, minutes, tokens) in cases.items()
|
|
]
|
|
)
|
|
)
|
|
return path
|
|
|
|
|
|
def test_lessons_help_when_quality_holds_and_something_falls(tmp_path):
|
|
from app.learning.proof import compare, render
|
|
|
|
without = report(
|
|
tmp_path, "run-a", case_02=(1.0, 2, 40, 5_000_000), other=(1, 0, 1, 1)
|
|
)
|
|
with_lessons = report(tmp_path, "run-b", case_02=(1.0, 0, 25, 3_000_000))
|
|
|
|
result = compare(without, with_lessons)
|
|
|
|
assert result["helps"] and result["lower"] == ["revisions", "seconds", "tokens"]
|
|
assert list(result["cases"]) == ["case_02"] # only what both ran
|
|
text = render(result, without, with_lessons)
|
|
assert "| case_02 | 1.00 → 1.00 | 2 → 0 | 40 → 25 | 5.00 → 3.00 |" in text
|
|
assert "Quality held and revisions, seconds, tokens fell: kept." in text
|
|
assert "`run-a`" in text and "`run-b`" in text
|
|
|
|
|
|
def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_path):
|
|
from app.learning.proof import compare, render
|
|
|
|
base = report(tmp_path, "a", case_02=(1.0, 1, 30, 4_000_000))
|
|
worse = report(tmp_path, "b", case_02=(0.9, 0, 20, 3_000_000))
|
|
same = report(tmp_path, "c", case_02=(1.0, 1, 31, 4_100_000))
|
|
|
|
assert not compare(base, worse)["helps"]
|
|
assert "scored lower" in render(compare(base, worse), base, worse)
|
|
assert not compare(base, same)["helps"]
|
|
assert "nothing fell" in render(compare(base, same), base, same)
|
|
|
|
|
|
def test_the_comparison_is_written_into_published_proposals_only(tmp_path):
|
|
from app.learning.proof import record
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
live = publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
waiting = proposal(issues, title="Another", evidence_key="zzz")
|
|
|
|
recorded = record([live, waiting], "| a table |")
|
|
|
|
assert [p.id for p in recorded] == ["L-01"]
|
|
assert parse(live.path).proof == "| a table |"
|
|
assert parse(waiting.path).proof == ""
|