Some checks failed
validation / verify (push) Failing after 10s
Evidence was grouped only by the kind of problem, and a group gets one lesson, so what kept recurring inside "validation proposed a Check" could not be seen. A proposed or unimplementable Check is now also counted under each provider setting or resource type it names; a subject seen in two or more Executions is its own group and reaches the curator past triage. Over 32 Executions that gave five proposals. The owner approved three, now published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09 backend service capabilities. L-10 is held for more evidence and L-11 rejected. Also: retiring a lesson marks its proposal, so publish no longer brings a retired lesson back. L-04 had returned to the Skill that way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
664 lines
23 KiB
Python
664 lines
23 KiB
Python
"""The lessons loop (ADR-0004): capture, distil, review, publish, retire.
|
|
|
|
No lesson reaches a Skill, and so an agent, without an issue file a person approved.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
|
|
from app.adk.roles import load_roles
|
|
from app.adk.runtime import Answer, RoleSpec
|
|
from app.learning.curator import LESSON_CURATOR, curate
|
|
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
|
from app.learning.proposals import (
|
|
APPROVED,
|
|
NEEDS_TRIAGE,
|
|
REJECTED,
|
|
load,
|
|
parse,
|
|
propose,
|
|
write,
|
|
)
|
|
from app.learning.skill import (
|
|
NotApproved,
|
|
deliverables_with_lessons,
|
|
publish,
|
|
published,
|
|
retire,
|
|
)
|
|
from app.providers import load_provider
|
|
|
|
AGENTS_DIR = Path(__file__).parents[1] / "agents"
|
|
CURATOR = RoleSpec(LESSON_CURATOR, "Propose lessons.", budget=30)
|
|
DAY = date(2026, 10, 9)
|
|
|
|
|
|
def record(execution: str, *candidates: dict, case: str = "acme-webhooks") -> dict:
|
|
return {
|
|
"execution_id": execution,
|
|
"customer_case": case,
|
|
"candidates": list(candidates),
|
|
}
|
|
|
|
|
|
def refused(message: str, deliverable="target_architecture") -> dict:
|
|
return {
|
|
"type": "refusal",
|
|
"deliverable": deliverable,
|
|
"tool": "submit_deliverable",
|
|
"message": message,
|
|
}
|
|
|
|
|
|
NO_CHECK = "Not submitted. The Deliverable is missing: Element E-{n} has no Check"
|
|
RECORDS = [
|
|
record("run-1", refused(NO_CHECK.format(n=3)), refused(NO_CHECK.format(n=9))),
|
|
record("run-2", refused(NO_CHECK.format(n=14))),
|
|
record(
|
|
"run-3",
|
|
refused(NO_CHECK.format(n=2)),
|
|
refused("Refused"), # a cloud read the Execution was not permitted
|
|
refused("Not submitted. Something that happened once."),
|
|
),
|
|
]
|
|
|
|
|
|
# --- distil ----------------------------------------------------------------------------
|
|
|
|
|
|
def test_what_recurs_across_executions_is_grouped_whatever_its_ids():
|
|
[group] = group_candidates(RECORDS, min_executions=2)
|
|
|
|
assert group.deliverable == "target_architecture" and group.kind == "refusal"
|
|
assert group.executions == {"run-1", "run-2", "run-3"}
|
|
assert group.occurrences == 4
|
|
assert "E-3" not in group.description and "has no Check" in group.description
|
|
assert len(group.examples) == 3
|
|
|
|
|
|
def test_a_one_off_and_a_refusal_by_policy_are_not_evidence():
|
|
groups = group_candidates(RECORDS, min_executions=1)
|
|
|
|
descriptions = [group.description for group in groups]
|
|
assert len(groups) == 2
|
|
assert not any(
|
|
description.endswith("refused: Refused") for description in descriptions
|
|
)
|
|
assert group_candidates(RECORDS[:1], min_executions=2) == []
|
|
|
|
|
|
def test_the_same_evidence_has_the_same_key_on_every_run():
|
|
first = group_candidates(RECORDS)[0].key
|
|
again = group_candidates(list(reversed(RECORDS)))[0].key
|
|
|
|
assert first == again
|
|
|
|
|
|
def test_candidates_are_read_where_the_executions_wrote_them(tmp_path):
|
|
folder = tmp_path / "run" / "lessons" / "candidates" / "acme-webhooks"
|
|
folder.mkdir(parents=True)
|
|
(folder / "run-1.json").write_text(json.dumps(RECORDS[0]))
|
|
(folder / "broken.json").write_text("not json")
|
|
|
|
# A folder given twice, or inside another that is given, is read once.
|
|
records = read_candidates([tmp_path, tmp_path / "run"])
|
|
|
|
assert [r["execution_id"] for r in records] == ["run-1"]
|
|
assert identifiers(records) == {"run-1", "acme-webhooks"}
|
|
|
|
|
|
# --- review: proposals are issue files ---------------------------------------------------
|
|
|
|
|
|
def proposal(folder: Path, **overrides):
|
|
values = {
|
|
"title": "Give every Element a Check",
|
|
"deliverable": "target_architecture",
|
|
"rule": "Before you submit, give every Element at least one Check.",
|
|
"reason": "A graph with an unchecked Element is refused, which costs a turn.",
|
|
"evidence": "Seen in 3 of 3 Executions, 4 times in all.",
|
|
"evidence_key": "abc1234567",
|
|
"folder": folder,
|
|
"today": DAY,
|
|
}
|
|
return propose(**{**values, **overrides})
|
|
|
|
|
|
def test_a_proposal_is_an_issue_file_awaiting_a_persons_decision(tmp_path):
|
|
made = proposal(tmp_path)
|
|
|
|
assert made.id == "L-01" and made.status == NEEDS_TRIAGE
|
|
assert made.path.name == "01-give-every-element-a-check.md"
|
|
text = made.path.read_text()
|
|
assert "Status: needs-triage" in text and "## Rule" in text and "## Proof" in text
|
|
assert parse(made.path) == made
|
|
assert proposal(tmp_path, title="Another").id == "L-02"
|
|
|
|
|
|
def test_a_persons_edits_to_the_file_are_what_is_read(tmp_path):
|
|
made = proposal(tmp_path)
|
|
text = made.path.read_text().replace("Status: needs-triage", "Status: wontfix")
|
|
made.path.write_text(text + "\nNot worth the tokens.\n")
|
|
|
|
[read] = load(tmp_path)
|
|
|
|
assert read.status == REJECTED and not read.approved
|
|
assert read.comments == "Not worth the tokens."
|
|
|
|
|
|
def test_an_unknown_status_is_an_error_not_an_approval(tmp_path):
|
|
made = proposal(tmp_path)
|
|
made.path.write_text(made.path.read_text().replace("needs-triage", "approved"))
|
|
|
|
with pytest.raises(ValueError, match="Status must be one of"):
|
|
load(tmp_path)
|
|
|
|
|
|
# --- publish and retire ------------------------------------------------------------------
|
|
|
|
|
|
def lessons_skill(tmp_path: Path) -> Path:
|
|
source = Path(__file__).parents[1] / "skills" / "google-cloud-lessons" / "SKILL.md"
|
|
skill = tmp_path / "google-cloud-lessons"
|
|
skill.mkdir()
|
|
# The real Skill's text, with whatever has been published since taken out.
|
|
empty = re.sub(
|
|
r"<!-- lessons:begin -->.*<!-- lessons:end -->",
|
|
"<!-- lessons:begin -->\nNo lessons have been published yet.\n"
|
|
"<!-- lessons:end -->",
|
|
source.read_text(),
|
|
flags=re.DOTALL,
|
|
)
|
|
(skill / "SKILL.md").write_text(empty)
|
|
return skill
|
|
|
|
|
|
def approve(made, folder: Path):
|
|
from dataclasses import replace
|
|
|
|
return write(replace(made, status=APPROVED), folder)
|
|
|
|
|
|
def test_no_lesson_reaches_the_skill_without_an_approved_proposal(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
made = proposal(tmp_path / "issues")
|
|
|
|
with pytest.raises(NotApproved, match="L-01 is needs-triage"):
|
|
publish(made, skill)
|
|
|
|
assert published(skill) == {}
|
|
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
|
|
|
|
|
|
def test_an_approved_lesson_is_written_for_its_deliverable_and_indexed(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
made = approve(proposal(tmp_path / "issues"), tmp_path / "issues")
|
|
|
|
done = publish(made, skill, today=DAY)
|
|
|
|
reference = (skill / "references" / "target_architecture.md").read_text()
|
|
assert "## L-01: Give every Element a Check" in reference
|
|
assert "**Rule.** Before you submit, give every Element at least one Check." in (
|
|
reference
|
|
)
|
|
assert "Published 2026-10-09, from lesson proposal 01." in reference
|
|
index = (skill / "SKILL.md").read_text()
|
|
assert "| target architecture | `references/target_architecture.md` | 1 |" in index
|
|
assert "No lessons have been published yet." not in index
|
|
assert done.published == "2026-10-09"
|
|
assert "Published: 2026-10-09" in done.path.read_text()
|
|
|
|
|
|
def test_publishing_again_replaces_a_lesson_and_never_duplicates_it(tmp_path):
|
|
from dataclasses import replace
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
first = publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
second = approve(
|
|
proposal(issues, title="Name settings exactly", evidence_key="def"), issues
|
|
)
|
|
publish(second, skill, today=DAY)
|
|
|
|
publish(replace(first, rule="Give each Element two Checks."), skill, today=DAY)
|
|
|
|
reference = (skill / "references" / "target_architecture.md").read_text()
|
|
assert reference.count("## L-01:") == 1 and reference.count("## L-02:") == 1
|
|
assert "Give each Element two Checks." in reference
|
|
assert published(skill) == {
|
|
"L-01": "target_architecture",
|
|
"L-02": "target_architecture",
|
|
}
|
|
|
|
|
|
def test_a_retired_lesson_is_removed_and_an_empty_reference_goes_with_it(tmp_path):
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
|
|
assert retire("L-01", skill) is True
|
|
assert retire("L-01", skill) is False
|
|
|
|
assert deliverables_with_lessons(skill) == []
|
|
assert not (skill / "references" / "target_architecture.md").exists()
|
|
assert "No lessons have been published yet." in (skill / "SKILL.md").read_text()
|
|
|
|
|
|
# --- the curator only proposes -----------------------------------------------------------
|
|
|
|
|
|
def curator_calls(*calls: Call) -> FakeAgentRuntime:
|
|
return FakeAgentRuntime({LESSON_CURATOR: [[*calls, Call("finish")]]})
|
|
|
|
|
|
LESSON = {
|
|
"deliverable": "target_architecture",
|
|
"title": "Give every Element a Check",
|
|
"rule": "Before you submit, give every Element at least one Check.",
|
|
"reason": "An unchecked Element is refused, which costs a turn.",
|
|
}
|
|
|
|
|
|
async def test_the_curator_is_given_the_evidence_and_its_proposals_await_review(
|
|
tmp_path,
|
|
):
|
|
groups = group_candidates(RECORDS)
|
|
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
|
|
|
|
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
[made] = proposed
|
|
assert made.status == NEEDS_TRIAGE and made.evidence_key == groups[0].key
|
|
assert made.evidence.startswith("Seen in 3 of 3 Executions, 4 times in all:")
|
|
# The issue file holds counts and a general description, never the examples.
|
|
assert "E-3" not in made.path.read_text()
|
|
context = runtime.runs[0].task.context
|
|
assert context["executions"] == 3 and context["groups"][0]["occurrences"] == 4
|
|
assert len(context["groups"][0]["examples"]) == 3
|
|
|
|
|
|
async def test_a_lesson_that_names_a_client_or_an_execution_is_refused(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
leaky = {**LESSON, "reason": "It happened to acme-webhooks in run-2."}
|
|
runtime = curator_calls(Call("propose_lesson", {"group": groups[0].key, **leaky}))
|
|
|
|
proposed = await curate(
|
|
runtime,
|
|
CURATOR,
|
|
groups,
|
|
executions=3,
|
|
identifiers=identifiers(RECORDS),
|
|
folder=tmp_path,
|
|
)
|
|
|
|
assert proposed == [] and load(tmp_path) == []
|
|
refusal = runtime.runs[0].results[0][1]
|
|
assert refusal.startswith("Refused: a lesson carries no client or Execution")
|
|
assert "acme-webhooks" in refusal and "run-2" in refusal
|
|
|
|
|
|
async def test_evidence_already_decided_is_not_put_to_the_curator_again(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
first = curator_calls(Call("propose_lesson", {"group": groups[0].key, **LESSON}))
|
|
await curate(first, CURATOR, groups, executions=3, folder=tmp_path)
|
|
again = FakeAgentRuntime({})
|
|
|
|
proposed = await curate(again, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
assert proposed == [] and again.runs == [] # nothing new: the curator is not run
|
|
|
|
|
|
async def test_a_group_the_curator_passes_over_makes_no_proposal(tmp_path):
|
|
groups = group_candidates(RECORDS)
|
|
runtime = curator_calls(
|
|
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."}),
|
|
Call("propose_lesson", {"group": "nope", **LESSON}),
|
|
)
|
|
|
|
proposed = await curate(runtime, CURATOR, groups, executions=3, folder=tmp_path)
|
|
|
|
assert proposed == []
|
|
assert runtime.runs[0].results[1][1].startswith("Refused: nope is not one")
|
|
|
|
|
|
# --- a Specialist is given the lessons for its own Deliverable ---------------------------
|
|
|
|
|
|
def bundle_with_lessons(tmp_path: Path, *lessons_for: str):
|
|
"""The real bundle, with its lessons Skill replaced by one that has lessons."""
|
|
import dataclasses
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
for number, kind in enumerate(lessons_for):
|
|
made = approve(
|
|
proposal(issues, deliverable=kind, evidence_key=f"k{number}"), issues
|
|
)
|
|
publish(made, skill, today=DAY)
|
|
bundle = load_provider("gcp")
|
|
name = bundle.lessons_skill
|
|
assert name is not None
|
|
skills = {
|
|
**bundle.skills,
|
|
name: dataclasses.replace(bundle.skills[name], path=skill),
|
|
}
|
|
return dataclasses.replace(bundle, skills=skills)
|
|
|
|
|
|
def test_a_specialist_is_given_the_lessons_skill_once_it_has_lessons_for_it(tmp_path):
|
|
bundle = bundle_with_lessons(tmp_path, "target_architecture")
|
|
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert roles["target-architect"].skills[-1] == "google-cloud-lessons"
|
|
# A Specialist with nothing to read is not sent to load it.
|
|
assert "google-cloud-lessons" not in roles["iac-author"].skills
|
|
assert "google-cloud-lessons" not in roles["orchestrator"].skills
|
|
|
|
|
|
def test_lessons_can_be_switched_off_to_prove_what_they_change(tmp_path):
|
|
bundle = bundle_with_lessons(tmp_path, "target_architecture")
|
|
|
|
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
|
|
|
|
assert all("google-cloud-lessons" not in role.skills for role in roles.values())
|
|
|
|
|
|
def test_the_real_bundle_names_its_lessons_skill_and_gives_it_only_where_it_has_lessons():
|
|
bundle = load_provider("gcp")
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert bundle.lessons_skill == "google-cloud-lessons"
|
|
learned = set(deliverables_with_lessons(bundle.skills["google-cloud-lessons"].path))
|
|
given = {
|
|
name for name, role in roles.items() if "google-cloud-lessons" in role.skills
|
|
}
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
|
|
|
assert given == {
|
|
spec.specialist
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
if kind.value in learned
|
|
}
|
|
|
|
|
|
def test_every_published_lesson_has_an_approved_proposal():
|
|
"""The real Skill and the real proposals: nothing is in one without the other."""
|
|
from app.learning.proposals import ISSUES
|
|
|
|
root = Path(__file__).parents[1]
|
|
live = published(root / "skills" / "google-cloud-lessons")
|
|
approved = {p.id: p for p in load(root / ISSUES) if p.approved}
|
|
|
|
assert set(live) <= set(approved)
|
|
for lesson, deliverable in live.items():
|
|
assert approved[lesson].deliverable == deliverable
|
|
assert approved[lesson].published
|
|
|
|
|
|
# --- triage: evidence a lesson already covers ---------------------------------------------
|
|
|
|
|
|
async def test_a_problem_an_existing_lesson_covers_is_not_put_to_the_curator(tmp_path):
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
|
|
proposal(tmp_path, evidence_key="older-evidence") # L-01 exists
|
|
groups = group_candidates(RECORDS)
|
|
decider = FakeDecider({groups[0].key: Answer("L-01", 0.9, {"L-01": 0.9})})
|
|
runtime = FakeAgentRuntime({})
|
|
|
|
proposed = await curate(
|
|
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
|
)
|
|
|
|
assert proposed == [] and runtime.runs == []
|
|
state, questions, _ = decider.asked[0]
|
|
assert "- L-01: Before you submit, give every Element" in state
|
|
assert questions[groups[0].key].options == ("L-01", "none")
|
|
|
|
|
|
async def test_a_problem_no_lesson_covers_still_reaches_the_curator(tmp_path):
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
|
|
proposal(tmp_path, evidence_key="older-evidence")
|
|
groups = group_candidates(RECORDS)
|
|
decider = FakeDecider({groups[0].key: Answer("none", 0.95, {"none": 0.95})})
|
|
runtime = curator_calls(
|
|
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."})
|
|
)
|
|
|
|
await curate(
|
|
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
|
)
|
|
|
|
assert len(runtime.runs) == 1
|
|
|
|
|
|
# --- prove: with against without -----------------------------------------------------------
|
|
|
|
|
|
def report(tmp_path: Path, name: str, **cases) -> Path:
|
|
folder = tmp_path / name
|
|
folder.mkdir()
|
|
path = folder / "report.json"
|
|
path.write_text(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"case_id": case,
|
|
"score": score,
|
|
"details": {
|
|
"revisions": {"target_architecture": revisions},
|
|
"seconds": minutes * 60,
|
|
"tokens": {"total": {"total": tokens}},
|
|
},
|
|
}
|
|
for case, (score, revisions, minutes, tokens) in cases.items()
|
|
]
|
|
)
|
|
)
|
|
return path
|
|
|
|
|
|
def test_lessons_help_when_quality_holds_and_something_falls(tmp_path):
|
|
from app.learning.proof import compare, render
|
|
|
|
without = report(
|
|
tmp_path, "run-a", case_02=(1.0, 2, 40, 5_000_000), other=(1, 0, 1, 1)
|
|
)
|
|
with_lessons = report(tmp_path, "run-b", case_02=(1.0, 0, 25, 3_000_000))
|
|
|
|
result = compare(without, with_lessons)
|
|
|
|
assert result["helps"] and result["lower"] == ["revisions", "seconds", "tokens"]
|
|
assert list(result["cases"]) == ["case_02"] # only what both ran
|
|
text = render(result, without, with_lessons)
|
|
assert "| case_02 | 1.00 → 1.00 | 2 → 0 | 40 → 25 | 5.00 → 3.00 |" in text
|
|
assert "Quality held and revisions, seconds, tokens fell: kept." in text
|
|
assert "`run-a`" in text and "`run-b`" in text
|
|
|
|
|
|
def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_path):
|
|
from app.learning.proof import compare, render
|
|
|
|
base = report(tmp_path, "a", case_02=(1.0, 1, 30, 4_000_000))
|
|
worse = report(tmp_path, "b", case_02=(0.9, 0, 20, 3_000_000))
|
|
same = report(tmp_path, "c", case_02=(1.0, 1, 31, 4_100_000))
|
|
|
|
assert not compare(base, worse)["helps"]
|
|
assert "scored lower" in render(compare(base, worse), base, worse)
|
|
assert not compare(base, same)["helps"]
|
|
assert "nothing fell" in render(compare(base, same), base, same)
|
|
|
|
|
|
def test_the_comparison_is_written_only_for_lessons_in_the_skill(tmp_path):
|
|
from app.learning.proof import record
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
live = publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
waiting = proposal(issues, title="Another", evidence_key="zzz")
|
|
gone = publish(
|
|
approve(proposal(issues, title="Retired", evidence_key="yyy"), issues),
|
|
skill,
|
|
today=DAY,
|
|
)
|
|
retire(gone.id, skill)
|
|
|
|
recorded = record([live, waiting, gone], "| a table |", published(skill))
|
|
|
|
assert [p.id for p in recorded] == ["L-01"]
|
|
assert parse(live.path).proof == "| a table |"
|
|
assert parse(waiting.path).proof == ""
|
|
# A retired lesson was not part of what was compared.
|
|
assert parse(gone.path).proof == ""
|
|
|
|
|
|
# --- lessons are read again when the Skills beside them change ---------------------------
|
|
|
|
|
|
def test_a_changed_upstream_skill_is_flagged_until_the_lessons_are_reviewed(tmp_path):
|
|
from app.learning.skill import mark_reviewed, upstream_changes
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
then = {"architecture": "aaa", "security": "bbb"}
|
|
mark_reviewed(skill, then)
|
|
|
|
assert upstream_changes(skill, then) == []
|
|
now = {"architecture": "aaa", "security": "ccc", "cost": "ddd"}
|
|
assert upstream_changes(skill, now) == ["cost", "security"]
|
|
|
|
mark_reviewed(skill, now)
|
|
assert upstream_changes(skill, now) == []
|
|
|
|
|
|
def test_nothing_is_flagged_with_no_lessons_or_nothing_recorded(tmp_path):
|
|
from app.learning.skill import mark_reviewed, upstream_changes
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
assert upstream_changes(skill, {"architecture": "aaa"}) == []
|
|
mark_reviewed(skill, {"architecture": "aaa"})
|
|
# No lesson is published, so there is nothing to review.
|
|
assert upstream_changes(skill, {"architecture": "zzz"}) == []
|
|
|
|
|
|
def test_the_published_lessons_were_reviewed_against_todays_vendored_skills():
|
|
"""Syncing the vendored Skills changes what the lessons sit beside. This fails
|
|
until a person has read each lesson again and run `curate_lessons reviewed`."""
|
|
from app.learning.skill import upstream_changes, upstream_reviewed
|
|
from app.providers import load_provider
|
|
from eval.curate_lessons import upstream_versions
|
|
|
|
bundle = load_provider("gcp")
|
|
skill = bundle.skills[bundle.lessons_skill].path
|
|
|
|
assert upstream_reviewed(skill), "run: python -m eval.curate_lessons reviewed"
|
|
assert upstream_changes(skill, upstream_versions(bundle)) == []
|
|
|
|
|
|
# --- what recurs inside a kind of problem ------------------------------------------------
|
|
|
|
|
|
def addition(statement: str) -> dict:
|
|
return {"type": "proposed_addition", "statement": statement, "serves": "NFR-1"}
|
|
|
|
|
|
def test_the_same_setting_missed_in_several_executions_is_its_own_group():
|
|
records = [
|
|
record("run-1", addition("`deletion_protection` is true on the database")),
|
|
record("run-2", addition("deletion_protection is enabled"), addition("x y")),
|
|
record("run-3", addition("`min_tls_version` is TLS_1_2")),
|
|
]
|
|
|
|
groups = {group.subject: group for group in group_candidates(records)}
|
|
|
|
# Every proposed Check is one broad group; the setting missed twice is another.
|
|
assert len(groups[None].executions) == 3
|
|
narrow = groups["deletion_protection"]
|
|
assert len(narrow.executions) == 2 and narrow.kind == "proposed_addition"
|
|
assert narrow.description == (
|
|
"validation proposed a Check about `deletion_protection` that the design lacked"
|
|
)
|
|
assert narrow.key != groups[None].key
|
|
# Seen in one Execution only, a setting is not a pattern.
|
|
assert "min_tls_version" not in groups
|
|
|
|
|
|
def test_a_resource_that_checks_keep_asking_too_much_of_is_its_own_group():
|
|
def cannot(why: str) -> dict:
|
|
return {
|
|
"type": "revision",
|
|
"deliverable": "target_architecture",
|
|
"reason": f"Check K-1 cannot be implemented as written ({why}).",
|
|
}
|
|
|
|
records = [
|
|
record("run-1", cannot("example_database has no backup key argument")),
|
|
record("run-2", cannot("example_database has no query access flag")),
|
|
]
|
|
|
|
subjects = {group.subject for group in group_candidates(records)}
|
|
|
|
assert subjects == {None, "example_database"}
|
|
|
|
|
|
async def test_a_group_about_one_setting_reaches_the_curator_past_triage(tmp_path):
|
|
issues = tmp_path / "issues"
|
|
approve(proposal(issues), issues) # a general lesson already exists
|
|
records = [
|
|
record("run-1", addition("`deletion_protection` is true")),
|
|
record("run-2", addition("`deletion_protection` is true")),
|
|
]
|
|
groups = group_candidates(records)
|
|
# Triage says the general lesson covers everything it is asked about.
|
|
decider = FakeDecider(
|
|
{group.key: Answer("L-01", 0.95) for group in groups if group.subject is None}
|
|
)
|
|
seen: list[str] = []
|
|
|
|
def turn(task):
|
|
seen.extend(group["what_happened"] for group in task.context["groups"])
|
|
return [Call("finish")]
|
|
|
|
runtime = FakeAgentRuntime({LESSON_CURATOR: [turn]})
|
|
|
|
await curate(
|
|
runtime,
|
|
CURATOR,
|
|
groups,
|
|
executions=2,
|
|
folder=issues,
|
|
decider=decider,
|
|
)
|
|
|
|
assert seen == [
|
|
"validation proposed a Check about `deletion_protection` that the design lacked"
|
|
]
|
|
|
|
|
|
def test_a_retired_lesson_is_not_published_again(tmp_path):
|
|
"""Retiring left the proposal approved, so the next publish put the lesson
|
|
back: L-04 returned to the Skill that way on 2026-10-10."""
|
|
from dataclasses import replace
|
|
|
|
skill = lessons_skill(tmp_path)
|
|
issues = tmp_path / "issues"
|
|
made = publish(approve(proposal(issues), issues), skill, today=DAY)
|
|
retire(made.id, skill)
|
|
retired = write(replace(parse(made.path), retired="2026-10-09"), issues)
|
|
|
|
assert parse(retired.path).retired == "2026-10-09"
|
|
assert not retired.approved
|
|
with pytest.raises(NotApproved, match="was retired on 2026-10-09"):
|
|
publish(retired, skill)
|
|
assert published(skill) == {}
|