feat: lessons from what recurs about one setting, and three more approved
Some checks failed
validation / verify (push) Failing after 10s
Some checks failed
validation / verify (push) Failing after 10s
Evidence was grouped only by the kind of problem, and a group gets one lesson, so what kept recurring inside "validation proposed a Check" could not be seen. A proposed or unimplementable Check is now also counted under each provider setting or resource type it names; a subject seen in two or more Executions is its own group and reaches the curator past triage. Over 32 Executions that gave five proposals. The owner approved three, now published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09 backend service capabilities. L-10 is held for more evidence and L-11 rejected. Also: retiring a lesson marks its proposal, so publish no longer brings a retired lesson back. L-04 had returned to the Skill that way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,9 +10,9 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
|
||||
from app.adk.roles import load_roles
|
||||
from app.adk.runtime import RoleSpec
|
||||
from app.adk.runtime import Answer, RoleSpec
|
||||
from app.learning.curator import LESSON_CURATOR, curate
|
||||
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
||||
from app.learning.proposals import (
|
||||
@@ -563,3 +563,101 @@ def test_the_published_lessons_were_reviewed_against_todays_vendored_skills():
|
||||
|
||||
assert upstream_reviewed(skill), "run: python -m eval.curate_lessons reviewed"
|
||||
assert upstream_changes(skill, upstream_versions(bundle)) == []
|
||||
|
||||
|
||||
# --- what recurs inside a kind of problem ------------------------------------------------
|
||||
|
||||
|
||||
def addition(statement: str) -> dict:
|
||||
return {"type": "proposed_addition", "statement": statement, "serves": "NFR-1"}
|
||||
|
||||
|
||||
def test_the_same_setting_missed_in_several_executions_is_its_own_group():
|
||||
records = [
|
||||
record("run-1", addition("`deletion_protection` is true on the database")),
|
||||
record("run-2", addition("deletion_protection is enabled"), addition("x y")),
|
||||
record("run-3", addition("`min_tls_version` is TLS_1_2")),
|
||||
]
|
||||
|
||||
groups = {group.subject: group for group in group_candidates(records)}
|
||||
|
||||
# Every proposed Check is one broad group; the setting missed twice is another.
|
||||
assert len(groups[None].executions) == 3
|
||||
narrow = groups["deletion_protection"]
|
||||
assert len(narrow.executions) == 2 and narrow.kind == "proposed_addition"
|
||||
assert narrow.description == (
|
||||
"validation proposed a Check about `deletion_protection` that the design lacked"
|
||||
)
|
||||
assert narrow.key != groups[None].key
|
||||
# Seen in one Execution only, a setting is not a pattern.
|
||||
assert "min_tls_version" not in groups
|
||||
|
||||
|
||||
def test_a_resource_that_checks_keep_asking_too_much_of_is_its_own_group():
|
||||
def cannot(why: str) -> dict:
|
||||
return {
|
||||
"type": "revision",
|
||||
"deliverable": "target_architecture",
|
||||
"reason": f"Check K-1 cannot be implemented as written ({why}).",
|
||||
}
|
||||
|
||||
records = [
|
||||
record("run-1", cannot("example_database has no backup key argument")),
|
||||
record("run-2", cannot("example_database has no query access flag")),
|
||||
]
|
||||
|
||||
subjects = {group.subject for group in group_candidates(records)}
|
||||
|
||||
assert subjects == {None, "example_database"}
|
||||
|
||||
|
||||
async def test_a_group_about_one_setting_reaches_the_curator_past_triage(tmp_path):
|
||||
issues = tmp_path / "issues"
|
||||
approve(proposal(issues), issues) # a general lesson already exists
|
||||
records = [
|
||||
record("run-1", addition("`deletion_protection` is true")),
|
||||
record("run-2", addition("`deletion_protection` is true")),
|
||||
]
|
||||
groups = group_candidates(records)
|
||||
# Triage says the general lesson covers everything it is asked about.
|
||||
decider = FakeDecider(
|
||||
{group.key: Answer("L-01", 0.95) for group in groups if group.subject is None}
|
||||
)
|
||||
seen: list[str] = []
|
||||
|
||||
def turn(task):
|
||||
seen.extend(group["what_happened"] for group in task.context["groups"])
|
||||
return [Call("finish")]
|
||||
|
||||
runtime = FakeAgentRuntime({LESSON_CURATOR: [turn]})
|
||||
|
||||
await curate(
|
||||
runtime,
|
||||
CURATOR,
|
||||
groups,
|
||||
executions=2,
|
||||
folder=issues,
|
||||
decider=decider,
|
||||
)
|
||||
|
||||
assert seen == [
|
||||
"validation proposed a Check about `deletion_protection` that the design lacked"
|
||||
]
|
||||
|
||||
|
||||
def test_a_retired_lesson_is_not_published_again(tmp_path):
|
||||
"""Retiring left the proposal approved, so the next publish put the lesson
|
||||
back: L-04 returned to the Skill that way on 2026-10-10."""
|
||||
from dataclasses import replace
|
||||
|
||||
skill = lessons_skill(tmp_path)
|
||||
issues = tmp_path / "issues"
|
||||
made = publish(approve(proposal(issues), issues), skill, today=DAY)
|
||||
retire(made.id, skill)
|
||||
retired = write(replace(parse(made.path), retired="2026-10-09"), issues)
|
||||
|
||||
assert parse(retired.path).retired == "2026-10-09"
|
||||
assert not retired.approved
|
||||
with pytest.raises(NotApproved, match="was retired on 2026-10-09"):
|
||||
publish(retired, skill)
|
||||
assert published(skill) == {}
|
||||
|
||||
Reference in New Issue
Block a user