feat: lessons from what recurs about one setting, and three more approved
Some checks failed
validation / verify (push) Failing after 10s

Evidence was grouped only by the kind of problem, and a group gets one lesson,
so what kept recurring inside "validation proposed a Check" could not be
seen. A proposed or unimplementable Check is now also counted under each
provider setting or resource type it names; a subject seen in two or more
Executions is its own group and reaches the curator past triage.

Over 32 Executions that gave five proposals. The owner approved three, now
published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09
backend service capabilities. L-10 is held for more evidence and L-11
rejected.

Also: retiring a lesson marks its proposal, so publish no longer brings a
retired lesson back. L-04 had returned to the Skill that way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 12:47:18 +01:00
parent ee1ba38867
commit 5feaaa6271
17 changed files with 354 additions and 13 deletions

View File

@@ -101,7 +101,12 @@ async def curate(
decided = {proposal.evidence_key for proposal in existing}
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
if decider is not None and existing:
fresh = await _not_already_covered(decider, fresh, existing, covered_at)
# A group about one setting is narrower than any lesson about its kind of
# problem, and it recurred with those lessons published: it goes to the
# curator, who is shown the lessons that exist and may still decline.
broad = [group for group in fresh if group.subject is None]
kept = await _not_already_covered(decider, broad, existing, covered_at)
fresh = [g for g in fresh if g.subject is not None or g in kept]
if not fresh:
return []
by_key = {group.key: group for group in fresh}

View File

@@ -10,6 +10,8 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.tools.schema_review import named_settings
EXAMPLES = 3
EXAMPLE_LIMIT = 400
# Ids and numbers vary between Executions; what recurs is the shape around them.
@@ -29,6 +31,9 @@ class Group:
executions: set[str] = field(default_factory=set)
occurrences: int = 0
examples: list[str] = field(default_factory=list) # client data: never published
# The provider setting or resource type the problem was about, for a group
# narrowed to one subject. None for a group of every problem of its kind.
subject: str | None = None
def as_context(self) -> dict[str, Any]:
return {
@@ -61,6 +66,10 @@ def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
return records
# A subject shorter than this is a stray word, not a setting.
_SUBJECT_MIN = 4
def _general(text: str, limit: int = 110) -> str:
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
@@ -104,6 +113,31 @@ def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
return None
def _subjects(candidate: Mapping[str, Any]) -> tuple[str, list[str]] | None:
"""For a problem about what a Check says: (how to describe it, the settings and
resource types its text names).
Problems of one kind are one group, and one group gets one lesson. "Validation
proposed a Check" was 20 of 32 Executions and a single lesson; what recurred
inside it, such as the same setting missed again and again, was invisible. A
setting's name is the provider's vocabulary, not a client's, so it can be the
subject of a narrower group.
"""
kind = str(candidate.get("type"))
if kind == "proposed_addition":
text = str(candidate.get("statement", ""))
what = "validation proposed a Check about `{}` that the design lacked"
elif kind == "revision" and "cannot be implemented" in str(
candidate.get("reason", "")
):
text = str(candidate.get("reason", ""))
what = "a Check about `{}` asked for something that resource cannot do"
else:
return None
names = {name.split(".")[-1] for name in named_settings(text)}
return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN)
def group_candidates(
records: Sequence[Mapping[str, Any]], *, min_executions: int = 2
) -> list[Group]:
@@ -138,6 +172,23 @@ def group_candidates(
and len(group.examples) < EXAMPLES
):
group.examples.append(example)
narrowed = _subjects(candidate)
for subject in narrowed[1] if narrowed else ():
about = narrowed[0].format(subject) # type: ignore[index]
subkey = hashlib.sha256(
f"{kind}|{deliverable}|{about}".encode()
).hexdigest()[:10]
narrow = groups.setdefault(
subkey, Group(subkey, kind, deliverable, about, subject=subject)
)
narrow.executions.add(execution)
narrow.occurrences += 1
if (
example
and example not in narrow.examples
and len(narrow.examples) < EXAMPLES
):
narrow.examples.append(example)
return sorted(
(g for g in groups.values() if len(g.executions) >= min_executions),
key=lambda g: (-len(g.executions), -g.occurrences, g.key),

View File

@@ -37,6 +37,9 @@ class Proposal:
status: str = NEEDS_TRIAGE
proposed: str = ""
published: str = ""
# The day it was taken out of the Skill again. A retired lesson is not
# published a second time, whatever its Status still says.
retired: str = ""
proof: str = ""
comments: str = ""
path: Path | None = None
@@ -47,7 +50,7 @@ class Proposal:
@property
def approved(self) -> bool:
return self.status == APPROVED
return self.status == APPROVED and not self.retired
def _slug(title: str) -> str:
@@ -65,6 +68,8 @@ def render(proposal: Proposal) -> str:
]
if proposal.published:
lines.append(f"Published: {proposal.published}")
if proposal.retired:
lines.append(f"Retired: {proposal.retired}")
lines += [
"",
"## Rule",
@@ -117,6 +122,7 @@ def parse(path: Path) -> Proposal:
status=status,
proposed=fields.get("Proposed", ""),
published=fields.get("Published", ""),
retired=fields.get("Retired", ""),
proof="" if proof == "Not yet measured." else proof,
comments=sections.get("Comments", ""),
path=path,

View File

@@ -111,6 +111,8 @@ def publish(
Raises NotApproved for any proposal a person has not approved: nothing reaches
an agent on a curator's say-so.
"""
if proposal.retired:
raise NotApproved(f"{proposal.id} was retired on {proposal.retired}")
if not proposal.approved:
raise NotApproved(
f"{proposal.id} is {proposal.status}; a person approves a lesson by "