feat: lessons from what recurs about one setting, and three more approved
Some checks failed
validation / verify (push) Failing after 10s
Some checks failed
validation / verify (push) Failing after 10s
Evidence was grouped only by the kind of problem, and a group gets one lesson, so what kept recurring inside "validation proposed a Check" could not be seen. A proposed or unimplementable Check is now also counted under each provider setting or resource type it names; a subject seen in two or more Executions is its own group and reaches the curator past triage. Over 32 Executions that gave five proposals. The owner approved three, now published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09 backend service capabilities. L-10 is held for more evidence and L-11 rejected. Also: retiring a lesson marks its proposal, so publish no longer brings a retired lesson back. L-04 had returned to the Skill that way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.tools.schema_review import named_settings
|
||||
|
||||
EXAMPLES = 3
|
||||
EXAMPLE_LIMIT = 400
|
||||
# Ids and numbers vary between Executions; what recurs is the shape around them.
|
||||
@@ -29,6 +31,9 @@ class Group:
|
||||
executions: set[str] = field(default_factory=set)
|
||||
occurrences: int = 0
|
||||
examples: list[str] = field(default_factory=list) # client data: never published
|
||||
# The provider setting or resource type the problem was about, for a group
|
||||
# narrowed to one subject. None for a group of every problem of its kind.
|
||||
subject: str | None = None
|
||||
|
||||
def as_context(self) -> dict[str, Any]:
|
||||
return {
|
||||
@@ -61,6 +66,10 @@ def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
|
||||
return records
|
||||
|
||||
|
||||
# A subject shorter than this is a stray word, not a setting.
|
||||
_SUBJECT_MIN = 4
|
||||
|
||||
|
||||
def _general(text: str, limit: int = 110) -> str:
|
||||
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
|
||||
|
||||
@@ -104,6 +113,31 @@ def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _subjects(candidate: Mapping[str, Any]) -> tuple[str, list[str]] | None:
|
||||
"""For a problem about what a Check says: (how to describe it, the settings and
|
||||
resource types its text names).
|
||||
|
||||
Problems of one kind are one group, and one group gets one lesson. "Validation
|
||||
proposed a Check" was 20 of 32 Executions and a single lesson; what recurred
|
||||
inside it, such as the same setting missed again and again, was invisible. A
|
||||
setting's name is the provider's vocabulary, not a client's, so it can be the
|
||||
subject of a narrower group.
|
||||
"""
|
||||
kind = str(candidate.get("type"))
|
||||
if kind == "proposed_addition":
|
||||
text = str(candidate.get("statement", ""))
|
||||
what = "validation proposed a Check about `{}` that the design lacked"
|
||||
elif kind == "revision" and "cannot be implemented" in str(
|
||||
candidate.get("reason", "")
|
||||
):
|
||||
text = str(candidate.get("reason", ""))
|
||||
what = "a Check about `{}` asked for something that resource cannot do"
|
||||
else:
|
||||
return None
|
||||
names = {name.split(".")[-1] for name in named_settings(text)}
|
||||
return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN)
|
||||
|
||||
|
||||
def group_candidates(
|
||||
records: Sequence[Mapping[str, Any]], *, min_executions: int = 2
|
||||
) -> list[Group]:
|
||||
@@ -138,6 +172,23 @@ def group_candidates(
|
||||
and len(group.examples) < EXAMPLES
|
||||
):
|
||||
group.examples.append(example)
|
||||
narrowed = _subjects(candidate)
|
||||
for subject in narrowed[1] if narrowed else ():
|
||||
about = narrowed[0].format(subject) # type: ignore[index]
|
||||
subkey = hashlib.sha256(
|
||||
f"{kind}|{deliverable}|{about}".encode()
|
||||
).hexdigest()[:10]
|
||||
narrow = groups.setdefault(
|
||||
subkey, Group(subkey, kind, deliverable, about, subject=subject)
|
||||
)
|
||||
narrow.executions.add(execution)
|
||||
narrow.occurrences += 1
|
||||
if (
|
||||
example
|
||||
and example not in narrow.examples
|
||||
and len(narrow.examples) < EXAMPLES
|
||||
):
|
||||
narrow.examples.append(example)
|
||||
return sorted(
|
||||
(g for g in groups.values() if len(g.executions) >= min_executions),
|
||||
key=lambda g: (-len(g.executions), -g.occurrences, g.key),
|
||||
|
||||
Reference in New Issue
Block a user