feat: lessons from what recurs about one setting, and three more approved
Some checks failed
validation / verify (push) Failing after 10s
Some checks failed
validation / verify (push) Failing after 10s
Evidence was grouped only by the kind of problem, and a group gets one lesson, so what kept recurring inside "validation proposed a Check" could not be seen. A proposed or unimplementable Check is now also counted under each provider setting or resource type it names; a subject seen in two or more Executions is its own group and reaches the curator past triage. Over 32 Executions that gave five proposals. The owner approved three, now published: L-07 location Checks, L-08 Cloud SQL backup encryption, L-09 backend service capabilities. L-10 is held for more evidence and L-11 rejected. Also: retiring a lesson marks its proposal, so publish no longer brings a retired lesson back. L-04 had returned to the Skill that way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -101,7 +101,12 @@ async def curate(
|
||||
decided = {proposal.evidence_key for proposal in existing}
|
||||
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
|
||||
if decider is not None and existing:
|
||||
fresh = await _not_already_covered(decider, fresh, existing, covered_at)
|
||||
# A group about one setting is narrower than any lesson about its kind of
|
||||
# problem, and it recurred with those lessons published: it goes to the
|
||||
# curator, who is shown the lessons that exist and may still decline.
|
||||
broad = [group for group in fresh if group.subject is None]
|
||||
kept = await _not_already_covered(decider, broad, existing, covered_at)
|
||||
fresh = [g for g in fresh if g.subject is not None or g in kept]
|
||||
if not fresh:
|
||||
return []
|
||||
by_key = {group.key: group for group in fresh}
|
||||
|
||||
@@ -10,6 +10,8 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.tools.schema_review import named_settings
|
||||
|
||||
EXAMPLES = 3
|
||||
EXAMPLE_LIMIT = 400
|
||||
# Ids and numbers vary between Executions; what recurs is the shape around them.
|
||||
@@ -29,6 +31,9 @@ class Group:
|
||||
executions: set[str] = field(default_factory=set)
|
||||
occurrences: int = 0
|
||||
examples: list[str] = field(default_factory=list) # client data: never published
|
||||
# The provider setting or resource type the problem was about, for a group
|
||||
# narrowed to one subject. None for a group of every problem of its kind.
|
||||
subject: str | None = None
|
||||
|
||||
def as_context(self) -> dict[str, Any]:
|
||||
return {
|
||||
@@ -61,6 +66,10 @@ def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
|
||||
return records
|
||||
|
||||
|
||||
# A subject shorter than this is a stray word, not a setting.
|
||||
_SUBJECT_MIN = 4
|
||||
|
||||
|
||||
def _general(text: str, limit: int = 110) -> str:
|
||||
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
|
||||
|
||||
@@ -104,6 +113,31 @@ def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _subjects(candidate: Mapping[str, Any]) -> tuple[str, list[str]] | None:
|
||||
"""For a problem about what a Check says: (how to describe it, the settings and
|
||||
resource types its text names).
|
||||
|
||||
Problems of one kind are one group, and one group gets one lesson. "Validation
|
||||
proposed a Check" was 20 of 32 Executions and a single lesson; what recurred
|
||||
inside it, such as the same setting missed again and again, was invisible. A
|
||||
setting's name is the provider's vocabulary, not a client's, so it can be the
|
||||
subject of a narrower group.
|
||||
"""
|
||||
kind = str(candidate.get("type"))
|
||||
if kind == "proposed_addition":
|
||||
text = str(candidate.get("statement", ""))
|
||||
what = "validation proposed a Check about `{}` that the design lacked"
|
||||
elif kind == "revision" and "cannot be implemented" in str(
|
||||
candidate.get("reason", "")
|
||||
):
|
||||
text = str(candidate.get("reason", ""))
|
||||
what = "a Check about `{}` asked for something that resource cannot do"
|
||||
else:
|
||||
return None
|
||||
names = {name.split(".")[-1] for name in named_settings(text)}
|
||||
return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN)
|
||||
|
||||
|
||||
def group_candidates(
|
||||
records: Sequence[Mapping[str, Any]], *, min_executions: int = 2
|
||||
) -> list[Group]:
|
||||
@@ -138,6 +172,23 @@ def group_candidates(
|
||||
and len(group.examples) < EXAMPLES
|
||||
):
|
||||
group.examples.append(example)
|
||||
narrowed = _subjects(candidate)
|
||||
for subject in narrowed[1] if narrowed else ():
|
||||
about = narrowed[0].format(subject) # type: ignore[index]
|
||||
subkey = hashlib.sha256(
|
||||
f"{kind}|{deliverable}|{about}".encode()
|
||||
).hexdigest()[:10]
|
||||
narrow = groups.setdefault(
|
||||
subkey, Group(subkey, kind, deliverable, about, subject=subject)
|
||||
)
|
||||
narrow.executions.add(execution)
|
||||
narrow.occurrences += 1
|
||||
if (
|
||||
example
|
||||
and example not in narrow.examples
|
||||
and len(narrow.examples) < EXAMPLES
|
||||
):
|
||||
narrow.examples.append(example)
|
||||
return sorted(
|
||||
(g for g in groups.values() if len(g.executions) >= min_executions),
|
||||
key=lambda g: (-len(g.executions), -g.occurrences, g.key),
|
||||
|
||||
@@ -37,6 +37,9 @@ class Proposal:
|
||||
status: str = NEEDS_TRIAGE
|
||||
proposed: str = ""
|
||||
published: str = ""
|
||||
# The day it was taken out of the Skill again. A retired lesson is not
|
||||
# published a second time, whatever its Status still says.
|
||||
retired: str = ""
|
||||
proof: str = ""
|
||||
comments: str = ""
|
||||
path: Path | None = None
|
||||
@@ -47,7 +50,7 @@ class Proposal:
|
||||
|
||||
@property
|
||||
def approved(self) -> bool:
|
||||
return self.status == APPROVED
|
||||
return self.status == APPROVED and not self.retired
|
||||
|
||||
|
||||
def _slug(title: str) -> str:
|
||||
@@ -65,6 +68,8 @@ def render(proposal: Proposal) -> str:
|
||||
]
|
||||
if proposal.published:
|
||||
lines.append(f"Published: {proposal.published}")
|
||||
if proposal.retired:
|
||||
lines.append(f"Retired: {proposal.retired}")
|
||||
lines += [
|
||||
"",
|
||||
"## Rule",
|
||||
@@ -117,6 +122,7 @@ def parse(path: Path) -> Proposal:
|
||||
status=status,
|
||||
proposed=fields.get("Proposed", ""),
|
||||
published=fields.get("Published", ""),
|
||||
retired=fields.get("Retired", ""),
|
||||
proof="" if proof == "Not yet measured." else proof,
|
||||
comments=sections.get("Comments", ""),
|
||||
path=path,
|
||||
|
||||
@@ -111,6 +111,8 @@ def publish(
|
||||
Raises NotApproved for any proposal a person has not approved: nothing reaches
|
||||
an agent on a curator's say-so.
|
||||
"""
|
||||
if proposal.retired:
|
||||
raise NotApproved(f"{proposal.id} was retired on {proposal.retired}")
|
||||
if not proposal.approved:
|
||||
raise NotApproved(
|
||||
f"{proposal.id} is {proposal.status}; a person approves a lesson by "
|
||||
|
||||
Reference in New Issue
Block a user