217 lines
8.6 KiB
Python
217 lines
8.6 KiB
Python
"""Lesson candidates across Executions, grouped into what recurs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
EXAMPLES = 3
|
|
EXAMPLE_LIMIT = 400
|
|
# Ids and numbers vary between Executions; what recurs is the shape around them.
|
|
_VARIABLE = re.compile(r"`[^`]*`|\"[^\"]*\"|\b[A-Z]{1,4}-[\w.-]+\b|\b\d[\w.]*\b")
|
|
# A refusal the Execution's own permissions caused is policy, not a mistake.
|
|
_POLICY = ("Refused: reading live cloud resources", "Permission to read live cloud")
|
|
|
|
|
|
@dataclass
|
|
class Group:
|
|
"""One thing that went wrong the same way in several Executions."""
|
|
|
|
key: str # stable across runs of the curator: the same evidence, the same key
|
|
kind: str # refusal | revision | proposed_addition | failed_check | ...
|
|
deliverable: str | None
|
|
description: str # general: no id, name or value from any Execution
|
|
executions: set[str] = field(default_factory=set)
|
|
occurrences: int = 0
|
|
examples: list[str] = field(default_factory=list) # client data: never published
|
|
# The provider setting or resource type the problem was about, for a group
|
|
# narrowed to one subject. None for a group of every problem of its kind.
|
|
subject: str | None = None
|
|
|
|
def as_context(self) -> dict[str, Any]:
|
|
return {
|
|
"group": self.key,
|
|
"kind": self.kind,
|
|
"deliverable": self.deliverable,
|
|
"what_happened": self.description,
|
|
"executions": len(self.executions),
|
|
"occurrences": self.occurrences,
|
|
"examples": self.examples,
|
|
}
|
|
|
|
|
|
def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
|
|
"""Every Execution's lesson-candidate record under the given folders."""
|
|
records = []
|
|
seen: set[Path] = set()
|
|
for root in roots:
|
|
for path in sorted(Path(root).rglob("lessons/candidates/*/*.json")):
|
|
# One folder may lie inside another that was also given.
|
|
if path.resolve() in seen:
|
|
continue
|
|
seen.add(path.resolve())
|
|
try:
|
|
record = json.loads(path.read_text(encoding="utf-8"))
|
|
except ValueError:
|
|
continue
|
|
if isinstance(record, dict) and isinstance(record.get("candidates"), list):
|
|
records.append(record)
|
|
return records
|
|
|
|
|
|
# A subject shorter than this is a stray word, not a setting.
|
|
_SUBJECT_MIN = 4
|
|
|
|
|
|
def _general(text: str, limit: int = 110) -> str:
|
|
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
|
|
|
|
|
|
def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
|
|
"""(kind, Deliverable, general description) of a candidate, or None to ignore it."""
|
|
kind = str(candidate.get("type"))
|
|
deliverable = candidate.get("deliverable")
|
|
if kind == "refusal":
|
|
message = str(candidate.get("message", ""))
|
|
if message.strip() == "Refused" or message.startswith(_POLICY):
|
|
return None
|
|
tool = candidate.get("tool")
|
|
return kind, deliverable, f"{tool} refused: {_general(message)}"
|
|
if kind in ("revision", "revision_limit"):
|
|
reason = str(candidate.get("reason", ""))
|
|
if reason.startswith("Q: ") or "revision limit" in reason:
|
|
return None # the router's own questions, recorded as reasons
|
|
if reason.startswith("Add a Check"):
|
|
why = "validation proposed a Check the design lacked"
|
|
elif "cannot be implemented" in reason:
|
|
why = "a Check named something its resource cannot do"
|
|
elif " failed" in reason[:60]:
|
|
why = "the code failed a Check"
|
|
else:
|
|
why = _general(reason, 70)
|
|
return "revision", deliverable, f"revised because {why}"
|
|
if kind == "proposed_addition":
|
|
return (
|
|
kind,
|
|
"target_architecture",
|
|
"validation proposed a Check the design lacked",
|
|
)
|
|
if kind in ("failed_check", "unverifiable_check"):
|
|
what = "failed" if kind == "failed_check" else "could not be verified"
|
|
return kind, "deployment_guidance", f"a Check {what} at validation"
|
|
if kind == "unfinished":
|
|
return kind, deliverable, f"the Specialist {candidate.get('reason')}"
|
|
if kind == "change_request":
|
|
return kind, None, "the user asked for changes at an Approval"
|
|
return None
|
|
|
|
|
|
def _subjects(
|
|
candidate: Mapping[str, Any], named: Callable[[str], Iterable[str]] | None
|
|
) -> tuple[str, list[str]] | None:
|
|
"""For a problem about what a Check says: (how to describe it, the settings and
|
|
resource types its text names).
|
|
|
|
Problems of one kind are one group, and one group gets one lesson. "Validation
|
|
proposed a Check" was 20 of 32 Executions and a single lesson; what recurred
|
|
inside it, such as the same setting missed again and again, was invisible. A
|
|
setting's name is the provider's vocabulary, not a client's, so it can be the
|
|
subject of a narrower group.
|
|
"""
|
|
kind = str(candidate.get("type"))
|
|
if kind == "proposed_addition":
|
|
text = str(candidate.get("statement", ""))
|
|
what = "validation proposed a Check about `{}` that the design lacked"
|
|
elif kind == "revision" and "cannot be implemented" in str(
|
|
candidate.get("reason", "")
|
|
):
|
|
text = str(candidate.get("reason", ""))
|
|
what = "a Check about `{}` asked for something that resource cannot do"
|
|
else:
|
|
return None
|
|
if named is None:
|
|
return None
|
|
names = {name.split(".")[-1] for name in named(text)}
|
|
return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN)
|
|
|
|
|
|
def group_candidates(
|
|
records: Sequence[Mapping[str, Any]],
|
|
*,
|
|
min_executions: int = 2,
|
|
named: Callable[[str], Iterable[str]] | None = None,
|
|
) -> list[Group]:
|
|
"""What recurred in at least `min_executions` Executions, most widespread first.
|
|
|
|
`named` finds, in a Check's text, the names in a Workflow's own vocabulary that
|
|
can be the subject of a narrower group, such as a provider's settings. Without
|
|
it, problems are grouped by kind alone.
|
|
"""
|
|
groups: dict[str, Group] = {}
|
|
for record in records:
|
|
execution = str(record.get("execution_id"))
|
|
for candidate in record.get("candidates", ()):
|
|
if not isinstance(candidate, Mapping):
|
|
continue
|
|
shape = _shape(candidate)
|
|
if shape is None:
|
|
continue
|
|
kind, deliverable, description = shape
|
|
key = hashlib.sha256(
|
|
f"{kind}|{deliverable}|{description}".encode()
|
|
).hexdigest()[:10]
|
|
group = groups.setdefault(key, Group(key, kind, deliverable, description))
|
|
group.executions.add(execution)
|
|
group.occurrences += 1
|
|
example = str(
|
|
candidate.get("message")
|
|
or candidate.get("reason")
|
|
or candidate.get("statement")
|
|
or candidate.get("evidence")
|
|
or candidate.get("feedback")
|
|
or ""
|
|
)[:EXAMPLE_LIMIT]
|
|
if (
|
|
example
|
|
and example not in group.examples
|
|
and len(group.examples) < EXAMPLES
|
|
):
|
|
group.examples.append(example)
|
|
narrowed = _subjects(candidate, named)
|
|
for subject in narrowed[1] if narrowed else ():
|
|
about = narrowed[0].format(subject) # type: ignore[index]
|
|
subkey = hashlib.sha256(
|
|
f"{kind}|{deliverable}|{about}".encode()
|
|
).hexdigest()[:10]
|
|
narrow = groups.setdefault(
|
|
subkey, Group(subkey, kind, deliverable, about, subject=subject)
|
|
)
|
|
narrow.executions.add(execution)
|
|
narrow.occurrences += 1
|
|
if (
|
|
example
|
|
and example not in narrow.examples
|
|
and len(narrow.examples) < EXAMPLES
|
|
):
|
|
narrow.examples.append(example)
|
|
return sorted(
|
|
(g for g in groups.values() if len(g.executions) >= min_executions),
|
|
key=lambda g: (-len(g.executions), -g.occurrences, g.key),
|
|
)
|
|
|
|
|
|
def identifiers(records: Sequence[Mapping[str, Any]]) -> set[str]:
|
|
"""Names that identify a client or an Execution, which no lesson may carry."""
|
|
found = set()
|
|
for record in records:
|
|
for name in ("execution_id", "customer_case"):
|
|
value = record.get(name)
|
|
if isinstance(value, str) and len(value) >= 4 and value != "unassigned":
|
|
found.add(value)
|
|
return found
|