"""Lesson candidates across Executions, grouped into what recurs.""" from __future__ import annotations import hashlib import json import re from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any EXAMPLES = 3 EXAMPLE_LIMIT = 400 # Ids and numbers vary between Executions; what recurs is the shape around them. _VARIABLE = re.compile(r"`[^`]*`|\"[^\"]*\"|\b[A-Z]{1,4}-[\w.-]+\b|\b\d[\w.]*\b") # A refusal the Execution's own permissions caused is policy, not a mistake. _POLICY = ("Refused: reading live cloud resources", "Permission to read live cloud") @dataclass class Group: """One thing that went wrong the same way in several Executions.""" key: str # stable across runs of the curator: the same evidence, the same key kind: str # refusal | revision | proposed_addition | failed_check | ... deliverable: str | None description: str # general: no id, name or value from any Execution executions: set[str] = field(default_factory=set) occurrences: int = 0 examples: list[str] = field(default_factory=list) # client data: never published # The provider setting or resource type the problem was about, for a group # narrowed to one subject. None for a group of every problem of its kind. subject: str | None = None def as_context(self) -> dict[str, Any]: return { "group": self.key, "kind": self.kind, "deliverable": self.deliverable, "what_happened": self.description, "executions": len(self.executions), "occurrences": self.occurrences, "examples": self.examples, } def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]: """Every Execution's lesson-candidate record under the given folders.""" records = [] seen: set[Path] = set() for root in roots: for path in sorted(Path(root).rglob("lessons/candidates/*/*.json")): # One folder may lie inside another that was also given. if path.resolve() in seen: continue seen.add(path.resolve()) try: record = json.loads(path.read_text(encoding="utf-8")) except ValueError: continue if isinstance(record, dict) and isinstance(record.get("candidates"), list): records.append(record) return records # A subject shorter than this is a stray word, not a setting. _SUBJECT_MIN = 4 def _general(text: str, limit: int = 110) -> str: return " ".join(_VARIABLE.sub("…", text).split())[:limit] def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None: """(kind, Deliverable, general description) of a candidate, or None to ignore it.""" kind = str(candidate.get("type")) deliverable = candidate.get("deliverable") if kind == "refusal": message = str(candidate.get("message", "")) if message.strip() == "Refused" or message.startswith(_POLICY): return None tool = candidate.get("tool") return kind, deliverable, f"{tool} refused: {_general(message)}" if kind in ("revision", "revision_limit"): reason = str(candidate.get("reason", "")) if reason.startswith("Q: ") or "revision limit" in reason: return None # the router's own questions, recorded as reasons if reason.startswith("Add a Check"): why = "validation proposed a Check the design lacked" elif "cannot be implemented" in reason: why = "a Check named something its resource cannot do" elif " failed" in reason[:60]: why = "the code failed a Check" else: why = _general(reason, 70) return "revision", deliverable, f"revised because {why}" if kind == "proposed_addition": return ( kind, "target_architecture", "validation proposed a Check the design lacked", ) if kind in ("failed_check", "unverifiable_check"): what = "failed" if kind == "failed_check" else "could not be verified" return kind, "deployment_guidance", f"a Check {what} at validation" if kind == "unfinished": return kind, deliverable, f"the Specialist {candidate.get('reason')}" if kind == "change_request": return kind, None, "the user asked for changes at an Approval" return None def _subjects( candidate: Mapping[str, Any], named: Callable[[str], Iterable[str]] | None ) -> tuple[str, list[str]] | None: """For a problem about what a Check says: (how to describe it, the settings and resource types its text names). Problems of one kind are one group, and one group gets one lesson. "Validation proposed a Check" was 20 of 32 Executions and a single lesson; what recurred inside it, such as the same setting missed again and again, was invisible. A setting's name is the provider's vocabulary, not a client's, so it can be the subject of a narrower group. """ kind = str(candidate.get("type")) if kind == "proposed_addition": text = str(candidate.get("statement", "")) what = "validation proposed a Check about `{}` that the design lacked" elif kind == "revision" and "cannot be implemented" in str( candidate.get("reason", "") ): text = str(candidate.get("reason", "")) what = "a Check about `{}` asked for something that resource cannot do" else: return None if named is None: return None names = {name.split(".")[-1] for name in named(text)} return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN) def group_candidates( records: Sequence[Mapping[str, Any]], *, min_executions: int = 2, named: Callable[[str], Iterable[str]] | None = None, ) -> list[Group]: """What recurred in at least `min_executions` Executions, most widespread first. `named` finds, in a Check's text, the names in a Workflow's own vocabulary that can be the subject of a narrower group, such as a provider's settings. Without it, problems are grouped by kind alone. """ groups: dict[str, Group] = {} for record in records: execution = str(record.get("execution_id")) for candidate in record.get("candidates", ()): if not isinstance(candidate, Mapping): continue shape = _shape(candidate) if shape is None: continue kind, deliverable, description = shape key = hashlib.sha256( f"{kind}|{deliverable}|{description}".encode() ).hexdigest()[:10] group = groups.setdefault(key, Group(key, kind, deliverable, description)) group.executions.add(execution) group.occurrences += 1 example = str( candidate.get("message") or candidate.get("reason") or candidate.get("statement") or candidate.get("evidence") or candidate.get("feedback") or "" )[:EXAMPLE_LIMIT] if ( example and example not in group.examples and len(group.examples) < EXAMPLES ): group.examples.append(example) narrowed = _subjects(candidate, named) for subject in narrowed[1] if narrowed else (): about = narrowed[0].format(subject) # type: ignore[index] subkey = hashlib.sha256( f"{kind}|{deliverable}|{about}".encode() ).hexdigest()[:10] narrow = groups.setdefault( subkey, Group(subkey, kind, deliverable, about, subject=subject) ) narrow.executions.add(execution) narrow.occurrences += 1 if ( example and example not in narrow.examples and len(narrow.examples) < EXAMPLES ): narrow.examples.append(example) return sorted( (g for g in groups.values() if len(g.executions) >= min_executions), key=lambda g: (-len(g.executions), -g.occurrences, g.key), ) def identifiers(records: Sequence[Mapping[str, Any]]) -> set[str]: """Names that identify a client or an Execution, which no lesson may carry.""" found = set() for record in records: for name in ("execution_id", "customer_case"): value = record.get(name) if isinstance(value, str) and len(value) >= 4 and value != "unassigned": found.add(value) return found