Files
gcp_solution_architecture_a…/app/learning/evidence.py
Jonathan Boniface a7bc5310f3 feat: the lessons loop, and six lessons approved and published
Capture, distil, review, publish, retire (ADR-0004). Lesson candidates of
past Executions are grouped into what recurred; a lesson-curator role,
outside any Execution, proposes a lesson per group worth one; each proposal
is an issue file a person approves or rejects; an approved lesson is
written into the provider's lessons Skill, which a Specialist is given only
once it holds lessons for that Specialist's Deliverable. LESSONS_ENABLED
off runs without them, for an evaluation with and without.

The curator proposed six lessons from 21 Executions. The owner approved all
six on 2026-10-09 and they are published: five for the target architect,
one for the IaC author. None is proved by an evaluation yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 15:19:32 +01:00

156 lines
6.0 KiB
Python

"""Lesson candidates across Executions, grouped into what recurs."""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
EXAMPLES = 3
EXAMPLE_LIMIT = 400
# Ids and numbers vary between Executions; what recurs is the shape around them.
_VARIABLE = re.compile(r"`[^`]*`|\"[^\"]*\"|\b[A-Z]{1,4}-[\w.-]+\b|\b\d[\w.]*\b")
# A refusal the Execution's own permissions caused is policy, not a mistake.
_POLICY = ("Refused: reading live cloud resources", "Permission to read live cloud")
@dataclass
class Group:
"""One thing that went wrong the same way in several Executions."""
key: str # stable across runs of the curator: the same evidence, the same key
kind: str # refusal | revision | proposed_addition | failed_check | ...
deliverable: str | None
description: str # general: no id, name or value from any Execution
executions: set[str] = field(default_factory=set)
occurrences: int = 0
examples: list[str] = field(default_factory=list) # client data: never published
def as_context(self) -> dict[str, Any]:
return {
"group": self.key,
"kind": self.kind,
"deliverable": self.deliverable,
"what_happened": self.description,
"executions": len(self.executions),
"occurrences": self.occurrences,
"examples": self.examples,
}
def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
"""Every Execution's lesson-candidate record under the given folders."""
records = []
seen: set[Path] = set()
for root in roots:
for path in sorted(Path(root).rglob("lessons/candidates/*/*.json")):
# One folder may lie inside another that was also given.
if path.resolve() in seen:
continue
seen.add(path.resolve())
try:
record = json.loads(path.read_text(encoding="utf-8"))
except ValueError:
continue
if isinstance(record, dict) and isinstance(record.get("candidates"), list):
records.append(record)
return records
def _general(text: str, limit: int = 110) -> str:
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
"""(kind, Deliverable, general description) of a candidate, or None to ignore it."""
kind = str(candidate.get("type"))
deliverable = candidate.get("deliverable")
if kind == "refusal":
message = str(candidate.get("message", ""))
if message.strip() == "Refused" or message.startswith(_POLICY):
return None
tool = candidate.get("tool")
return kind, deliverable, f"{tool} refused: {_general(message)}"
if kind in ("revision", "revision_limit"):
reason = str(candidate.get("reason", ""))
if reason.startswith("Q: ") or "revision limit" in reason:
return None # the router's own questions, recorded as reasons
if reason.startswith("Add a Check"):
why = "validation proposed a Check the design lacked"
elif "cannot be implemented" in reason:
why = "a Check named something its resource cannot do"
elif " failed" in reason[:60]:
why = "the code failed a Check"
else:
why = _general(reason, 70)
return "revision", deliverable, f"revised because {why}"
if kind == "proposed_addition":
return (
kind,
"target_architecture",
"validation proposed a Check the design lacked",
)
if kind in ("failed_check", "unverifiable_check"):
what = "failed" if kind == "failed_check" else "could not be verified"
return kind, "deployment_guidance", f"a Check {what} at validation"
if kind == "unfinished":
return kind, deliverable, f"the Specialist {candidate.get('reason')}"
if kind == "change_request":
return kind, None, "the user asked for changes at an Approval"
return None
def group_candidates(
records: Sequence[Mapping[str, Any]], *, min_executions: int = 2
) -> list[Group]:
"""What recurred in at least `min_executions` Executions, most widespread first."""
groups: dict[str, Group] = {}
for record in records:
execution = str(record.get("execution_id"))
for candidate in record.get("candidates", ()):
if not isinstance(candidate, Mapping):
continue
shape = _shape(candidate)
if shape is None:
continue
kind, deliverable, description = shape
key = hashlib.sha256(
f"{kind}|{deliverable}|{description}".encode()
).hexdigest()[:10]
group = groups.setdefault(key, Group(key, kind, deliverable, description))
group.executions.add(execution)
group.occurrences += 1
example = str(
candidate.get("message")
or candidate.get("reason")
or candidate.get("statement")
or candidate.get("evidence")
or candidate.get("feedback")
or ""
)[:EXAMPLE_LIMIT]
if (
example
and example not in group.examples
and len(group.examples) < EXAMPLES
):
group.examples.append(example)
return sorted(
(g for g in groups.values() if len(g.executions) >= min_executions),
key=lambda g: (-len(g.executions), -g.occurrences, g.key),
)
def identifiers(records: Sequence[Mapping[str, Any]]) -> set[str]:
"""Names that identify a client or an Execution, which no lesson may carry."""
found = set()
for record in records:
for name in ("execution_id", "customer_case"):
value = record.get(name)
if isinstance(value, str) and len(value) >= 4 and value != "unassigned":
found.add(value)
return found