Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
7.1 KiB
Python
217 lines
7.1 KiB
Python
"""The lesson curator: proposes a lesson for each recurring problem worth one.
|
|
|
|
It runs outside any Execution, on the evidence of many. It only proposes: every
|
|
proposal is written as an issue file for a person to decide (ADR-0004).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
AgentRuntime,
|
|
AgentTask,
|
|
Decider,
|
|
Question,
|
|
RoleSpec,
|
|
ToolResult,
|
|
ToolSpec,
|
|
TurnEnded,
|
|
)
|
|
from app.learning.evidence import Group
|
|
from app.learning.proposals import ISSUES, Proposal, load, propose
|
|
from app.workflows.deliverables import DeliverableKind
|
|
|
|
LESSON_CURATOR = "lesson-curator"
|
|
MAX_GROUPS = 25
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def _not_already_covered(
|
|
decider: Decider,
|
|
groups: Sequence[Group],
|
|
lessons: Sequence[Proposal],
|
|
covered_at: float,
|
|
) -> list[Group]:
|
|
"""The groups no existing lesson already covers (ADR-0007: lesson triage).
|
|
|
|
One choice per group among the lessons that exist, or none. A group a lesson
|
|
covers is not put to the curator again, so the same lesson is not proposed
|
|
twice in other words. With no decision every group is kept.
|
|
"""
|
|
live = [lesson for lesson in lessons if lesson.status != "wontfix"]
|
|
if not groups or not live:
|
|
return list(groups)
|
|
answers = await decider.decide(
|
|
"Lessons that already exist:\n"
|
|
+ "\n".join(f"- {lesson.id}: {lesson.rule}" for lesson in live),
|
|
{
|
|
group.key: Question(
|
|
"choice",
|
|
f"A problem that recurred: {group.description} (on "
|
|
f"{group.deliverable}). Which one existing lesson would have "
|
|
"prevented it? Answer none when no lesson would.",
|
|
(*(lesson.id for lesson in live), "none"),
|
|
)
|
|
for group in groups
|
|
},
|
|
scope="lesson-triage",
|
|
)
|
|
if not answers:
|
|
return list(groups)
|
|
kept = []
|
|
for group in groups:
|
|
answer = answers.get(group.key)
|
|
if (
|
|
answer is not None
|
|
and answer.value != "none"
|
|
and answer.confidence >= covered_at
|
|
):
|
|
logger.info("%s is already covered by %s", group.key, answer.value)
|
|
continue
|
|
kept.append(group)
|
|
return kept
|
|
|
|
|
|
def _evidence(group: Group, executions: int) -> str:
|
|
return (
|
|
f"Seen in {len(group.executions)} of {executions} Executions, "
|
|
f"{group.occurrences} times in all: {group.description}."
|
|
)
|
|
|
|
|
|
async def curate(
|
|
runtime: AgentRuntime,
|
|
role: RoleSpec,
|
|
groups: Sequence[Group],
|
|
*,
|
|
executions: int,
|
|
identifiers: set[str] = frozenset(), # type: ignore[assignment]
|
|
folder: Path = ISSUES,
|
|
decider: Decider | None = None,
|
|
covered_at: float = 0.6,
|
|
) -> list[Proposal]:
|
|
"""Run the curator over the groups that have no proposal yet; return what it
|
|
proposed. Groups it judged not worth a lesson are left for the next run."""
|
|
existing = load(folder)
|
|
decided = {proposal.evidence_key for proposal in existing}
|
|
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
|
|
if decider is not None and existing:
|
|
fresh = await _not_already_covered(decider, fresh, existing, covered_at)
|
|
if not fresh:
|
|
return []
|
|
by_key = {group.key: group for group in fresh}
|
|
kinds = [kind.value for kind in DeliverableKind]
|
|
proposed: list[Proposal] = []
|
|
declined: dict[str, str] = {}
|
|
|
|
async def propose_lesson(
|
|
group: str, deliverable: str, title: str, rule: str, reason: str
|
|
) -> ToolResult:
|
|
if group not in by_key:
|
|
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
|
if deliverable not in kinds:
|
|
return ToolResult(
|
|
f"Refused: deliverable must be one of {', '.join(kinds)}."
|
|
)
|
|
if any(p.evidence_key == group for p in proposed):
|
|
return ToolResult(f"Refused: {group} already has a proposal.")
|
|
text = f"{title} {rule} {reason}"
|
|
leaked = sorted(name for name in identifiers if name in text)
|
|
if leaked:
|
|
return ToolResult(
|
|
"Refused: a lesson carries no client or Execution detail; remove "
|
|
+ ", ".join(leaked)
|
|
)
|
|
if not (title.strip() and rule.strip() and reason.strip()):
|
|
return ToolResult("Refused: a lesson needs a title, a rule and a reason.")
|
|
proposal = propose(
|
|
title=title,
|
|
deliverable=deliverable,
|
|
rule=rule,
|
|
reason=reason,
|
|
evidence=_evidence(by_key[group], executions),
|
|
evidence_key=group,
|
|
folder=folder,
|
|
)
|
|
proposed.append(proposal)
|
|
return ToolResult(f"Proposed {proposal.id} for a person to review.")
|
|
|
|
async def no_lesson(group: str, why: str) -> ToolResult:
|
|
if group not in by_key:
|
|
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
|
declined[group] = why
|
|
return ToolResult(f"Noted: no lesson for {group}.")
|
|
|
|
async def finish() -> ToolResult:
|
|
return ToolResult("Done.", TurnEnded("finish"))
|
|
|
|
text = {"type": "string"}
|
|
tools = [
|
|
ToolSpec(
|
|
"propose_lesson",
|
|
"Propose one lesson for a group. A person reviews it before any agent "
|
|
"sees it.",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"group": text,
|
|
"deliverable": {"type": "string", "enum": kinds},
|
|
"title": text,
|
|
"rule": text,
|
|
"reason": text,
|
|
},
|
|
"required": ["group", "deliverable", "title", "rule", "reason"],
|
|
},
|
|
propose_lesson,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"no_lesson",
|
|
"Say that a group is not worth a lesson, and why.",
|
|
{
|
|
"type": "object",
|
|
"properties": {"group": text, "why": text},
|
|
"required": ["group", "why"],
|
|
},
|
|
no_lesson,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"finish",
|
|
"End your turn once every group has a proposal or a reason for none.",
|
|
{"type": "object", "properties": {}},
|
|
finish,
|
|
tier="execution",
|
|
),
|
|
]
|
|
context: dict[str, Any] = {
|
|
"executions": executions,
|
|
"groups": [group.as_context() for group in fresh],
|
|
"published_or_proposed": [
|
|
{
|
|
"id": p.id,
|
|
"deliverable": p.deliverable,
|
|
"rule": p.rule,
|
|
"status": p.status,
|
|
}
|
|
for p in existing
|
|
],
|
|
}
|
|
await runtime.run(
|
|
role,
|
|
AgentTask(
|
|
role=role.name,
|
|
prompt="Propose lessons from what recurred across these Executions.",
|
|
context=context,
|
|
),
|
|
tools,
|
|
)
|
|
for group, why in declined.items():
|
|
logger.info("no lesson for %s: %s", group, why)
|
|
return proposed
|