222 lines
7.5 KiB
Python
222 lines
7.5 KiB
Python
"""The lesson curator: proposes a lesson for each recurring problem worth one.
|
|
|
|
It runs outside any Execution, on the evidence of many. It only proposes: every
|
|
proposal is written as an issue file for a person to decide (ADR-0004).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from execution_engine.adk.runtime import (
|
|
AgentRuntime,
|
|
AgentTask,
|
|
Decider,
|
|
Question,
|
|
RoleSpec,
|
|
ToolResult,
|
|
ToolSpec,
|
|
TurnEnded,
|
|
)
|
|
from execution_engine.learning.evidence import Group
|
|
from execution_engine.learning.proposals import ISSUES, Proposal, load, propose
|
|
|
|
LESSON_CURATOR = "lesson-curator"
|
|
MAX_GROUPS = 25
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def _not_already_covered(
|
|
decider: Decider,
|
|
groups: Sequence[Group],
|
|
lessons: Sequence[Proposal],
|
|
covered_at: float,
|
|
) -> list[Group]:
|
|
"""The groups no existing lesson already covers (ADR-0007: lesson triage).
|
|
|
|
One choice per group among the lessons that exist, or none. A group a lesson
|
|
covers is not put to the curator again, so the same lesson is not proposed
|
|
twice in other words. With no decision every group is kept.
|
|
"""
|
|
live = [lesson for lesson in lessons if lesson.status != "wontfix"]
|
|
if not groups or not live:
|
|
return list(groups)
|
|
answers = await decider.decide(
|
|
"Lessons that already exist:\n"
|
|
+ "\n".join(f"- {lesson.id}: {lesson.rule}" for lesson in live),
|
|
{
|
|
group.key: Question(
|
|
"choice",
|
|
f"A problem that recurred: {group.description} (on "
|
|
f"{group.deliverable}). Which one existing lesson would have "
|
|
"prevented it? Answer none when no lesson would.",
|
|
(*(lesson.id for lesson in live), "none"),
|
|
)
|
|
for group in groups
|
|
},
|
|
scope="lesson-triage",
|
|
)
|
|
if not answers:
|
|
return list(groups)
|
|
kept = []
|
|
for group in groups:
|
|
answer = answers.get(group.key)
|
|
if (
|
|
answer is not None
|
|
and answer.value != "none"
|
|
and answer.confidence >= covered_at
|
|
):
|
|
logger.info("%s is already covered by %s", group.key, answer.value)
|
|
continue
|
|
kept.append(group)
|
|
return kept
|
|
|
|
|
|
def _evidence(group: Group, executions: int) -> str:
|
|
return (
|
|
f"Seen in {len(group.executions)} of {executions} Executions, "
|
|
f"{group.occurrences} times in all: {group.description}."
|
|
)
|
|
|
|
|
|
async def curate(
|
|
runtime: AgentRuntime,
|
|
role: RoleSpec,
|
|
groups: Sequence[Group],
|
|
*,
|
|
deliverables: Sequence[str],
|
|
executions: int,
|
|
identifiers: set[str] = frozenset(), # type: ignore[assignment]
|
|
folder: Path = ISSUES,
|
|
decider: Decider | None = None,
|
|
covered_at: float = 0.6,
|
|
) -> list[Proposal]:
|
|
"""Run the curator over the groups that have no proposal yet; return what it
|
|
proposed. `deliverables` are the Workflow's kinds, which a lesson is filed under. Groups it judged not worth a lesson are left for the next run."""
|
|
existing = load(folder)
|
|
decided = {proposal.evidence_key for proposal in existing}
|
|
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
|
|
if decider is not None and existing:
|
|
# A group about one setting is narrower than any lesson about its kind of
|
|
# problem, and it recurred with those lessons published: it goes to the
|
|
# curator, who is shown the lessons that exist and may still decline.
|
|
broad = [group for group in fresh if group.subject is None]
|
|
kept = await _not_already_covered(decider, broad, existing, covered_at)
|
|
fresh = [g for g in fresh if g.subject is not None or g in kept]
|
|
if not fresh:
|
|
return []
|
|
by_key = {group.key: group for group in fresh}
|
|
kinds = [str(kind) for kind in deliverables]
|
|
proposed: list[Proposal] = []
|
|
declined: dict[str, str] = {}
|
|
|
|
async def propose_lesson(
|
|
group: str, deliverable: str, title: str, rule: str, reason: str
|
|
) -> ToolResult:
|
|
if group not in by_key:
|
|
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
|
if deliverable not in kinds:
|
|
return ToolResult(
|
|
f"Refused: deliverable must be one of {', '.join(kinds)}."
|
|
)
|
|
if any(p.evidence_key == group for p in proposed):
|
|
return ToolResult(f"Refused: {group} already has a proposal.")
|
|
text = f"{title} {rule} {reason}"
|
|
leaked = sorted(name for name in identifiers if name in text)
|
|
if leaked:
|
|
return ToolResult(
|
|
"Refused: a lesson carries no client or Execution detail; remove "
|
|
+ ", ".join(leaked)
|
|
)
|
|
if not (title.strip() and rule.strip() and reason.strip()):
|
|
return ToolResult("Refused: a lesson needs a title, a rule and a reason.")
|
|
proposal = propose(
|
|
title=title,
|
|
deliverable=deliverable,
|
|
rule=rule,
|
|
reason=reason,
|
|
evidence=_evidence(by_key[group], executions),
|
|
evidence_key=group,
|
|
folder=folder,
|
|
)
|
|
proposed.append(proposal)
|
|
return ToolResult(f"Proposed {proposal.id} for a person to review.")
|
|
|
|
async def no_lesson(group: str, why: str) -> ToolResult:
|
|
if group not in by_key:
|
|
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
|
declined[group] = why
|
|
return ToolResult(f"Noted: no lesson for {group}.")
|
|
|
|
async def finish() -> ToolResult:
|
|
return ToolResult("Done.", TurnEnded("finish"))
|
|
|
|
text = {"type": "string"}
|
|
tools = [
|
|
ToolSpec(
|
|
"propose_lesson",
|
|
"Propose one lesson for a group. A person reviews it before any agent "
|
|
"sees it.",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"group": text,
|
|
"deliverable": {"type": "string", "enum": kinds},
|
|
"title": text,
|
|
"rule": text,
|
|
"reason": text,
|
|
},
|
|
"required": ["group", "deliverable", "title", "rule", "reason"],
|
|
},
|
|
propose_lesson,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"no_lesson",
|
|
"Say that a group is not worth a lesson, and why.",
|
|
{
|
|
"type": "object",
|
|
"properties": {"group": text, "why": text},
|
|
"required": ["group", "why"],
|
|
},
|
|
no_lesson,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"finish",
|
|
"End your turn once every group has a proposal or a reason for none.",
|
|
{"type": "object", "properties": {}},
|
|
finish,
|
|
tier="execution",
|
|
),
|
|
]
|
|
context: dict[str, Any] = {
|
|
"executions": executions,
|
|
"groups": [group.as_context() for group in fresh],
|
|
"published_or_proposed": [
|
|
{
|
|
"id": p.id,
|
|
"deliverable": p.deliverable,
|
|
"rule": p.rule,
|
|
"status": p.status,
|
|
}
|
|
for p in existing
|
|
],
|
|
}
|
|
await runtime.run(
|
|
role,
|
|
AgentTask(
|
|
role=role.name,
|
|
prompt="Propose lessons from what recurred across these Executions.",
|
|
context=context,
|
|
),
|
|
tools,
|
|
)
|
|
for group, why in declined.items():
|
|
logger.info("no lesson for %s: %s", group, why)
|
|
return proposed
|