Files
orchestration_engine/execution_engine/learning/curator.py

222 lines
7.5 KiB
Python

"""The lesson curator: proposes a lesson for each recurring problem worth one.
It runs outside any Execution, on the evidence of many. It only proposes: every
proposal is written as an issue file for a person to decide (ADR-0004).
"""
from __future__ import annotations
import logging
from collections.abc import Sequence
from pathlib import Path
from typing import Any
from execution_engine.adk.runtime import (
AgentRuntime,
AgentTask,
Decider,
Question,
RoleSpec,
ToolResult,
ToolSpec,
TurnEnded,
)
from execution_engine.learning.evidence import Group
from execution_engine.learning.proposals import ISSUES, Proposal, load, propose
LESSON_CURATOR = "lesson-curator"
MAX_GROUPS = 25
logger = logging.getLogger(__name__)
async def _not_already_covered(
decider: Decider,
groups: Sequence[Group],
lessons: Sequence[Proposal],
covered_at: float,
) -> list[Group]:
"""The groups no existing lesson already covers (ADR-0007: lesson triage).
One choice per group among the lessons that exist, or none. A group a lesson
covers is not put to the curator again, so the same lesson is not proposed
twice in other words. With no decision every group is kept.
"""
live = [lesson for lesson in lessons if lesson.status != "wontfix"]
if not groups or not live:
return list(groups)
answers = await decider.decide(
"Lessons that already exist:\n"
+ "\n".join(f"- {lesson.id}: {lesson.rule}" for lesson in live),
{
group.key: Question(
"choice",
f"A problem that recurred: {group.description} (on "
f"{group.deliverable}). Which one existing lesson would have "
"prevented it? Answer none when no lesson would.",
(*(lesson.id for lesson in live), "none"),
)
for group in groups
},
scope="lesson-triage",
)
if not answers:
return list(groups)
kept = []
for group in groups:
answer = answers.get(group.key)
if (
answer is not None
and answer.value != "none"
and answer.confidence >= covered_at
):
logger.info("%s is already covered by %s", group.key, answer.value)
continue
kept.append(group)
return kept
def _evidence(group: Group, executions: int) -> str:
return (
f"Seen in {len(group.executions)} of {executions} Executions, "
f"{group.occurrences} times in all: {group.description}."
)
async def curate(
runtime: AgentRuntime,
role: RoleSpec,
groups: Sequence[Group],
*,
deliverables: Sequence[str],
executions: int,
identifiers: set[str] = frozenset(), # type: ignore[assignment]
folder: Path = ISSUES,
decider: Decider | None = None,
covered_at: float = 0.6,
) -> list[Proposal]:
"""Run the curator over the groups that have no proposal yet; return what it
proposed. `deliverables` are the Workflow's kinds, which a lesson is filed under. Groups it judged not worth a lesson are left for the next run."""
existing = load(folder)
decided = {proposal.evidence_key for proposal in existing}
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
if decider is not None and existing:
# A group about one setting is narrower than any lesson about its kind of
# problem, and it recurred with those lessons published: it goes to the
# curator, who is shown the lessons that exist and may still decline.
broad = [group for group in fresh if group.subject is None]
kept = await _not_already_covered(decider, broad, existing, covered_at)
fresh = [g for g in fresh if g.subject is not None or g in kept]
if not fresh:
return []
by_key = {group.key: group for group in fresh}
kinds = [str(kind) for kind in deliverables]
proposed: list[Proposal] = []
declined: dict[str, str] = {}
async def propose_lesson(
group: str, deliverable: str, title: str, rule: str, reason: str
) -> ToolResult:
if group not in by_key:
return ToolResult(f"Refused: {group} is not one of the groups given.")
if deliverable not in kinds:
return ToolResult(
f"Refused: deliverable must be one of {', '.join(kinds)}."
)
if any(p.evidence_key == group for p in proposed):
return ToolResult(f"Refused: {group} already has a proposal.")
text = f"{title} {rule} {reason}"
leaked = sorted(name for name in identifiers if name in text)
if leaked:
return ToolResult(
"Refused: a lesson carries no client or Execution detail; remove "
+ ", ".join(leaked)
)
if not (title.strip() and rule.strip() and reason.strip()):
return ToolResult("Refused: a lesson needs a title, a rule and a reason.")
proposal = propose(
title=title,
deliverable=deliverable,
rule=rule,
reason=reason,
evidence=_evidence(by_key[group], executions),
evidence_key=group,
folder=folder,
)
proposed.append(proposal)
return ToolResult(f"Proposed {proposal.id} for a person to review.")
async def no_lesson(group: str, why: str) -> ToolResult:
if group not in by_key:
return ToolResult(f"Refused: {group} is not one of the groups given.")
declined[group] = why
return ToolResult(f"Noted: no lesson for {group}.")
async def finish() -> ToolResult:
return ToolResult("Done.", TurnEnded("finish"))
text = {"type": "string"}
tools = [
ToolSpec(
"propose_lesson",
"Propose one lesson for a group. A person reviews it before any agent "
"sees it.",
{
"type": "object",
"properties": {
"group": text,
"deliverable": {"type": "string", "enum": kinds},
"title": text,
"rule": text,
"reason": text,
},
"required": ["group", "deliverable", "title", "rule", "reason"],
},
propose_lesson,
tier="execution",
),
ToolSpec(
"no_lesson",
"Say that a group is not worth a lesson, and why.",
{
"type": "object",
"properties": {"group": text, "why": text},
"required": ["group", "why"],
},
no_lesson,
tier="execution",
),
ToolSpec(
"finish",
"End your turn once every group has a proposal or a reason for none.",
{"type": "object", "properties": {}},
finish,
tier="execution",
),
]
context: dict[str, Any] = {
"executions": executions,
"groups": [group.as_context() for group in fresh],
"published_or_proposed": [
{
"id": p.id,
"deliverable": p.deliverable,
"rule": p.rule,
"status": p.status,
}
for p in existing
],
}
await runtime.run(
role,
AgentTask(
role=role.name,
prompt="Propose lessons from what recurred across these Executions.",
context=context,
),
tools,
)
for group, why in declined.items():
logger.info("no lesson for %s: %s", group, why)
return proposed