feat: working orhcestration engine
This commit is contained in:
17
execution_engine/learning/__init__.py
Normal file
17
execution_engine/learning/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""The lessons loop (ADR-0004): what Executions got wrong becomes reviewed Skill changes.
|
||||
|
||||
capture each Execution records lesson candidates app/states/lessons.py
|
||||
distil recurring candidates are grouped, and a curator
|
||||
proposes a lesson for each group worth one evidence.py, curator.py
|
||||
review each proposal is an issue file a person decides proposals.py
|
||||
publish an approved lesson is written into the lessons
|
||||
Skill the provider bundle assigns to roles skill.py
|
||||
prove the evaluation runs with and without lessons
|
||||
retire a lesson that did not help, or that the upstream
|
||||
Skills now cover, is removed skill.py
|
||||
|
||||
Nothing here runs inside an Execution, and no lesson reaches an agent without a
|
||||
person's approval. Lesson candidates are client data: they are read from the
|
||||
Executions' records and never copied into the repository; a lesson and its issue
|
||||
file carry counts and a general description only.
|
||||
"""
|
||||
221
execution_engine/learning/curator.py
Normal file
221
execution_engine/learning/curator.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""The lesson curator: proposes a lesson for each recurring problem worth one.
|
||||
|
||||
It runs outside any Execution, on the evidence of many. It only proposes: every
|
||||
proposal is written as an issue file for a person to decide (ADR-0004).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.adk.runtime import (
|
||||
AgentRuntime,
|
||||
AgentTask,
|
||||
Decider,
|
||||
Question,
|
||||
RoleSpec,
|
||||
ToolResult,
|
||||
ToolSpec,
|
||||
TurnEnded,
|
||||
)
|
||||
from execution_engine.learning.evidence import Group
|
||||
from execution_engine.learning.proposals import ISSUES, Proposal, load, propose
|
||||
|
||||
LESSON_CURATOR = "lesson-curator"
|
||||
MAX_GROUPS = 25
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _not_already_covered(
|
||||
decider: Decider,
|
||||
groups: Sequence[Group],
|
||||
lessons: Sequence[Proposal],
|
||||
covered_at: float,
|
||||
) -> list[Group]:
|
||||
"""The groups no existing lesson already covers (ADR-0007: lesson triage).
|
||||
|
||||
One choice per group among the lessons that exist, or none. A group a lesson
|
||||
covers is not put to the curator again, so the same lesson is not proposed
|
||||
twice in other words. With no decision every group is kept.
|
||||
"""
|
||||
live = [lesson for lesson in lessons if lesson.status != "wontfix"]
|
||||
if not groups or not live:
|
||||
return list(groups)
|
||||
answers = await decider.decide(
|
||||
"Lessons that already exist:\n"
|
||||
+ "\n".join(f"- {lesson.id}: {lesson.rule}" for lesson in live),
|
||||
{
|
||||
group.key: Question(
|
||||
"choice",
|
||||
f"A problem that recurred: {group.description} (on "
|
||||
f"{group.deliverable}). Which one existing lesson would have "
|
||||
"prevented it? Answer none when no lesson would.",
|
||||
(*(lesson.id for lesson in live), "none"),
|
||||
)
|
||||
for group in groups
|
||||
},
|
||||
scope="lesson-triage",
|
||||
)
|
||||
if not answers:
|
||||
return list(groups)
|
||||
kept = []
|
||||
for group in groups:
|
||||
answer = answers.get(group.key)
|
||||
if (
|
||||
answer is not None
|
||||
and answer.value != "none"
|
||||
and answer.confidence >= covered_at
|
||||
):
|
||||
logger.info("%s is already covered by %s", group.key, answer.value)
|
||||
continue
|
||||
kept.append(group)
|
||||
return kept
|
||||
|
||||
|
||||
def _evidence(group: Group, executions: int) -> str:
|
||||
return (
|
||||
f"Seen in {len(group.executions)} of {executions} Executions, "
|
||||
f"{group.occurrences} times in all: {group.description}."
|
||||
)
|
||||
|
||||
|
||||
async def curate(
|
||||
runtime: AgentRuntime,
|
||||
role: RoleSpec,
|
||||
groups: Sequence[Group],
|
||||
*,
|
||||
deliverables: Sequence[str],
|
||||
executions: int,
|
||||
identifiers: set[str] = frozenset(), # type: ignore[assignment]
|
||||
folder: Path = ISSUES,
|
||||
decider: Decider | None = None,
|
||||
covered_at: float = 0.6,
|
||||
) -> list[Proposal]:
|
||||
"""Run the curator over the groups that have no proposal yet; return what it
|
||||
proposed. `deliverables` are the Workflow's kinds, which a lesson is filed under. Groups it judged not worth a lesson are left for the next run."""
|
||||
existing = load(folder)
|
||||
decided = {proposal.evidence_key for proposal in existing}
|
||||
fresh = [group for group in groups if group.key not in decided][:MAX_GROUPS]
|
||||
if decider is not None and existing:
|
||||
# A group about one setting is narrower than any lesson about its kind of
|
||||
# problem, and it recurred with those lessons published: it goes to the
|
||||
# curator, who is shown the lessons that exist and may still decline.
|
||||
broad = [group for group in fresh if group.subject is None]
|
||||
kept = await _not_already_covered(decider, broad, existing, covered_at)
|
||||
fresh = [g for g in fresh if g.subject is not None or g in kept]
|
||||
if not fresh:
|
||||
return []
|
||||
by_key = {group.key: group for group in fresh}
|
||||
kinds = [str(kind) for kind in deliverables]
|
||||
proposed: list[Proposal] = []
|
||||
declined: dict[str, str] = {}
|
||||
|
||||
async def propose_lesson(
|
||||
group: str, deliverable: str, title: str, rule: str, reason: str
|
||||
) -> ToolResult:
|
||||
if group not in by_key:
|
||||
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
||||
if deliverable not in kinds:
|
||||
return ToolResult(
|
||||
f"Refused: deliverable must be one of {', '.join(kinds)}."
|
||||
)
|
||||
if any(p.evidence_key == group for p in proposed):
|
||||
return ToolResult(f"Refused: {group} already has a proposal.")
|
||||
text = f"{title} {rule} {reason}"
|
||||
leaked = sorted(name for name in identifiers if name in text)
|
||||
if leaked:
|
||||
return ToolResult(
|
||||
"Refused: a lesson carries no client or Execution detail; remove "
|
||||
+ ", ".join(leaked)
|
||||
)
|
||||
if not (title.strip() and rule.strip() and reason.strip()):
|
||||
return ToolResult("Refused: a lesson needs a title, a rule and a reason.")
|
||||
proposal = propose(
|
||||
title=title,
|
||||
deliverable=deliverable,
|
||||
rule=rule,
|
||||
reason=reason,
|
||||
evidence=_evidence(by_key[group], executions),
|
||||
evidence_key=group,
|
||||
folder=folder,
|
||||
)
|
||||
proposed.append(proposal)
|
||||
return ToolResult(f"Proposed {proposal.id} for a person to review.")
|
||||
|
||||
async def no_lesson(group: str, why: str) -> ToolResult:
|
||||
if group not in by_key:
|
||||
return ToolResult(f"Refused: {group} is not one of the groups given.")
|
||||
declined[group] = why
|
||||
return ToolResult(f"Noted: no lesson for {group}.")
|
||||
|
||||
async def finish() -> ToolResult:
|
||||
return ToolResult("Done.", TurnEnded("finish"))
|
||||
|
||||
text = {"type": "string"}
|
||||
tools = [
|
||||
ToolSpec(
|
||||
"propose_lesson",
|
||||
"Propose one lesson for a group. A person reviews it before any agent "
|
||||
"sees it.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"group": text,
|
||||
"deliverable": {"type": "string", "enum": kinds},
|
||||
"title": text,
|
||||
"rule": text,
|
||||
"reason": text,
|
||||
},
|
||||
"required": ["group", "deliverable", "title", "rule", "reason"],
|
||||
},
|
||||
propose_lesson,
|
||||
tier="execution",
|
||||
),
|
||||
ToolSpec(
|
||||
"no_lesson",
|
||||
"Say that a group is not worth a lesson, and why.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"group": text, "why": text},
|
||||
"required": ["group", "why"],
|
||||
},
|
||||
no_lesson,
|
||||
tier="execution",
|
||||
),
|
||||
ToolSpec(
|
||||
"finish",
|
||||
"End your turn once every group has a proposal or a reason for none.",
|
||||
{"type": "object", "properties": {}},
|
||||
finish,
|
||||
tier="execution",
|
||||
),
|
||||
]
|
||||
context: dict[str, Any] = {
|
||||
"executions": executions,
|
||||
"groups": [group.as_context() for group in fresh],
|
||||
"published_or_proposed": [
|
||||
{
|
||||
"id": p.id,
|
||||
"deliverable": p.deliverable,
|
||||
"rule": p.rule,
|
||||
"status": p.status,
|
||||
}
|
||||
for p in existing
|
||||
],
|
||||
}
|
||||
await runtime.run(
|
||||
role,
|
||||
AgentTask(
|
||||
role=role.name,
|
||||
prompt="Propose lessons from what recurred across these Executions.",
|
||||
context=context,
|
||||
),
|
||||
tools,
|
||||
)
|
||||
for group, why in declined.items():
|
||||
logger.info("no lesson for %s: %s", group, why)
|
||||
return proposed
|
||||
216
execution_engine/learning/evidence.py
Normal file
216
execution_engine/learning/evidence.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""Lesson candidates across Executions, grouped into what recurs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
EXAMPLES = 3
|
||||
EXAMPLE_LIMIT = 400
|
||||
# Ids and numbers vary between Executions; what recurs is the shape around them.
|
||||
_VARIABLE = re.compile(r"`[^`]*`|\"[^\"]*\"|\b[A-Z]{1,4}-[\w.-]+\b|\b\d[\w.]*\b")
|
||||
# A refusal the Execution's own permissions caused is policy, not a mistake.
|
||||
_POLICY = ("Refused: reading live cloud resources", "Permission to read live cloud")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Group:
|
||||
"""One thing that went wrong the same way in several Executions."""
|
||||
|
||||
key: str # stable across runs of the curator: the same evidence, the same key
|
||||
kind: str # refusal | revision | proposed_addition | failed_check | ...
|
||||
deliverable: str | None
|
||||
description: str # general: no id, name or value from any Execution
|
||||
executions: set[str] = field(default_factory=set)
|
||||
occurrences: int = 0
|
||||
examples: list[str] = field(default_factory=list) # client data: never published
|
||||
# The provider setting or resource type the problem was about, for a group
|
||||
# narrowed to one subject. None for a group of every problem of its kind.
|
||||
subject: str | None = None
|
||||
|
||||
def as_context(self) -> dict[str, Any]:
|
||||
return {
|
||||
"group": self.key,
|
||||
"kind": self.kind,
|
||||
"deliverable": self.deliverable,
|
||||
"what_happened": self.description,
|
||||
"executions": len(self.executions),
|
||||
"occurrences": self.occurrences,
|
||||
"examples": self.examples,
|
||||
}
|
||||
|
||||
|
||||
def read_candidates(roots: Iterable[Path]) -> list[dict[str, Any]]:
|
||||
"""Every Execution's lesson-candidate record under the given folders."""
|
||||
records = []
|
||||
seen: set[Path] = set()
|
||||
for root in roots:
|
||||
for path in sorted(Path(root).rglob("lessons/candidates/*/*.json")):
|
||||
# One folder may lie inside another that was also given.
|
||||
if path.resolve() in seen:
|
||||
continue
|
||||
seen.add(path.resolve())
|
||||
try:
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(record, dict) and isinstance(record.get("candidates"), list):
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
# A subject shorter than this is a stray word, not a setting.
|
||||
_SUBJECT_MIN = 4
|
||||
|
||||
|
||||
def _general(text: str, limit: int = 110) -> str:
|
||||
return " ".join(_VARIABLE.sub("…", text).split())[:limit]
|
||||
|
||||
|
||||
def _shape(candidate: Mapping[str, Any]) -> tuple[str, str | None, str] | None:
|
||||
"""(kind, Deliverable, general description) of a candidate, or None to ignore it."""
|
||||
kind = str(candidate.get("type"))
|
||||
deliverable = candidate.get("deliverable")
|
||||
if kind == "refusal":
|
||||
message = str(candidate.get("message", ""))
|
||||
if message.strip() == "Refused" or message.startswith(_POLICY):
|
||||
return None
|
||||
tool = candidate.get("tool")
|
||||
return kind, deliverable, f"{tool} refused: {_general(message)}"
|
||||
if kind in ("revision", "revision_limit"):
|
||||
reason = str(candidate.get("reason", ""))
|
||||
if reason.startswith("Q: ") or "revision limit" in reason:
|
||||
return None # the router's own questions, recorded as reasons
|
||||
if reason.startswith("Add a Check"):
|
||||
why = "validation proposed a Check the design lacked"
|
||||
elif "cannot be implemented" in reason:
|
||||
why = "a Check named something its resource cannot do"
|
||||
elif " failed" in reason[:60]:
|
||||
why = "the code failed a Check"
|
||||
else:
|
||||
why = _general(reason, 70)
|
||||
return "revision", deliverable, f"revised because {why}"
|
||||
if kind == "proposed_addition":
|
||||
return (
|
||||
kind,
|
||||
"target_architecture",
|
||||
"validation proposed a Check the design lacked",
|
||||
)
|
||||
if kind in ("failed_check", "unverifiable_check"):
|
||||
what = "failed" if kind == "failed_check" else "could not be verified"
|
||||
return kind, "deployment_guidance", f"a Check {what} at validation"
|
||||
if kind == "unfinished":
|
||||
return kind, deliverable, f"the Specialist {candidate.get('reason')}"
|
||||
if kind == "change_request":
|
||||
return kind, None, "the user asked for changes at an Approval"
|
||||
return None
|
||||
|
||||
|
||||
def _subjects(
|
||||
candidate: Mapping[str, Any], named: Callable[[str], Iterable[str]] | None
|
||||
) -> tuple[str, list[str]] | None:
|
||||
"""For a problem about what a Check says: (how to describe it, the settings and
|
||||
resource types its text names).
|
||||
|
||||
Problems of one kind are one group, and one group gets one lesson. "Validation
|
||||
proposed a Check" was 20 of 32 Executions and a single lesson; what recurred
|
||||
inside it, such as the same setting missed again and again, was invisible. A
|
||||
setting's name is the provider's vocabulary, not a client's, so it can be the
|
||||
subject of a narrower group.
|
||||
"""
|
||||
kind = str(candidate.get("type"))
|
||||
if kind == "proposed_addition":
|
||||
text = str(candidate.get("statement", ""))
|
||||
what = "validation proposed a Check about `{}` that the design lacked"
|
||||
elif kind == "revision" and "cannot be implemented" in str(
|
||||
candidate.get("reason", "")
|
||||
):
|
||||
text = str(candidate.get("reason", ""))
|
||||
what = "a Check about `{}` asked for something that resource cannot do"
|
||||
else:
|
||||
return None
|
||||
if named is None:
|
||||
return None
|
||||
names = {name.split(".")[-1] for name in named(text)}
|
||||
return what, sorted(name for name in names if len(name) >= _SUBJECT_MIN)
|
||||
|
||||
|
||||
def group_candidates(
|
||||
records: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
min_executions: int = 2,
|
||||
named: Callable[[str], Iterable[str]] | None = None,
|
||||
) -> list[Group]:
|
||||
"""What recurred in at least `min_executions` Executions, most widespread first.
|
||||
|
||||
`named` finds, in a Check's text, the names in a Workflow's own vocabulary that
|
||||
can be the subject of a narrower group, such as a provider's settings. Without
|
||||
it, problems are grouped by kind alone.
|
||||
"""
|
||||
groups: dict[str, Group] = {}
|
||||
for record in records:
|
||||
execution = str(record.get("execution_id"))
|
||||
for candidate in record.get("candidates", ()):
|
||||
if not isinstance(candidate, Mapping):
|
||||
continue
|
||||
shape = _shape(candidate)
|
||||
if shape is None:
|
||||
continue
|
||||
kind, deliverable, description = shape
|
||||
key = hashlib.sha256(
|
||||
f"{kind}|{deliverable}|{description}".encode()
|
||||
).hexdigest()[:10]
|
||||
group = groups.setdefault(key, Group(key, kind, deliverable, description))
|
||||
group.executions.add(execution)
|
||||
group.occurrences += 1
|
||||
example = str(
|
||||
candidate.get("message")
|
||||
or candidate.get("reason")
|
||||
or candidate.get("statement")
|
||||
or candidate.get("evidence")
|
||||
or candidate.get("feedback")
|
||||
or ""
|
||||
)[:EXAMPLE_LIMIT]
|
||||
if (
|
||||
example
|
||||
and example not in group.examples
|
||||
and len(group.examples) < EXAMPLES
|
||||
):
|
||||
group.examples.append(example)
|
||||
narrowed = _subjects(candidate, named)
|
||||
for subject in narrowed[1] if narrowed else ():
|
||||
about = narrowed[0].format(subject) # type: ignore[index]
|
||||
subkey = hashlib.sha256(
|
||||
f"{kind}|{deliverable}|{about}".encode()
|
||||
).hexdigest()[:10]
|
||||
narrow = groups.setdefault(
|
||||
subkey, Group(subkey, kind, deliverable, about, subject=subject)
|
||||
)
|
||||
narrow.executions.add(execution)
|
||||
narrow.occurrences += 1
|
||||
if (
|
||||
example
|
||||
and example not in narrow.examples
|
||||
and len(narrow.examples) < EXAMPLES
|
||||
):
|
||||
narrow.examples.append(example)
|
||||
return sorted(
|
||||
(g for g in groups.values() if len(g.executions) >= min_executions),
|
||||
key=lambda g: (-len(g.executions), -g.occurrences, g.key),
|
||||
)
|
||||
|
||||
|
||||
def identifiers(records: Sequence[Mapping[str, Any]]) -> set[str]:
|
||||
"""Names that identify a client or an Execution, which no lesson may carry."""
|
||||
found = set()
|
||||
for record in records:
|
||||
for name in ("execution_id", "customer_case"):
|
||||
value = record.get(name)
|
||||
if isinstance(value, str) and len(value) >= 4 and value != "unassigned":
|
||||
found.add(value)
|
||||
return found
|
||||
117
execution_engine/learning/proof.py
Normal file
117
execution_engine/learning/proof.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""Proving lessons: the evaluation with them against the evaluation without (ADR-0004).
|
||||
|
||||
A lesson is kept only if quality holds and Revisions, time or tokens fall. Both runs
|
||||
are ordinary evaluation reports; one is made with LESSONS_ENABLED=false.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Collection, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.learning.proposals import Proposal, write
|
||||
|
||||
METRICS = ("score", "revisions", "seconds", "tokens")
|
||||
_CAVEAT = (
|
||||
"This compares every published lesson together, on one run each way. Runs of "
|
||||
"the same case vary, so one pair suggests a direction and does not settle it."
|
||||
)
|
||||
|
||||
|
||||
def _measures(report: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]:
|
||||
cases = {}
|
||||
for item in report:
|
||||
details = item.get("details") or {}
|
||||
tokens = ((details.get("tokens") or {}).get("total") or {}).get("total") or 0
|
||||
cases[str(item.get("case_id"))] = {
|
||||
"score": float(item.get("score") or 0),
|
||||
"revisions": float(sum((details.get("revisions") or {}).values())),
|
||||
"seconds": float(details.get("seconds") or 0),
|
||||
"tokens": float(tokens),
|
||||
}
|
||||
return cases
|
||||
|
||||
|
||||
def compare(without: Path, with_lessons: Path) -> dict[str, Any]:
|
||||
"""The two reports side by side, for the cases both ran, and whether the
|
||||
lessons helped: no case scored lower, and Revisions, time or tokens fell."""
|
||||
before = _measures(json.loads(without.read_text(encoding="utf-8")))
|
||||
after = _measures(json.loads(with_lessons.read_text(encoding="utf-8")))
|
||||
cases = sorted(set(before) & set(after))
|
||||
totals = {
|
||||
metric: (
|
||||
sum(before[case][metric] for case in cases),
|
||||
sum(after[case][metric] for case in cases),
|
||||
)
|
||||
for metric in METRICS
|
||||
}
|
||||
quality_holds = bool(cases) and all(
|
||||
after[case]["score"] >= before[case]["score"] for case in cases
|
||||
)
|
||||
lower = [
|
||||
metric
|
||||
for metric in ("revisions", "seconds", "tokens")
|
||||
if totals[metric][1] < totals[metric][0]
|
||||
]
|
||||
return {
|
||||
"cases": {
|
||||
case: {"without": before[case], "with": after[case]} for case in cases
|
||||
},
|
||||
"totals": {
|
||||
metric: {"without": a, "with": b} for metric, (a, b) in totals.items()
|
||||
},
|
||||
"quality_holds": quality_holds,
|
||||
"lower": lower,
|
||||
"helps": quality_holds and bool(lower),
|
||||
}
|
||||
|
||||
|
||||
def render(result: Mapping[str, Any], without: Path, with_lessons: Path) -> str:
|
||||
"""The comparison as the text of a proposal's Proof section."""
|
||||
header = (
|
||||
f"Evaluation without lessons: `{without.parent.name}`; with: "
|
||||
f"`{with_lessons.parent.name}`."
|
||||
)
|
||||
lines = [
|
||||
header,
|
||||
"",
|
||||
"| Case | Score | Revisions | Minutes | Tokens (M) |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for case, pair in result["cases"].items():
|
||||
a, b = pair["without"], pair["with"]
|
||||
lines.append(
|
||||
f"| {case} | {a['score']:.2f} → {b['score']:.2f} | "
|
||||
f"{a['revisions']:.0f} → {b['revisions']:.0f} | "
|
||||
f"{a['seconds'] / 60:.0f} → {b['seconds'] / 60:.0f} | "
|
||||
f"{a['tokens'] / 1e6:.2f} → {b['tokens'] / 1e6:.2f} |"
|
||||
)
|
||||
if not result["cases"]:
|
||||
verdict = "The two runs share no case, so nothing is shown."
|
||||
elif result["helps"]:
|
||||
verdict = f"Quality held and {', '.join(result['lower'])} fell: kept."
|
||||
elif not result["quality_holds"]:
|
||||
verdict = "A case scored lower with the lessons: not shown to help."
|
||||
else:
|
||||
verdict = "Quality held, and nothing fell: not shown to help."
|
||||
lines += [
|
||||
"",
|
||||
verdict,
|
||||
"",
|
||||
_CAVEAT,
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def record(
|
||||
proposals: Sequence[Proposal], proof: str, live: Collection[str]
|
||||
) -> list[Proposal]:
|
||||
"""Write the comparison into the proposal of every lesson in the Skill.
|
||||
|
||||
`live` is the ids the Skill holds now. A retired lesson was not part of what
|
||||
was compared, so its proposal is left as it was.
|
||||
"""
|
||||
return [write(replace(p, proof=proof)) for p in proposals if p.id in live]
|
||||
173
execution_engine/learning/proposals.py
Normal file
173
execution_engine/learning/proposals.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""Lesson proposals as issue files: where a person reviews each one.
|
||||
|
||||
.scratch/lessons/issues/NN-<slug>.md
|
||||
|
||||
A proposal starts as `needs-triage`. A person approves it by setting its Status to
|
||||
`ready-for-agent`, or rejects it with `wontfix`. Only an approved proposal can be
|
||||
published. The file carries the rule, its reason and how widespread the evidence
|
||||
was, and nothing that identifies a client or an Execution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import UTC, date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
NEEDS_TRIAGE = "needs-triage"
|
||||
APPROVED = "ready-for-agent"
|
||||
REJECTED = "wontfix"
|
||||
STATUSES = (NEEDS_TRIAGE, "needs-info", APPROVED, "ready-for-human", REJECTED)
|
||||
ISSUES = Path(".scratch") / "lessons" / "issues"
|
||||
|
||||
_FIELD = re.compile(r"^([A-Z][\w-]*):\s*(.*)$", re.MULTILINE)
|
||||
_SECTION = re.compile(r"^## (.+?)\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Proposal:
|
||||
number: int
|
||||
title: str
|
||||
deliverable: str
|
||||
rule: str
|
||||
reason: str
|
||||
evidence: str # counts and a general description
|
||||
evidence_key: str
|
||||
status: str = NEEDS_TRIAGE
|
||||
proposed: str = ""
|
||||
published: str = ""
|
||||
# The day it was taken out of the Skill again. A retired lesson is not
|
||||
# published a second time, whatever its Status still says.
|
||||
retired: str = ""
|
||||
proof: str = ""
|
||||
comments: str = ""
|
||||
path: Path | None = None
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return f"L-{self.number:02d}"
|
||||
|
||||
@property
|
||||
def approved(self) -> bool:
|
||||
return self.status == APPROVED and not self.retired
|
||||
|
||||
|
||||
def _slug(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")[:60] or "lesson"
|
||||
|
||||
|
||||
def render(proposal: Proposal) -> str:
|
||||
lines = [
|
||||
f"# {proposal.id}: {proposal.title}",
|
||||
"",
|
||||
f"Status: {proposal.status}",
|
||||
f"Deliverable: {proposal.deliverable}",
|
||||
f"Evidence-key: {proposal.evidence_key}",
|
||||
f"Proposed: {proposal.proposed}",
|
||||
]
|
||||
if proposal.published:
|
||||
lines.append(f"Published: {proposal.published}")
|
||||
if proposal.retired:
|
||||
lines.append(f"Retired: {proposal.retired}")
|
||||
lines += [
|
||||
"",
|
||||
"## Rule",
|
||||
"",
|
||||
proposal.rule.strip(),
|
||||
"",
|
||||
"## Why",
|
||||
"",
|
||||
proposal.reason.strip(),
|
||||
"",
|
||||
"## Evidence",
|
||||
"",
|
||||
proposal.evidence.strip(),
|
||||
"",
|
||||
"## Proof",
|
||||
"",
|
||||
proposal.proof.strip() or "Not yet measured.",
|
||||
"",
|
||||
"## Comments",
|
||||
"",
|
||||
proposal.comments.strip(),
|
||||
]
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def parse(path: Path) -> Proposal:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
heading = re.match(r"# L-(\d+): (.+)", text)
|
||||
if not heading:
|
||||
raise ValueError(f"{path}: not a lesson proposal")
|
||||
head = text.split("\n## ", 1)[0]
|
||||
fields = {name: value.strip() for name, value in _FIELD.findall(head)}
|
||||
sections: dict[str, str] = {}
|
||||
matches = list(_SECTION.finditer(text))
|
||||
for index, match in enumerate(matches):
|
||||
end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
|
||||
sections[match.group(1)] = text[match.end() : end].strip()
|
||||
status = fields.get("Status", NEEDS_TRIAGE)
|
||||
if status not in STATUSES:
|
||||
raise ValueError(f"{path}: Status must be one of {', '.join(STATUSES)}")
|
||||
proof = sections.get("Proof", "")
|
||||
return Proposal(
|
||||
number=int(heading.group(1)),
|
||||
title=heading.group(2).strip(),
|
||||
deliverable=fields.get("Deliverable", ""),
|
||||
rule=sections.get("Rule", ""),
|
||||
reason=sections.get("Why", ""),
|
||||
evidence=sections.get("Evidence", ""),
|
||||
evidence_key=fields.get("Evidence-key", ""),
|
||||
status=status,
|
||||
proposed=fields.get("Proposed", ""),
|
||||
published=fields.get("Published", ""),
|
||||
retired=fields.get("Retired", ""),
|
||||
proof="" if proof == "Not yet measured." else proof,
|
||||
comments=sections.get("Comments", ""),
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def load(folder: Path = ISSUES) -> list[Proposal]:
|
||||
if not folder.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
(parse(path) for path in folder.glob("[0-9]*-*.md")), key=lambda p: p.number
|
||||
)
|
||||
|
||||
|
||||
def write(proposal: Proposal, folder: Path = ISSUES) -> Proposal:
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
path = proposal.path or folder / f"{proposal.number:02d}-{_slug(proposal.title)}.md"
|
||||
path.write_text(render(proposal), encoding="utf-8")
|
||||
return replace(proposal, path=path)
|
||||
|
||||
|
||||
def propose(
|
||||
*,
|
||||
title: str,
|
||||
deliverable: str,
|
||||
rule: str,
|
||||
reason: str,
|
||||
evidence: str,
|
||||
evidence_key: str,
|
||||
folder: Path = ISSUES,
|
||||
today: date | None = None,
|
||||
) -> Proposal:
|
||||
"""A new proposal, numbered after the last, awaiting a person's decision."""
|
||||
existing = load(folder)
|
||||
number = max((p.number for p in existing), default=0) + 1
|
||||
return write(
|
||||
Proposal(
|
||||
number=number,
|
||||
title=title.strip(),
|
||||
deliverable=deliverable,
|
||||
rule=rule,
|
||||
reason=reason,
|
||||
evidence=evidence,
|
||||
evidence_key=evidence_key,
|
||||
proposed=(today or datetime.now(UTC).date()).isoformat(),
|
||||
),
|
||||
folder,
|
||||
)
|
||||
185
execution_engine/learning/skill.py
Normal file
185
execution_engine/learning/skill.py
Normal file
@@ -0,0 +1,185 @@
|
||||
"""The lessons Skill: where approved lessons are published, and retired from.
|
||||
|
||||
A lessons Skill is an ordinary Agent Skill, per Cloud provider, that the provider
|
||||
bundle names. It has one reference file per Deliverable that has lessons, so a
|
||||
Specialist reads only the lessons for its own work. Both the reference files and
|
||||
the index in SKILL.md are written here, from approved proposals, and never by hand.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from execution_engine.learning.proposals import Proposal, write
|
||||
|
||||
REFERENCES = "references"
|
||||
_INDEX = re.compile(r"<!-- lessons:begin -->.*<!-- lessons:end -->", re.DOTALL)
|
||||
_ENTRY = re.compile(r"^## (L-\d+): .*?(?=^## L-\d+: |\Z)", re.MULTILINE | re.DOTALL)
|
||||
|
||||
|
||||
class NotApproved(ValueError):
|
||||
"""A lesson cannot be published: no person has approved its proposal."""
|
||||
|
||||
|
||||
def reference(skill_dir: Path, deliverable: str) -> Path:
|
||||
return skill_dir / REFERENCES / f"{deliverable}.md"
|
||||
|
||||
|
||||
def deliverables_with_lessons(skill_dir: Path) -> list[str]:
|
||||
folder = skill_dir / REFERENCES
|
||||
if not folder.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
path.stem
|
||||
for path in folder.glob("*.md")
|
||||
if _ENTRY.search(path.read_text(encoding="utf-8"))
|
||||
)
|
||||
|
||||
|
||||
def published(skill_dir: Path) -> dict[str, str]:
|
||||
"""Lesson id -> the Deliverable whose reference holds it."""
|
||||
found = {}
|
||||
for deliverable in deliverables_with_lessons(skill_dir):
|
||||
text = reference(skill_dir, deliverable).read_text(encoding="utf-8")
|
||||
for match in _ENTRY.finditer(text):
|
||||
found[match.group(1)] = deliverable
|
||||
return found
|
||||
|
||||
|
||||
def _entry(proposal: Proposal, today: str) -> str:
|
||||
return (
|
||||
f"## {proposal.id}: {proposal.title}\n\n"
|
||||
f"**Rule.** {' '.join(proposal.rule.split())}\n\n"
|
||||
f"**Why.** {' '.join(proposal.reason.split())}\n\n"
|
||||
f"Published {today}, from lesson proposal {proposal.number:02d}.\n"
|
||||
)
|
||||
|
||||
|
||||
def _write_reference(skill_dir: Path, deliverable: str, entries: list[str]) -> None:
|
||||
path = reference(skill_dir, deliverable)
|
||||
if not entries:
|
||||
path.unlink(missing_ok=True)
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
title = deliverable.replace("_", " ")
|
||||
path.write_text(
|
||||
f"# Lessons: {title}\n\n"
|
||||
"Rules learned from earlier work. Follow every one that applies.\n\n"
|
||||
+ "\n".join(entries),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _entries(skill_dir: Path, deliverable: str) -> dict[str, str]:
|
||||
path = reference(skill_dir, deliverable)
|
||||
if not path.is_file():
|
||||
return {}
|
||||
text = path.read_text(encoding="utf-8")
|
||||
return {m.group(1): m.group(0).rstrip() + "\n" for m in _ENTRY.finditer(text)}
|
||||
|
||||
|
||||
def _reindex(skill_dir: Path) -> None:
|
||||
"""Rewrite the list of references in SKILL.md from the files that exist."""
|
||||
skill = skill_dir / "SKILL.md"
|
||||
rows = [
|
||||
f"| {name.replace('_', ' ')} | `{REFERENCES}/{name}.md` | "
|
||||
f"{len(_entries(skill_dir, name))} |"
|
||||
for name in deliverables_with_lessons(skill_dir)
|
||||
]
|
||||
table = (
|
||||
"| Deliverable | Reference | Lessons |\n| --- | --- | --- |\n" + "\n".join(rows)
|
||||
if rows
|
||||
else "No lessons have been published yet."
|
||||
)
|
||||
text = skill.read_text(encoding="utf-8")
|
||||
skill.write_text(
|
||||
_INDEX.sub(f"<!-- lessons:begin -->\n{table}\n<!-- lessons:end -->", text),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def publish(
|
||||
proposal: Proposal, skill_dir: Path, *, today: date | None = None
|
||||
) -> Proposal:
|
||||
"""Write an approved lesson into the Skill and mark its proposal published.
|
||||
|
||||
Raises NotApproved for any proposal a person has not approved: nothing reaches
|
||||
an agent on a curator's say-so.
|
||||
"""
|
||||
if proposal.retired:
|
||||
raise NotApproved(f"{proposal.id} was retired on {proposal.retired}")
|
||||
if not proposal.approved:
|
||||
raise NotApproved(
|
||||
f"{proposal.id} is {proposal.status}; a person approves a lesson by "
|
||||
"setting its Status to ready-for-agent"
|
||||
)
|
||||
if not proposal.deliverable or not proposal.rule.strip():
|
||||
raise NotApproved(f"{proposal.id} has no Deliverable or no rule")
|
||||
stamp = (today or datetime.now(UTC).date()).isoformat()
|
||||
for deliverable in deliverables_with_lessons(skill_dir):
|
||||
if deliverable != proposal.deliverable:
|
||||
entries = _entries(skill_dir, deliverable)
|
||||
if entries.pop(proposal.id, None): # it moved to another Deliverable
|
||||
_write_reference(skill_dir, deliverable, list(entries.values()))
|
||||
entries = _entries(skill_dir, proposal.deliverable)
|
||||
entries[proposal.id] = _entry(proposal, stamp)
|
||||
ordered = [entries[key] for key in sorted(entries)]
|
||||
_write_reference(skill_dir, proposal.deliverable, ordered)
|
||||
_reindex(skill_dir)
|
||||
return write(replace(proposal, published=proposal.published or stamp))
|
||||
|
||||
|
||||
# The versions of the Skills the lessons were written beside, kept in the lessons
|
||||
# Skill. A lesson is a correction to how agents follow those Skills, so when one
|
||||
# of them changes each lesson is worth a second look: it may now be said upstream,
|
||||
# or contradict it.
|
||||
UPSTREAM_FILE = "upstream.json"
|
||||
|
||||
|
||||
def upstream_reviewed(skill_dir: Path) -> dict[str, str]:
|
||||
"""The upstream Skill versions the lessons were last reviewed against."""
|
||||
path = skill_dir / UPSTREAM_FILE
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except ValueError:
|
||||
return {}
|
||||
return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def mark_reviewed(skill_dir: Path, current: Mapping[str, str]) -> None:
|
||||
"""Record that the lessons stand against these upstream Skill versions."""
|
||||
(skill_dir / UPSTREAM_FILE).write_text(
|
||||
json.dumps(dict(sorted(current.items())), indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def upstream_changes(skill_dir: Path, current: Mapping[str, str]) -> list[str]:
|
||||
"""The upstream Skills that changed, appeared or went since the lessons were
|
||||
last reviewed. Empty when there are no lessons, or nothing was ever recorded."""
|
||||
reviewed = upstream_reviewed(skill_dir)
|
||||
if not reviewed or not published(skill_dir):
|
||||
return []
|
||||
return sorted(
|
||||
name
|
||||
for name in set(reviewed) | set(current)
|
||||
if reviewed.get(name) != current.get(name)
|
||||
)
|
||||
|
||||
|
||||
def retire(lesson_id: str, skill_dir: Path) -> bool:
|
||||
"""Remove a lesson from the Skill. Returns whether it was there."""
|
||||
deliverable = published(skill_dir).get(lesson_id)
|
||||
if deliverable is None:
|
||||
return False
|
||||
entries = _entries(skill_dir, deliverable)
|
||||
entries.pop(lesson_id)
|
||||
_write_reference(skill_dir, deliverable, [entries[key] for key in sorted(entries)])
|
||||
_reindex(skill_dir)
|
||||
return True
|
||||
Reference in New Issue
Block a user