Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
88 lines
3.3 KiB
Python
88 lines
3.3 KiB
Python
"""Which references to read first (ADR-0007, iteration 5 step 8).
|
|
|
|
A Skill with many references leaves its reader to find the ones a task needs. For a
|
|
Specialist whose Skills hold enough of them, this asks the decider, one graded
|
|
question per reference, how much the task needs it, and names the ones it needs in
|
|
the Specialist's context as `read_first`. It is a place to start, not a limit: the
|
|
Specialist still loads whatever else it finds it needs.
|
|
|
|
Without a decision nothing is named and the Specialist searches its Skills itself.
|
|
|
|
Measured against live Jev on 2026-10-09, with the 14 infrastructure-code patterns and
|
|
four tasks: every reference a task's Elements call for scored 1.39 to 1.92, and every
|
|
one they do not, 0.01 to 0.96, except the Artifact Registry pattern for two designs
|
|
that run containers (1.28 and 1.38). At 1.25 nothing needed is missed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Mapping
|
|
from pathlib import Path
|
|
|
|
from app.adk.runtime import Decider, Question
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
NEEDED = ("not needed", "might help", "needed")
|
|
# A Skill with fewer references than this is quicker read than asked about.
|
|
MIN_REFERENCES = 4
|
|
_TASK_LIMIT = 12_000
|
|
_SUMMARY_LIMIT = 300
|
|
|
|
|
|
def references_of(skill_path: Path) -> dict[str, str]:
|
|
"""Each reference file of a Skill with what it says it covers: its title and its
|
|
first paragraph."""
|
|
described: dict[str, str] = {}
|
|
for file in sorted((skill_path / "references").glob("*.md")):
|
|
lines = [line.strip() for line in file.read_text().splitlines()]
|
|
title = next((line.lstrip("# ") for line in lines if line.startswith("#")), "")
|
|
first = next(
|
|
(line for line in lines if line and not line.startswith(("#", "-", "|"))),
|
|
"",
|
|
)
|
|
described[f"references/{file.name}"] = (
|
|
f"{title}. {first}".strip(". ")[:_SUMMARY_LIMIT] or file.stem
|
|
)
|
|
return described
|
|
|
|
|
|
def routable(skills: Mapping[str, Path]) -> dict[str, dict[str, str]]:
|
|
"""The Skills worth routing within: those with enough references to choose from."""
|
|
found = {name: references_of(path) for name, path in skills.items()}
|
|
return {name: refs for name, refs in found.items() if len(refs) >= MIN_REFERENCES}
|
|
|
|
|
|
async def read_first(
|
|
decider: Decider,
|
|
task: str,
|
|
references: Mapping[str, Mapping[str, str]],
|
|
*,
|
|
threshold: float = 1.25,
|
|
scope: str = "",
|
|
) -> list[dict[str, str]] | None:
|
|
"""The references the task needs, as `{"skill", "resource"}`. An empty list means
|
|
none stood out. None means no decision was available."""
|
|
asked = {
|
|
f"{skill}/{resource}": Question(
|
|
"score",
|
|
f"How much does this task need a reference on: {covers}?",
|
|
NEEDED,
|
|
)
|
|
for skill, resources in references.items()
|
|
for resource, covers in resources.items()
|
|
}
|
|
if not task.strip() or not asked:
|
|
return None
|
|
answers = await decider.decide(task[:_TASK_LIMIT], asked, scope=scope)
|
|
if answers is None:
|
|
return None
|
|
chosen = []
|
|
for key in asked:
|
|
skill, resource = key.split("/", 1)
|
|
if float(answers[key].value) >= threshold:
|
|
chosen.append({"skill": skill, "resource": resource})
|
|
logger.info("Read first: %s", [item["resource"] for item in chosen] or "none")
|
|
return chosen
|