feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)

Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 17:45:56 +01:00
parent 844987587c
commit 567aede5ca
38 changed files with 1956 additions and 58 deletions

View File

@@ -46,7 +46,7 @@ from app.nodes.router import (
)
from app.providers import ProviderBundle, load_provider
from app.recorder import DatabaseRecorder, ExecutionRecorder
from app.skills.agent_skills import skill_version
from app.skills.agent_skills import discover_skills, skill_version
from app.states.execution_state import check_customer_case
from app.states.ledger import Ledger
from app.tools.coverage import CoverageReview
@@ -56,6 +56,7 @@ from app.tools.local import TerraformRunner, local_tools
from app.tools.mcp import McpCatalog, auth_from_config
from app.tools.requirements_review import RequirementsReview
from app.tools.schema_review import Reviews, SchemaReview
from app.tools.skill_routing import routable
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.execution_graph import build_execution_graph
@@ -94,6 +95,14 @@ class ExecutionEngine:
confirm_drafts: bool = False,
decider: Decider | None = None,
pillar_threshold: float = 1.5,
reworded_threshold: float = 0.25,
overlays: Mapping[str, str] | None = None,
default_overlays: Sequence[str] = (),
guard_user_text: bool = False,
guard_threshold: float = 0.75,
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
skill_threshold: float = 1.25,
assess_at_end: bool = False,
) -> None:
required = {
ORCHESTRATOR,
@@ -104,6 +113,12 @@ class ExecutionEngine:
raise ValueError(f"Missing roles: {', '.join(missing)}")
self._max_revisions = max_revisions
self._output_root = output_root
# Overlay Skills an Execution may name, with the version of each (ADR-0004).
self._overlays = dict(overlays or {})
self._default_overlays = tuple(default_overlays)
unknown = sorted(set(self._default_overlays) - set(self._overlays))
if unknown:
raise ValueError(f"Unknown default overlay Skills: {', '.join(unknown)}")
self._graph = build_execution_graph(
runtime=runtime,
roles=roles,
@@ -118,6 +133,12 @@ class ExecutionEngine:
confirm_drafts=confirm_drafts,
decider=decider,
pillar_threshold=pillar_threshold,
reworded_threshold=reworded_threshold,
guard_user_text=guard_user_text,
guard_threshold=guard_threshold,
skill_references=skill_references,
skill_threshold=skill_threshold,
assess_at_end=assess_at_end,
).compile(checkpointer=checkpointer)
async def start(
@@ -131,6 +152,7 @@ class ExecutionEngine:
execution_id: str | None = None,
scripted_answers: Sequence[Sequence[str]] = (),
baseline: str | None = None,
overlay_skills: Sequence[str] = (),
) -> Interaction:
"""Start an Execution. `scripted_answers` are (regex, answer[, asker]) entries a
pre-approved Execution uses to answer questions, for evaluation runs.
@@ -139,11 +161,27 @@ class ExecutionEngine:
this a Change request (ADR-0006): the request says what has changed, and the
Execution starts from that Baseline's Deliverables. It is refused, before
anything runs or is recorded, when the Baseline cannot be one.
`overlay_skills` names Skills that carry an organisation's standards or
this engagement's constraints. Each must be one the deployment has
configured; an unknown name refuses the start. Every Specialist loads them
with the ones the deployment loads by default.
"""
customer_case = check_customer_case(customer_case)
execution_id = execution_id or str(uuid.uuid4())
if await self._exists(execution_id):
raise ValueError(f"Execution {execution_id} already exists")
if isinstance(overlay_skills, str) or not all(
isinstance(name, str) for name in overlay_skills
):
raise ValueError("overlay_skills must be a list of Skill names")
chosen = list(dict.fromkeys([*self._default_overlays, *overlay_skills]))
unknown = sorted(set(chosen) - set(self._overlays))
if unknown:
raise ValueError(
f"Unknown overlay Skills: {', '.join(unknown)}. Available: "
f"{', '.join(sorted(self._overlays)) or 'none'}"
)
loaded = (
load_baseline(self._output_root, customer_case, baseline)
if baseline
@@ -168,6 +206,9 @@ class ExecutionEngine:
"workflow_request": workflow_request,
# The Baseline of a Change request, as read from its files; else None.
"baseline": loaded,
# Overlay Skills this Execution loads, each with the version it had.
"overlay_skills": {name: self._overlays[name] for name in chosen},
"risks": [],
"pre_approved": pre_approved,
"allow_cloud_read": allow_cloud_read,
"ledger": Ledger(
@@ -262,6 +303,46 @@ class Assembly:
intake: Intake | None = None # reads a message that starts something
def discover_overlays(settings: Settings, bundle: ProviderBundle) -> dict[str, Any]:
"""The overlay Skills this deployment has configured, by name (ADR-0004).
They come only from the folders the deployment names, pass the same validation
as the provider's Skills, and may not reuse one of the provider's names.
"""
folders = [folder for folder in settings.overlay_dirs() if folder.is_dir()]
overlays = discover_skills(folders)
clash = sorted(set(overlays) & set(bundle.skills))
if clash:
raise ValueError(
f"Overlay Skills reuse the names of provider Skills: {', '.join(clash)}"
)
return overlays
def skill_dirs(settings: Settings, bundle: ProviderBundle) -> dict[str, Path]:
"""Every Skill a runtime may be asked to load: the provider's and the overlays."""
return {
**{name: meta.path for name, meta in bundle.skills.items()},
**{
name: meta.path
for name, meta in discover_overlays(settings, bundle).items()
},
}
def role_references(
roles: Mapping[str, RoleSpec], bundle: ProviderBundle
) -> dict[str, dict[str, dict[str, str]]]:
"""For each role, its Skills that hold enough references to choose among, with
what each reference covers (iteration 5 step 8)."""
worth = routable({name: meta.path for name, meta in bundle.skills.items()})
by_role = {
name: {skill: worth[skill] for skill in role.skills if skill in worth}
for name, role in roles.items()
}
return {name: skills for name, skills in by_role.items() if skills}
def open_decider(settings: Settings) -> Decider:
"""The Jev client when a key is configured; otherwise a decider that always
sends its caller to the fallback."""
@@ -299,11 +380,12 @@ async def open_assembly(
include_preview_mcp=settings.MCP_ENABLE_PREVIEW,
lessons=settings.LESSONS_ENABLED,
)
overlays = discover_overlays(settings, bundle)
if runtime is None:
from app.adk.adk_runtime import AdkAgentRuntime
runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
skill_dirs=skill_dirs(settings, bundle),
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
@@ -325,6 +407,7 @@ async def open_assembly(
offered = [name for names in bundle.tools.values() for name in names]
# One cap for Specialists and the pillar advisors they fan out to.
limiter = RunLimiter(settings.MAX_PARALLEL_SPECIALISTS)
decider = open_decider(settings)
tool_factory = make_tool_factory(
runtime,
roles,
@@ -332,9 +415,12 @@ async def open_assembly(
local_tools=lambda working_copy: local_tools(offered, runner, working_copy),
catalog=catalog,
limiter=limiter,
guard=decider
if settings.JEV_GUARDRAIL and isinstance(decider, JevDecider)
else None,
withhold_at=settings.JEV_GUARDRAIL_THRESHOLD,
)
decider = open_decider(settings)
orchestrator: OrchestratorPolicy | None = None
if settings.ORCHESTRATOR == "router":
# The Orchestrator routes: Jev reads, the playbook decides, and a model only
@@ -387,6 +473,18 @@ async def open_assembly(
confirm_drafts=settings.CONFIRM_DRAFTS,
decider=decider,
pillar_threshold=settings.JEV_PILLAR_THRESHOLD,
reworded_threshold=settings.JEV_REWORDED_THRESHOLD,
guard_user_text=settings.JEV_GUARDRAIL,
guard_threshold=settings.JEV_GUARDRAIL_THRESHOLD,
skill_references=role_references(roles, bundle)
if settings.JEV_SKILL_ROUTING
else None,
skill_threshold=settings.JEV_SKILL_THRESHOLD,
assess_at_end=settings.JEV_ASSESS_EXECUTIONS,
overlays={
name: skill_version(meta.path) for name, meta in overlays.items()
},
default_overlays=settings.default_overlays(),
)
try:
yield Assembly(