feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)

Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 17:45:56 +01:00
parent 844987587c
commit 567aede5ca
38 changed files with 1956 additions and 58 deletions

View File

@@ -6,6 +6,7 @@ append outcomes that the next Orchestrator turn applies in order.
from __future__ import annotations
import asyncio
import dataclasses
import json
import logging
@@ -48,15 +49,19 @@ from app.states.execution_state import (
from app.states.ledger import DeliverableStatus, Ledger, LedgerRefusal
from app.states.lessons import collect_lesson_candidates, refusal
from app.tools import workspace
from app.tools.assessment import assess
from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools
from app.tools.guardrail import EXECUTION_SCOPE, instructs_agent
from app.tools.meaning import settle_meaning
from app.tools.pillars import ADVICE_KEY, KEPT_FOR_CHANGES, pillars_touched
from app.tools.skill_routing import read_first
from app.workflows.chain import (
IMPLEMENTATION_FILE,
VERDICTS_FILE,
unimplementable_checks,
validation_outcome,
)
from app.workflows.changes import CHANGES_FILE, change_summary, is_empty
from app.workflows.changes import CHANGES_FILE, _graph, change_summary, is_empty
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
PHASE_ORDER,
@@ -113,6 +118,12 @@ def make_execution_nodes(
confirm_drafts: bool = False,
decider: Decider | None = None,
pillar_threshold: float = 1.5,
reworded_threshold: float = 0.25,
guard_user_text: bool = False,
guard_threshold: float = 0.75,
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
skill_threshold: float = 1.25,
assess_at_end: bool = False,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
# How an Orchestrator turn is decided. The default drives the tools from the
@@ -139,7 +150,10 @@ def make_execution_nodes(
outcomes=state.get("outcomes", []),
orchestrator_refusals=state.get("refusals", []),
role_skills={name: role.skills for name, role in roles.items()},
skill_versions=skill_versions or {},
skill_versions={
**(skill_versions or {}),
**(state.get("overlay_skills") or {}),
},
)
# Kept apart from deliverables/: an Execution that ends with warnings writes
# no Deliverable files, but what went wrong in it is still worth keeping.
@@ -267,6 +281,7 @@ def make_execution_nodes(
)
return finish_with("end_with_warnings", warnings=warnings)
update["risks"] = await _flagged(state, turns)
toolbox = OrchestratorToolbox(
ledger, deliverables, pending_questions, state.get("baseline")
)
@@ -323,11 +338,27 @@ def make_execution_nodes(
hints = await _pillar_hints(
state, ledger, deliverables, toolbox.started, decider, pillar_threshold
)
reading = await _reading_hints(
state,
ledger,
deliverables,
toolbox.started,
decider,
skill_references or {},
skill_threshold,
)
sends = [
Send(
"specialist_task",
_specialist_input(
state, ledger, deliverables, drafts, kind, attempt, hints
state,
ledger,
deliverables,
drafts,
kind,
attempt,
hints,
reading.get(kind),
),
)
for kind, attempt in toolbox.started.items()
@@ -389,8 +420,44 @@ def make_execution_nodes(
)
return ask({"questions": [{"from": ORCHESTRATOR, "question": message}]})
async def _flagged(state: ExecutionState, turn: int) -> list[dict[str, Any]]:
"""Risk flags on what the user wrote: recorded for a person, never acted on."""
risks = list(state.get("risks", []))
if not guard_user_text or decider is None:
return risks
reply = state.get("last_reply") or {}
text = state["workflow_request"] if turn == 1 else str(reply.get("text", ""))
if not text.strip():
return risks
try:
likelihood = await instructs_agent(decider, text, state["execution_id"])
except Exception: # a flag is never worth an Execution
logger.exception("The guardrail failed on the user's text")
return risks
if likelihood is not None and likelihood >= guard_threshold:
risks.append(
{
"risk": "instructs_agent",
"where": "workflow_request" if turn == 1 else "user_reply",
"turn": turn,
"likelihood": round(likelihood, 2),
}
)
logger.warning(
"[%s] the user's text reads as instructions to the agent (%.2f)",
state["execution_id"],
likelihood,
)
return risks
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
role = roles[task_input["role"]]
named = tuple(task_input["context"].get("overlay_skills") or ())
if named:
# The Execution's overlay Skills, on top of the role's own (ADR-0004).
role = dataclasses.replace(
role, skills=(*role.skills, *(n for n in named if n not in role.skills))
)
kind = DeliverableKind(task_input["kind"])
working: dict[str, Any] = dict(task_input.get("working") or {})
upstream = task_input["context"].get("dependencies") or {}
@@ -484,6 +551,7 @@ def make_execution_nodes(
task_input["attempt"],
)
started = time.monotonic()
EXECUTION_SCOPE.set(task_input["execution_id"])
try:
if limiter:
async with limiter.slot():
@@ -588,23 +656,74 @@ def make_execution_nodes(
update.update(ledger=ledger.to_dict(), conversation=conversation)
return Command(update=update, goto="orchestrator_turn")
async def settled_changes(state: ExecutionState) -> dict[str, Any] | None:
"""What a Change request changed, with rewordings told from real changes
where a decision is available."""
baseline = state.get("baseline") or None
if not baseline:
return None
deliverables = state.get("deliverables", {})
summary = change_summary(baseline.get("deliverables"), deliverables)
if decider is None:
return summary
try:
return await settle_meaning(
decider,
summary,
baseline.get("deliverables"),
deliverables,
unchanged_at=reworded_threshold,
scope=state["execution_id"],
)
except Exception: # the summary must be written whatever the decider does
logger.exception("Settling what was reworded failed; listing as changed")
return summary
async def assessed(state: ExecutionState) -> dict[str, Any] | None:
"""How the design recommendations are judged, for the Execution's record."""
if decider is None or not assess_at_end:
return None
try:
return await assess(
decider, state.get("deliverables", {}), scope=state["execution_id"]
)
except Exception: # an Execution ends whatever its assessment does
logger.exception("Assessing the Execution failed; it carries none")
return None
async def finalize(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
_write_outputs(output_root, state, "completed", ledger)
assessment = await assessed(state)
_write_outputs(
output_root,
state,
"completed",
ledger,
await settled_changes(state),
assessment,
)
close(state, "completed", has_files=True)
await record_progress(state, ledger, "completed")
await record_lessons(state, ledger, "completed")
return {"status": "completed"}
return {"status": "completed", "assessment": assessment}
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
# What was produced is kept, marked as unfinished: a person deciding what to
# do next needs to read it, and it is the evidence of what went wrong.
wrote = _write_outputs(output_root, state, "completed_with_warnings", ledger)
assessment = await assessed(state)
wrote = _write_outputs(
output_root,
state,
"completed_with_warnings",
ledger,
await settled_changes(state),
assessment,
)
close(state, "completed_with_warnings", has_files=wrote)
await record_progress(state, ledger, "completed_with_warnings")
await record_lessons(state, ledger, "completed_with_warnings")
return {"status": "completed_with_warnings"}
return {"status": "completed_with_warnings", "assessment": assessment}
return ExecutionNodes(
orchestrator_turn=orchestrator_turn,
@@ -725,6 +844,7 @@ def _specialist_input(
kind: DeliverableKind,
attempt: int,
hints: Mapping[str, Any] | None = None,
read_first: Sequence[Mapping[str, str]] | None = None,
) -> dict[str, Any]:
spec = DEPENDENCY_GRAPH[kind]
given = dependencies_for(kind, deliverables)
@@ -768,9 +888,15 @@ def _specialist_input(
state.get("working", {}).get(kind.value, {})
),
"attempt": attempt,
# Skills carrying the organisation's standards or this engagement's
# constraints. Where one disagrees with general guidance, it wins.
"overlay_skills": sorted(state.get("overlay_skills") or {}),
# The pillars a change touches, when that could be judged: the design
# advisor consults those and keeps its earlier advice for the rest.
"pillars_to_consult": (hints or {}).get(kind.value),
# The references of its Skills this task needs, when that could be
# judged: where to start reading, not all it may read.
"read_first": list(read_first) if read_first else None,
# Set only in a Change request: what this Execution changes, and from what.
"baseline": _baseline_context(kind, baseline, before, deliverables)
if baseline
@@ -817,6 +943,66 @@ async def _pillar_hints(
return {} if chosen is None else {kind.value: chosen}
async def _reading_hints(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
started: Mapping[DeliverableKind, int],
decider: Decider | None,
references: Mapping[str, Mapping[str, Mapping[str, str]]],
threshold: float,
) -> dict[DeliverableKind, list[dict[str, str]]]:
"""For each Specialist being started whose Skills hold many references, the ones
its task needs (iteration 5 step 8)."""
if decider is None:
return {}
asked = {
kind: references[DEPENDENCY_GRAPH[kind].specialist]
for kind in started
if references.get(DEPENDENCY_GRAPH[kind].specialist)
}
if not asked:
return {}
elements = _graph(deliverables)["elements"]
design = "\n".join(
f"- {item.get('name', item.get('id', ''))} ({item.get('product', '')}, "
f"{item.get('resource_type', '')})"
for item in elements
)
def task(kind: DeliverableKind) -> str:
feedback = list(ledger.feedback(kind))
doing = "Revise" if feedback else "Write"
parts = [
(
f"{doing} the {kind.value.replace('_', ' ')} Deliverable of a cloud "
f"solution architecture, as its {DEPENDENCY_GRAPH[kind].specialist}."
)
]
if feedback:
parts.append("Feedback to address:\n" + "\n".join(feedback))
parts.append(
f"Elements of the Target architecture:\n{design}"
if design
else f"The request:\n{state['workflow_request']}"
)
return "\n".join(parts)
chosen = await asyncio.gather(
*(
read_first(
decider,
task(kind),
skills,
threshold=threshold,
scope=state["execution_id"],
)
for kind, skills in asked.items()
)
)
return {kind: found for kind, found in zip(asked, chosen, strict=True) if found}
# The solution guide is assembled afresh from the revised Deliverables, so the
# Baseline's guide, the longest document there is, is not handed back to its author.
_NOT_HANDED_BACK = {DeliverableKind.SOLUTION_GUIDE}
@@ -989,7 +1175,12 @@ def _case(state: ExecutionState) -> str:
def _write_outputs(
output_root: Path, state: ExecutionState, status: str, ledger: Ledger
output_root: Path,
state: ExecutionState,
status: str,
ledger: Ledger,
changes: Mapping[str, Any] | None = None,
assessment: Mapping[str, Any] | None = None,
) -> bool:
"""Write an Execution's files under its Customer case, with a manifest that ties
the request to what was generated for it. Returns whether anything was written.
@@ -1032,7 +1223,7 @@ def _write_outputs(
# What this Change request changed against its Baseline, worked out from
# the ids in the two versions' data files.
files[CHANGES_FILE] = json.dumps(
change_summary(baseline.get("deliverables"), deliverables),
changes or change_summary(baseline.get("deliverables"), deliverables),
indent=2,
ensure_ascii=False,
)
@@ -1049,8 +1240,14 @@ def _write_outputs(
"workflow_request": state["workflow_request"],
# The Execution this one changes, for a Change request.
"baseline": (state.get("baseline") or {}).get("execution_id"),
# The overlay Skills it loaded, with the version of each.
"overlay_skills": dict(state.get("overlay_skills") or {}),
"status": status,
"warnings": list(state.get("warnings", [])),
# Flags a person should look at; nothing was done about them.
"risks": list(state.get("risks", [])),
# How its design recommendations were judged when it ended, if they were.
"assessment": dict(assessment) if assessment else None,
"phases": {
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
},