feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ append outcomes that the next Orchestrator turn applies in order.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
@@ -48,15 +49,19 @@ from app.states.execution_state import (
|
||||
from app.states.ledger import DeliverableStatus, Ledger, LedgerRefusal
|
||||
from app.states.lessons import collect_lesson_candidates, refusal
|
||||
from app.tools import workspace
|
||||
from app.tools.assessment import assess
|
||||
from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools
|
||||
from app.tools.guardrail import EXECUTION_SCOPE, instructs_agent
|
||||
from app.tools.meaning import settle_meaning
|
||||
from app.tools.pillars import ADVICE_KEY, KEPT_FOR_CHANGES, pillars_touched
|
||||
from app.tools.skill_routing import read_first
|
||||
from app.workflows.chain import (
|
||||
IMPLEMENTATION_FILE,
|
||||
VERDICTS_FILE,
|
||||
unimplementable_checks,
|
||||
validation_outcome,
|
||||
)
|
||||
from app.workflows.changes import CHANGES_FILE, change_summary, is_empty
|
||||
from app.workflows.changes import CHANGES_FILE, _graph, change_summary, is_empty
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
@@ -113,6 +118,12 @@ def make_execution_nodes(
|
||||
confirm_drafts: bool = False,
|
||||
decider: Decider | None = None,
|
||||
pillar_threshold: float = 1.5,
|
||||
reworded_threshold: float = 0.25,
|
||||
guard_user_text: bool = False,
|
||||
guard_threshold: float = 0.75,
|
||||
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
|
||||
skill_threshold: float = 1.25,
|
||||
assess_at_end: bool = False,
|
||||
) -> ExecutionNodes:
|
||||
tool_factory = tool_factory or _no_tools
|
||||
# How an Orchestrator turn is decided. The default drives the tools from the
|
||||
@@ -139,7 +150,10 @@ def make_execution_nodes(
|
||||
outcomes=state.get("outcomes", []),
|
||||
orchestrator_refusals=state.get("refusals", []),
|
||||
role_skills={name: role.skills for name, role in roles.items()},
|
||||
skill_versions=skill_versions or {},
|
||||
skill_versions={
|
||||
**(skill_versions or {}),
|
||||
**(state.get("overlay_skills") or {}),
|
||||
},
|
||||
)
|
||||
# Kept apart from deliverables/: an Execution that ends with warnings writes
|
||||
# no Deliverable files, but what went wrong in it is still worth keeping.
|
||||
@@ -267,6 +281,7 @@ def make_execution_nodes(
|
||||
)
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
|
||||
update["risks"] = await _flagged(state, turns)
|
||||
toolbox = OrchestratorToolbox(
|
||||
ledger, deliverables, pending_questions, state.get("baseline")
|
||||
)
|
||||
@@ -323,11 +338,27 @@ def make_execution_nodes(
|
||||
hints = await _pillar_hints(
|
||||
state, ledger, deliverables, toolbox.started, decider, pillar_threshold
|
||||
)
|
||||
reading = await _reading_hints(
|
||||
state,
|
||||
ledger,
|
||||
deliverables,
|
||||
toolbox.started,
|
||||
decider,
|
||||
skill_references or {},
|
||||
skill_threshold,
|
||||
)
|
||||
sends = [
|
||||
Send(
|
||||
"specialist_task",
|
||||
_specialist_input(
|
||||
state, ledger, deliverables, drafts, kind, attempt, hints
|
||||
state,
|
||||
ledger,
|
||||
deliverables,
|
||||
drafts,
|
||||
kind,
|
||||
attempt,
|
||||
hints,
|
||||
reading.get(kind),
|
||||
),
|
||||
)
|
||||
for kind, attempt in toolbox.started.items()
|
||||
@@ -389,8 +420,44 @@ def make_execution_nodes(
|
||||
)
|
||||
return ask({"questions": [{"from": ORCHESTRATOR, "question": message}]})
|
||||
|
||||
async def _flagged(state: ExecutionState, turn: int) -> list[dict[str, Any]]:
|
||||
"""Risk flags on what the user wrote: recorded for a person, never acted on."""
|
||||
risks = list(state.get("risks", []))
|
||||
if not guard_user_text or decider is None:
|
||||
return risks
|
||||
reply = state.get("last_reply") or {}
|
||||
text = state["workflow_request"] if turn == 1 else str(reply.get("text", ""))
|
||||
if not text.strip():
|
||||
return risks
|
||||
try:
|
||||
likelihood = await instructs_agent(decider, text, state["execution_id"])
|
||||
except Exception: # a flag is never worth an Execution
|
||||
logger.exception("The guardrail failed on the user's text")
|
||||
return risks
|
||||
if likelihood is not None and likelihood >= guard_threshold:
|
||||
risks.append(
|
||||
{
|
||||
"risk": "instructs_agent",
|
||||
"where": "workflow_request" if turn == 1 else "user_reply",
|
||||
"turn": turn,
|
||||
"likelihood": round(likelihood, 2),
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
"[%s] the user's text reads as instructions to the agent (%.2f)",
|
||||
state["execution_id"],
|
||||
likelihood,
|
||||
)
|
||||
return risks
|
||||
|
||||
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
|
||||
role = roles[task_input["role"]]
|
||||
named = tuple(task_input["context"].get("overlay_skills") or ())
|
||||
if named:
|
||||
# The Execution's overlay Skills, on top of the role's own (ADR-0004).
|
||||
role = dataclasses.replace(
|
||||
role, skills=(*role.skills, *(n for n in named if n not in role.skills))
|
||||
)
|
||||
kind = DeliverableKind(task_input["kind"])
|
||||
working: dict[str, Any] = dict(task_input.get("working") or {})
|
||||
upstream = task_input["context"].get("dependencies") or {}
|
||||
@@ -484,6 +551,7 @@ def make_execution_nodes(
|
||||
task_input["attempt"],
|
||||
)
|
||||
started = time.monotonic()
|
||||
EXECUTION_SCOPE.set(task_input["execution_id"])
|
||||
try:
|
||||
if limiter:
|
||||
async with limiter.slot():
|
||||
@@ -588,23 +656,74 @@ def make_execution_nodes(
|
||||
update.update(ledger=ledger.to_dict(), conversation=conversation)
|
||||
return Command(update=update, goto="orchestrator_turn")
|
||||
|
||||
async def settled_changes(state: ExecutionState) -> dict[str, Any] | None:
|
||||
"""What a Change request changed, with rewordings told from real changes
|
||||
where a decision is available."""
|
||||
baseline = state.get("baseline") or None
|
||||
if not baseline:
|
||||
return None
|
||||
deliverables = state.get("deliverables", {})
|
||||
summary = change_summary(baseline.get("deliverables"), deliverables)
|
||||
if decider is None:
|
||||
return summary
|
||||
try:
|
||||
return await settle_meaning(
|
||||
decider,
|
||||
summary,
|
||||
baseline.get("deliverables"),
|
||||
deliverables,
|
||||
unchanged_at=reworded_threshold,
|
||||
scope=state["execution_id"],
|
||||
)
|
||||
except Exception: # the summary must be written whatever the decider does
|
||||
logger.exception("Settling what was reworded failed; listing as changed")
|
||||
return summary
|
||||
|
||||
async def assessed(state: ExecutionState) -> dict[str, Any] | None:
|
||||
"""How the design recommendations are judged, for the Execution's record."""
|
||||
if decider is None or not assess_at_end:
|
||||
return None
|
||||
try:
|
||||
return await assess(
|
||||
decider, state.get("deliverables", {}), scope=state["execution_id"]
|
||||
)
|
||||
except Exception: # an Execution ends whatever its assessment does
|
||||
logger.exception("Assessing the Execution failed; it carries none")
|
||||
return None
|
||||
|
||||
async def finalize(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
_write_outputs(output_root, state, "completed", ledger)
|
||||
assessment = await assessed(state)
|
||||
_write_outputs(
|
||||
output_root,
|
||||
state,
|
||||
"completed",
|
||||
ledger,
|
||||
await settled_changes(state),
|
||||
assessment,
|
||||
)
|
||||
close(state, "completed", has_files=True)
|
||||
await record_progress(state, ledger, "completed")
|
||||
await record_lessons(state, ledger, "completed")
|
||||
return {"status": "completed"}
|
||||
return {"status": "completed", "assessment": assessment}
|
||||
|
||||
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
# What was produced is kept, marked as unfinished: a person deciding what to
|
||||
# do next needs to read it, and it is the evidence of what went wrong.
|
||||
wrote = _write_outputs(output_root, state, "completed_with_warnings", ledger)
|
||||
assessment = await assessed(state)
|
||||
wrote = _write_outputs(
|
||||
output_root,
|
||||
state,
|
||||
"completed_with_warnings",
|
||||
ledger,
|
||||
await settled_changes(state),
|
||||
assessment,
|
||||
)
|
||||
close(state, "completed_with_warnings", has_files=wrote)
|
||||
await record_progress(state, ledger, "completed_with_warnings")
|
||||
await record_lessons(state, ledger, "completed_with_warnings")
|
||||
return {"status": "completed_with_warnings"}
|
||||
return {"status": "completed_with_warnings", "assessment": assessment}
|
||||
|
||||
return ExecutionNodes(
|
||||
orchestrator_turn=orchestrator_turn,
|
||||
@@ -725,6 +844,7 @@ def _specialist_input(
|
||||
kind: DeliverableKind,
|
||||
attempt: int,
|
||||
hints: Mapping[str, Any] | None = None,
|
||||
read_first: Sequence[Mapping[str, str]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
spec = DEPENDENCY_GRAPH[kind]
|
||||
given = dependencies_for(kind, deliverables)
|
||||
@@ -768,9 +888,15 @@ def _specialist_input(
|
||||
state.get("working", {}).get(kind.value, {})
|
||||
),
|
||||
"attempt": attempt,
|
||||
# Skills carrying the organisation's standards or this engagement's
|
||||
# constraints. Where one disagrees with general guidance, it wins.
|
||||
"overlay_skills": sorted(state.get("overlay_skills") or {}),
|
||||
# The pillars a change touches, when that could be judged: the design
|
||||
# advisor consults those and keeps its earlier advice for the rest.
|
||||
"pillars_to_consult": (hints or {}).get(kind.value),
|
||||
# The references of its Skills this task needs, when that could be
|
||||
# judged: where to start reading, not all it may read.
|
||||
"read_first": list(read_first) if read_first else None,
|
||||
# Set only in a Change request: what this Execution changes, and from what.
|
||||
"baseline": _baseline_context(kind, baseline, before, deliverables)
|
||||
if baseline
|
||||
@@ -817,6 +943,66 @@ async def _pillar_hints(
|
||||
return {} if chosen is None else {kind.value: chosen}
|
||||
|
||||
|
||||
async def _reading_hints(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
started: Mapping[DeliverableKind, int],
|
||||
decider: Decider | None,
|
||||
references: Mapping[str, Mapping[str, Mapping[str, str]]],
|
||||
threshold: float,
|
||||
) -> dict[DeliverableKind, list[dict[str, str]]]:
|
||||
"""For each Specialist being started whose Skills hold many references, the ones
|
||||
its task needs (iteration 5 step 8)."""
|
||||
if decider is None:
|
||||
return {}
|
||||
asked = {
|
||||
kind: references[DEPENDENCY_GRAPH[kind].specialist]
|
||||
for kind in started
|
||||
if references.get(DEPENDENCY_GRAPH[kind].specialist)
|
||||
}
|
||||
if not asked:
|
||||
return {}
|
||||
elements = _graph(deliverables)["elements"]
|
||||
design = "\n".join(
|
||||
f"- {item.get('name', item.get('id', ''))} ({item.get('product', '')}, "
|
||||
f"{item.get('resource_type', '')})"
|
||||
for item in elements
|
||||
)
|
||||
|
||||
def task(kind: DeliverableKind) -> str:
|
||||
feedback = list(ledger.feedback(kind))
|
||||
doing = "Revise" if feedback else "Write"
|
||||
parts = [
|
||||
(
|
||||
f"{doing} the {kind.value.replace('_', ' ')} Deliverable of a cloud "
|
||||
f"solution architecture, as its {DEPENDENCY_GRAPH[kind].specialist}."
|
||||
)
|
||||
]
|
||||
if feedback:
|
||||
parts.append("Feedback to address:\n" + "\n".join(feedback))
|
||||
parts.append(
|
||||
f"Elements of the Target architecture:\n{design}"
|
||||
if design
|
||||
else f"The request:\n{state['workflow_request']}"
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
chosen = await asyncio.gather(
|
||||
*(
|
||||
read_first(
|
||||
decider,
|
||||
task(kind),
|
||||
skills,
|
||||
threshold=threshold,
|
||||
scope=state["execution_id"],
|
||||
)
|
||||
for kind, skills in asked.items()
|
||||
)
|
||||
)
|
||||
return {kind: found for kind, found in zip(asked, chosen, strict=True) if found}
|
||||
|
||||
|
||||
# The solution guide is assembled afresh from the revised Deliverables, so the
|
||||
# Baseline's guide, the longest document there is, is not handed back to its author.
|
||||
_NOT_HANDED_BACK = {DeliverableKind.SOLUTION_GUIDE}
|
||||
@@ -989,7 +1175,12 @@ def _case(state: ExecutionState) -> str:
|
||||
|
||||
|
||||
def _write_outputs(
|
||||
output_root: Path, state: ExecutionState, status: str, ledger: Ledger
|
||||
output_root: Path,
|
||||
state: ExecutionState,
|
||||
status: str,
|
||||
ledger: Ledger,
|
||||
changes: Mapping[str, Any] | None = None,
|
||||
assessment: Mapping[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Write an Execution's files under its Customer case, with a manifest that ties
|
||||
the request to what was generated for it. Returns whether anything was written.
|
||||
@@ -1032,7 +1223,7 @@ def _write_outputs(
|
||||
# What this Change request changed against its Baseline, worked out from
|
||||
# the ids in the two versions' data files.
|
||||
files[CHANGES_FILE] = json.dumps(
|
||||
change_summary(baseline.get("deliverables"), deliverables),
|
||||
changes or change_summary(baseline.get("deliverables"), deliverables),
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -1049,8 +1240,14 @@ def _write_outputs(
|
||||
"workflow_request": state["workflow_request"],
|
||||
# The Execution this one changes, for a Change request.
|
||||
"baseline": (state.get("baseline") or {}).get("execution_id"),
|
||||
# The overlay Skills it loaded, with the version of each.
|
||||
"overlay_skills": dict(state.get("overlay_skills") or {}),
|
||||
"status": status,
|
||||
"warnings": list(state.get("warnings", [])),
|
||||
# Flags a person should look at; nothing was done about them.
|
||||
"risks": list(state.get("risks", [])),
|
||||
# How its design recommendations were judged when it ended, if they were.
|
||||
"assessment": dict(assessment) if assessment else None,
|
||||
"phases": {
|
||||
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user