Files
orchestration_engine/execution_engine/nodes/execution_nodes.py

1482 lines
59 KiB
Python

"""The nodes of the Execution graph (ADR-0003); app/workflows/execution_graph.py wires them.
Only orchestrator_turn and await_user change the Ledger; parallel Specialist tasks
append outcomes that the next Orchestrator turn applies in order.
"""
from __future__ import annotations
import asyncio
import copy
import dataclasses
import json
import logging
import re
import time
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any, cast
import yaml
from langgraph.types import Command, Send, interrupt
from execution_engine.adk.runtime import (
REJECTION_PREFIXES,
AgentRuntime,
AgentTask,
BudgetExhausted,
Decider,
NeedsInput,
Outcome,
RoleSpec,
RunLimiter,
Stopped,
SubmissionReview,
Submitted,
ToolFactory,
ToolSpec,
TurnEnded,
)
from execution_engine.baseline import WORKING_FILENAME
from execution_engine.customer_case import (
IN_PROGRESS,
SCOPE_COMPLETED,
close_intent,
intent_dir,
)
from execution_engine.nodes.router import (
ModelOrchestrator,
OrchestratorPolicy,
TurnView,
)
from execution_engine.recorder import ExecutionRecorder
from execution_engine.scope import APPROVED, CASE_FIELD, LOCKED, UNDER_REVIEW
from execution_engine.states.execution_state import (
UNASSIGNED_CASE,
ExecutionState,
execution_dir,
lessons_file,
)
from execution_engine.states.ledger import (
DeliverableStatus,
Ledger,
LedgerRefusal,
PhaseStatus,
)
from execution_engine.states.lessons import collect_lesson_candidates, refusal
from execution_engine.tools import workspace
from execution_engine.tools.execution import (
ORCHESTRATOR,
OrchestratorToolbox,
specialist_tools,
)
from execution_engine.tools.guardrail import EXECUTION_SCOPE, instructs_agent
from execution_engine.tools.scoping import (
SCOPE_DATA_FILE,
SCOPE_FILE,
SCOPE_KEY,
Scoping,
working_from_completed,
)
from execution_engine.tools.skill_routing import read_first
from execution_engine.workflows.workflow import Hints, Workflow
logger = logging.getLogger(__name__)
# Working-memory key: the last submission of a Deliverable with what it was written from.
STANDING = "standing"
# Questions put to the user running, with no work done between, before stopping.
MAX_IDLE_ASKS = 3
# Runs of one Specialist running that end without a submission, in an Execution with
# nobody to answer, before it stops. Each is answered automatically and run again;
# a Specialist that cannot finish would otherwise go on until the turn limit.
MAX_UNFINISHED_RUNS = 3
NO_USER_REPLY = (
"No user is available. State your assumption, record it in the Deliverable under "
"'Assumptions', and proceed."
)
MANIFEST_FILENAME = "execution.json" # request, status and files of one Execution
def _no_tools(
role: RoleSpec, deliverable: str | None, working: MutableMapping[str, Any]
) -> Sequence[ToolSpec]:
return ()
@dataclass(frozen=True)
class ExecutionNodes:
"""The node functions of one Execution graph, bound to its runtime and recorder."""
orchestrator_turn: Callable[[ExecutionState], Awaitable[Command]]
# Receives its Send payload, not the Execution state.
specialist_task: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
await_user: Callable[[ExecutionState], Awaitable[Command]]
finalize: Callable[[ExecutionState], Awaitable[dict[str, Any]]]
end_with_warnings: Callable[[ExecutionState], Awaitable[dict[str, Any]]]
def make_execution_nodes(
*,
workflow: Workflow,
runtime: AgentRuntime,
roles: Mapping[str, RoleSpec],
recorder: ExecutionRecorder,
output_root: Path,
max_orchestrator_turns: int = 100,
tool_factory: ToolFactory | None = None,
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
decider: Decider | None = None,
reworded_threshold: float = 0.25,
guard_user_text: bool = False,
guard_threshold: float = 0.75,
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
skill_threshold: float = 1.25,
assess_at_end: bool = False,
scoping: Scoping | None = None,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
# How an Orchestrator turn is decided. The default drives the tools from the
# Orchestrator role on the runtime; the assembly supplies the router (ADR-0007).
orchestrator = orchestrator or ModelOrchestrator(runtime, roles[ORCHESTRATOR])
def close(state: ExecutionState, status: str, *, has_files: bool) -> None:
close_intent(
output_root,
_case(state),
execution_id=state["execution_id"],
status=status,
has_files=has_files,
)
async def record_lessons(
state: ExecutionState, ledger: Ledger, status: str
) -> None:
record = collect_lesson_candidates(
execution_id=state["execution_id"],
customer_case=_case(state),
status=status,
ledger=ledger,
outcomes=state.get("outcomes", []),
orchestrator_refusals=state.get("refusals", []),
role_skills={name: role.skills for name, role in roles.items()},
skill_versions={
**(skill_versions or {}),
**(state.get("overlay_skills") or {}),
},
)
# Kept apart from deliverables/: an Execution that ends with warnings writes
# no Deliverable files, but what went wrong in it is still worth keeping.
path = lessons_file(output_root, state["execution_id"], _case(state))
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8"
)
await recorder.record_lesson_candidates(state["execution_id"], record)
async def record_progress(
state: ExecutionState, ledger: Ledger, status: str | None = None
) -> int:
recorded = state.get("events_recorded", 0)
for event in ledger.events[recorded:]:
logger.info(
"[%s] ledger %s %s by %s%s",
state["execution_id"],
event.type,
event.subject,
event.actor,
f" ({event.detail})" if event.detail else "",
)
await recorder.record_events(state["execution_id"], ledger.events[recorded:])
if status:
await recorder.record_status(
state["execution_id"],
session_id=state["session_id"],
workflow_request=state["workflow_request"],
status=status,
current_phase=_current_phase(ledger),
details={
"revisions": sum(
ledger.revisions(kind) for kind in workflow.deliverables
),
"warnings": state.get("warnings", []),
"customer_case": _case(state),
},
)
return len(ledger.events)
async def orchestrator_turn(state: ExecutionState) -> Command:
ledger = Ledger.from_dict(workflow, state["ledger"])
deliverables = dict(state.get("deliverables", {}))
pending_questions = list(state.get("pending_questions", []))
notes = list(state.get("notes", []))
pending_permission = state.get("pending_permission")
warnings = list(state.get("warnings", []))
shown = list(state.get("drafts_shown", []))
drafts = dict(state.get("drafts", {}))
outcomes = state.get("outcomes", [])
arrived = False
pre_approved = bool(state.get("pre_approved"))
unfinished = dict(state.get("unfinished_runs") or {})
for record in outcomes[state.get("outcomes_applied", 0) :]:
kind = workflow.kind(record["kind"])
try:
if record["type"] == "submitted":
ledger.submit(kind, record["attempt"], actor=record["role"])
arrived = True
deliverables[kind] = {
"content": record["content"],
"attachments": record["attachments"],
}
drafts.pop(kind, None)
unfinished.pop(kind, None)
await recorder.record_submission(
state["execution_id"],
kind,
record["content"],
record["attachments"],
)
continue
ledger.release(kind, record["attempt"], reason=record["type"])
except LedgerRefusal as exc:
notes.append(f"Discarded a superseded result for {kind}: {exc}")
continue
if pre_approved and not record.get("permission"):
unfinished[kind] = unfinished.get(kind, 0) + 1
if record["type"] == "needs_input" and record.get("permission"):
if state.get("allow_cloud_read") is None:
requested = (pending_permission or {}).get("requested_by", [])
pending_permission = {
"tier": record["permission"],
"requested_by": [*requested, kind],
}
continue
if record.get("draft"):
drafts[kind] = {
"content": record["draft"],
"attachments": record.get("draft_attachments", {}),
}
# The user is being shown this Deliverable.
if kind not in shown:
shown.append(kind)
questions = record.get("questions") or [
(
f"The {record['role']} stopped: {record['summary']} "
"How should it proceed?"
)
]
pending_questions.append({"kind": kind, "questions": questions})
if arrived:
# What has been submitted is on disk as soon as it exists, marked as in
# progress. A process that is killed leaves that much behind.
try:
_write_outputs(
output_root,
cast(ExecutionState, {**state, "deliverables": deliverables}),
IN_PROGRESS,
ledger,
)
except OSError:
logger.exception("Writing the Execution's files so far failed")
turns = state.get("turns", 0) + 1
update: dict[str, Any] = {
"deliverables": deliverables,
"drafts": drafts,
"drafts_shown": shown,
"pending_questions": pending_questions,
"pending_permission": pending_permission,
"outcomes_applied": len(outcomes),
"unfinished_runs": unfinished,
"turns": turns,
}
def finish_with(goto: str, **extra: Any) -> Command:
return Command(
update={**update, "ledger": ledger.to_dict(), "notes": notes, **extra},
goto=goto,
)
if pending_permission and not state.get("pre_approved"):
update["events_recorded"] = await record_progress(
state, ledger, "awaiting_user"
)
return finish_with("await_user", prompt={"permission": pending_permission})
stuck = sorted(
kind for kind, runs in unfinished.items() if runs >= MAX_UNFINISHED_RUNS
)
if stuck:
# Nobody is there to help, and asking again has not helped.
warnings += [
f"Stopped: {workflow.deliverables[kind].specialist} ended "
f"{unfinished[kind]} runs running without submitting {kind}, with "
"nobody to answer it."
for kind in stuck
]
update["events_recorded"] = await record_progress(state, ledger)
return finish_with("end_with_warnings", warnings=warnings)
if turns > max_orchestrator_turns:
warnings.append(
f"Stopped after {max_orchestrator_turns} Orchestrator turns without "
"finishing."
)
return finish_with("end_with_warnings", warnings=warnings)
update["risks"] = await _flagged(state, turns)
toolbox = OrchestratorToolbox(
ledger, deliverables, pending_questions, state.get("baseline")
)
refusals = list(state.get("refusals", []))
update["refusals"] = refusals
view = TurnView(
execution_id=state["execution_id"],
pre_approved=bool(state.get("pre_approved")),
ledger=ledger,
deliverables=deliverables,
pending_questions=pending_questions,
outcomes=outcomes,
last_reply=state.get("last_reply"),
pending_change=state.get("pending_change"),
awaiting_gap_decision=bool(state.get("awaiting_gap_decision")),
validation_settled=int(state.get("validation_settled", 0)),
awaiting_limit_decision=bool(state.get("awaiting_limit_decision")),
revision_limit_asked=tuple(state.get("revision_limit_asked", ())),
context=_orchestrator_context(
state, ledger, deliverables, pending_questions, notes
),
)
started = time.monotonic()
decided = await orchestrator.turn(
view, _noting_refusals(toolbox.tools(), refusals, ORCHESTRATOR, None)
)
outcome = decided.outcome
update.update(decided.updates)
if decided.decisions:
# Every decision is kept: what was read, which group applied, what was done.
update["decisions"] = [
*state.get("decisions", []),
*({"turn": turns, **decision} for decision in decided.decisions),
]
logger.info(
"[%s] orchestrator turn %d (%s phase) ended in %.0fs: %s",
state["execution_id"],
turns,
_current_phase(ledger),
time.monotonic() - started,
_summarise_outcome(outcome),
)
notes = []
pre_approved = bool(state.get("pre_approved"))
conversation = list(state.get("conversation", []))
def ask(prompt: dict[str, Any], **extra: Any) -> Command:
conversation.append({"from": ORCHESTRATOR, "text": _render_prompt(prompt)})
return finish_with(
"await_user", prompt=prompt, conversation=conversation, **extra
)
if isinstance(outcome, TurnEnded) and outcome.tool == "delegate":
hints = await _task_hints(
state, ledger, deliverables, toolbox.started, decider
)
reading = await _reading_hints(
state,
ledger,
deliverables,
toolbox.started,
decider,
skill_references or {},
skill_threshold,
)
sends = [
Send(
"specialist_task",
_specialist_input(
state,
ledger,
deliverables,
drafts,
kind,
attempt,
hints,
reading.get(kind),
),
)
for kind, attempt in toolbox.started.items()
]
update["events_recorded"] = await record_progress(state, ledger)
return Command(
update={**update, "ledger": ledger.to_dict(), "notes": notes},
goto=sends,
)
if isinstance(outcome, TurnEnded) and outcome.tool == "finish":
update["events_recorded"] = await record_progress(state, ledger)
return finish_with("finalize")
if isinstance(outcome, TurnEnded) and outcome.tool == "escalate":
warnings.append(toolbox.escalation or "Escalated without detail.")
update["events_recorded"] = await record_progress(state, ledger)
return finish_with("end_with_warnings", warnings=warnings)
if isinstance(outcome, TurnEnded) and outcome.tool == "request_approval":
update["events_recorded"] = await record_progress(
state, ledger, None if pre_approved else "awaiting_user"
)
return ask({"approval": toolbox.approval})
if isinstance(outcome, TurnEnded) and outcome.tool == "ask_user":
# Asking again and again with nothing done in between is a loop, not a
# conversation: stop it, whatever caused it.
fingerprint = _progress(ledger)
idle = (
state.get("idle_asks", 0) + 1
if fingerprint == state.get("ask_fingerprint")
else 0
)
update.update(idle_asks=idle, ask_fingerprint=fingerprint)
if idle >= MAX_IDLE_ASKS:
warnings.append(
f"Stopped: the user was asked {idle + 1} times running with "
"no work done in between."
)
update["events_recorded"] = await record_progress(state, ledger)
return finish_with("end_with_warnings", warnings=warnings)
update["events_recorded"] = await record_progress(
state, ledger, None if pre_approved else "awaiting_user"
)
return ask(
{"questions": toolbox.questions},
pending_questions=[],
awaiting_questions=pending_questions,
)
message = _describe_unfinished(outcome)
if pre_approved:
warnings.append(f"The Orchestrator could not continue: {message}")
update["events_recorded"] = await record_progress(state, ledger)
return finish_with("end_with_warnings", warnings=warnings)
update["events_recorded"] = await record_progress(
state, ledger, "awaiting_user"
)
return ask({"questions": [{"from": ORCHESTRATOR, "question": message}]})
async def _flagged(state: ExecutionState, turn: int) -> list[dict[str, Any]]:
"""Risk flags on what the user wrote: recorded for a person, never acted on."""
risks = list(state.get("risks", []))
if not guard_user_text or decider is None:
return risks
reply = state.get("last_reply") or {}
text = state["workflow_request"] if turn == 1 else str(reply.get("text", ""))
if not text.strip():
return risks
try:
likelihood = await instructs_agent(decider, text, state["execution_id"])
except Exception: # a flag is never worth an Execution
logger.exception("The guardrail failed on the user's text")
return risks
if likelihood is not None and likelihood >= guard_threshold:
risks.append(
{
"risk": "instructs_agent",
"where": "workflow_request" if turn == 1 else "user_reply",
"turn": turn,
"likelihood": round(likelihood, 2),
}
)
logger.warning(
"[%s] the user's text reads as instructions to the agent (%.2f)",
state["execution_id"],
likelihood,
)
return risks
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
role = roles[task_input["role"]]
named = tuple(task_input["context"].get("overlay_skills") or ())
if named:
# The Execution's overlay Skills, on top of the role's own (ADR-0004).
role = dataclasses.replace(
role, skills=(*role.skills, *(n for n in named if n not in role.skills))
)
scope_skill = scoping.skill if scoping else task_input.get("scope_skill")
if scope_skill and scope_skill not in role.skills:
# An Execution with a scope spec: every Specialist can read what it means.
role = dataclasses.replace(role, skills=(*role.skills, scope_skill))
if workflow.prepare_role:
role = workflow.prepare_role(role, task_input)
if role.first_attempt_model and task_input.get("first_attempt"):
logger.info(
"[%s] %s takes %s for a first attempt",
task_input["execution_id"],
role.name,
role.first_attempt_model,
)
role = dataclasses.replace(role, model=role.first_attempt_model)
kind = workflow.kind(task_input["kind"])
working: dict[str, Any] = dict(task_input.get("working") or {})
upstream = (
task_input.get("upstream")
or task_input["context"].get("dependencies")
or {}
)
before = task_input.get("baseline")
if before and not working:
# A Change request starts from the Baseline's version, not from nothing:
# its graph, its files and its description are the working copy.
_start_from(
workflow, kind, before, working, task_input.get("baseline_basis")
)
# And from its working material: the advice it was written from.
working.update(task_input.get("baseline_working") or {})
if kind == workflow.spec_deliverable and scoping and SCOPE_KEY not in working:
# The spec the customer supplied, or an empty one to draft into.
delivered = ((before or {}).get("attachments") or {}).get(SCOPE_FILE)
working[SCOPE_KEY] = (
# A Change request starts from the scope that was delivered.
(working_from_completed(delivered, scoping) if delivered else None)
or copy.deepcopy(
task_input.get("scope_working") or {"values": {}, "states": {}}
)
)
spec = workflow.deliverables[kind]
source = spec.working_copy
if source and source != kind:
# The tools work on the files as they were last submitted.
working[workspace.FILES] = dict(
upstream.get(source, {}).get("attachments", {})
)
# Feedback on this Deliverable itself. A note that a dependency was revised
# is not a reason to write it again; the basis says whether that matters.
feedback = [
item
for item in task_input["context"].get("feedback") or []
if not str(item).startswith("Dependency ")
]
basis = spec.written_from(upstream) if spec.written_from else None
standing = working.get(STANDING)
if (
basis
and isinstance(standing, Mapping)
and standing.get("basis") == basis
and standing.get("feedback") == feedback
):
# What it was written from has not changed, so the last submission
# still stands: no Specialist is run.
logger.info(
"[%s] %s kept %s (attempt %d): written from an unchanged basis",
task_input["execution_id"],
role.name,
kind,
task_input["attempt"],
)
kept = Submitted(standing["content"], dict(standing["attachments"]))
return {
"outcomes": [
{
**_outcome_record(role.name, kind, task_input["attempt"], kept),
"refusals": [],
}
],
"working": {kind: working},
}
async def review(attachments: Mapping[str, str]) -> Sequence[str]:
assert submission_review is not None
return await submission_review(
kind, attachments, upstream, task_input["execution_id"]
)
refusals: list[dict[str, Any]] = []
tools = _noting_refusals(
specialist_tools(
workflow,
kind,
extra=tool_factory(role, kind, working),
allow_cloud_read=task_input["allow_cloud_read"],
upstream=upstream,
working=working,
library=task_input.get("library"),
review=review if submission_review else None,
baseline=before,
scoping=scoping,
# With a user present, their first sight of a Deliverable comes
# before it is submitted, whatever the Specialist intends.
confirm_first=confirm_drafts
and not task_input["context"].get("pre_approved", True)
and not task_input.get("shown"),
),
refusals,
role.name,
kind,
)
task = AgentTask(
role=role.name,
prompt=f"Produce the {kind} Deliverable.",
context=task_input["context"],
deliverable=kind,
)
logger.info(
"[%s] %s started %s (attempt %d)",
task_input["execution_id"],
role.name,
kind,
task_input["attempt"],
)
started = time.monotonic()
EXECUTION_SCOPE.set(task_input["execution_id"])
try:
if limiter:
async with limiter.slot():
outcome: Outcome = await runtime.run(role, task, tools)
else:
outcome = await runtime.run(role, task, tools)
except Exception as exc:
logger.exception("Specialist %s failed on %s", role.name, kind)
outcome = Stopped(f"it failed with {type(exc).__name__}: {exc}")
logger.info(
"[%s] %s finished %s in %.0fs: %s",
task_input["execution_id"],
role.name,
kind,
time.monotonic() - started,
_summarise_outcome(outcome),
)
if basis and isinstance(outcome, Submitted):
working[STANDING] = {
"basis": basis,
"feedback": feedback,
"content": outcome.content,
"attachments": dict(outcome.attachments),
}
return {
"outcomes": [
{
**_outcome_record(role.name, kind, task_input["attempt"], outcome),
"refusals": refusals,
}
],
"working": {kind: working},
}
async def await_user(state: ExecutionState) -> Command:
prompt = state["prompt"] or {}
if state.get("pre_approved"):
reply = automatic_reply(prompt, state.get("scripted_answers", []))
else:
reply = interrupt(prompt)
ledger = Ledger.from_dict(workflow, state["ledger"])
conversation = list(state.get("conversation", []))
update: dict[str, Any] = {"prompt": None}
said: dict[str, Any] | None = None # what the user said, for the next turn
if "approval" in prompt:
phase = prompt["approval"]["phase"]
decision = reply.get("approval") if isinstance(reply, Mapping) else None
if (
isinstance(decision, Mapping)
and decision.get("phase") == phase
and decision.get("decision") in {"approve", "changes"}
):
phase_id = workflow.phase(phase)
if decision["decision"] == "approve":
ledger.approve(phase_id, actor="user")
text = f"Approved {phase}."
else:
feedback = str(decision.get("feedback", ""))
ledger.request_changes(phase_id, feedback, actor="user")
text = f"Requested changes to {phase}: {feedback}"
said = {"kind": "changes", "phase": phase, "text": feedback}
conversation.append({"from": "user", "text": text})
else:
conversation.append({"from": "user", "text": _reply_text(reply)})
said = {"kind": "reply", "text": _reply_text(reply)}
elif "permission" in prompt:
tier = prompt["permission"]["tier"]
permission = reply.get("permission") if isinstance(reply, Mapping) else None
grant = permission.get(tier) if isinstance(permission, Mapping) else None
if isinstance(grant, bool):
update["allow_cloud_read"] = grant
update["pending_permission"] = None
verdict = "granted" if grant else "denied"
conversation.append(
{
"from": "user",
"text": f"Permission to use {tier} tools {verdict}.",
}
)
else:
conversation.append({"from": "user", "text": _reply_text(reply)})
else:
text = _reply_text(reply)
conversation.append({"from": "user", "text": text})
put = [item["question"] for item in prompt.get("questions", [])]
# What is routed is what the user said, not our questions quoted back.
said = {"kind": "reply", "text": _own_words(text, put), "asked": put}
answers = {
kind: list(items) for kind, items in state.get("answers", {}).items()
}
for asked in state.get("awaiting_questions", []):
answers.setdefault(asked["kind"], []).append(
{"questions": list(asked["questions"]), "reply": text}
)
update.update(answers=answers, awaiting_questions=[])
# Only a real user's words are routed; a pre-approved Execution has none.
update["last_reply"] = None if state.get("pre_approved") else said
update["events_recorded"] = await record_progress(state, ledger)
update.update(ledger=ledger.to_dict(), conversation=conversation)
return Command(update=update, goto="orchestrator_turn")
async def settled_changes(state: ExecutionState) -> dict[str, Any] | None:
"""What a Change request changed, with rewordings told from real changes
where a decision is available."""
baseline = state.get("baseline") or None
if not baseline:
return None
deliverables = state.get("deliverables", {})
policy = workflow.changes
if policy is None:
return None
summary = policy.summary(baseline.get("deliverables"), deliverables)
if decider is None or policy.settle is None:
return summary
try:
return await policy.settle(
decider,
summary,
baseline.get("deliverables"),
deliverables,
unchanged_at=reworded_threshold,
scope=state["execution_id"],
)
except Exception: # the summary must be written whatever the decider does
logger.exception("Settling what was reworded failed; listing as changed")
return summary
async def assessed(state: ExecutionState) -> dict[str, Any] | None:
"""How the finished work is judged, for the Execution's record."""
if decider is None or not assess_at_end or workflow.assess is None:
return None
try:
return await workflow.assess(
decider, state.get("deliverables", {}), scope=state["execution_id"]
)
except Exception: # an Execution ends whatever its assessment does
logger.exception("Assessing the Execution failed; it carries none")
return None
async def finalize(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(workflow, state["ledger"])
assessment = await assessed(state)
_write_outputs(
output_root,
state,
"completed",
ledger,
await settled_changes(state),
assessment,
)
close(state, "completed", has_files=True)
await record_progress(state, ledger, "completed")
await record_lessons(state, ledger, "completed")
return {"status": "completed", "assessment": assessment}
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(workflow, state["ledger"])
# What was produced is kept, marked as unfinished: a person deciding what to
# do next needs to read it, and it is the evidence of what went wrong.
assessment = await assessed(state)
wrote = _write_outputs(
output_root,
state,
"completed_with_warnings",
ledger,
await settled_changes(state),
assessment,
)
close(state, "completed_with_warnings", has_files=wrote)
await record_progress(state, ledger, "completed_with_warnings")
await record_lessons(state, ledger, "completed_with_warnings")
return {"status": "completed_with_warnings", "assessment": assessment}
return ExecutionNodes(
orchestrator_turn=orchestrator_turn,
specialist_task=specialist_task,
await_user=await_user,
finalize=finalize,
end_with_warnings=end_with_warnings,
)
# --- helpers ---------------------------------------------------------------------------
def _noting_refusals(
tools: Sequence[ToolSpec],
sink: list[dict[str, Any]],
role: str,
deliverable: str | None,
) -> list[ToolSpec]:
"""The same tools, noting each reply that means the call did nothing."""
def noting(tool: ToolSpec) -> ToolSpec:
async def handler(**kwargs: Any) -> Any:
result = await tool.handler(**kwargs)
if result.outcome is None and result.message.startswith(REJECTION_PREFIXES):
sink.append(
refusal(role, deliverable, tool.name, tool.tier, result.message)
)
return result
return dataclasses.replace(tool, handler=handler)
return [noting(tool) for tool in tools]
def _summarise_outcome(outcome: Outcome) -> str:
if isinstance(outcome, TurnEnded):
return f"called {outcome.tool}"
if isinstance(outcome, Submitted):
return f"submitted ({len(outcome.content)} chars)"
if isinstance(outcome, NeedsInput):
return f"needs input ({len(outcome.questions)} questions)"
if isinstance(outcome, BudgetExhausted):
return f"budget exhausted: {outcome.summary[:200]}"
return f"stopped: {outcome.message[:200]}"
def _current_phase(ledger: Ledger) -> str:
phases = ledger.workflow.phases
for phase in phases:
if ledger.phase_status(phase).value != "approved":
return str(phase)
return str(phases[-1])
def _orchestrator_context(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
pending_questions: Sequence[Mapping[str, Any]],
notes: Sequence[str],
) -> dict[str, Any]:
workflow = ledger.workflow
policy = workflow.validation
# Only a validation that still stands: once anything it judged is reopened, its
# verdicts are about work that no longer exists.
validation = (
deliverables.get(policy.kind)
if policy and ledger.status(policy.kind) is DeliverableStatus.SUBMITTED
else None
)
return {
# Checks the implementation's author reported cannot be met as written:
# faults in the design, known before any validation.
"unimplementable_checks": policy.unimplementable(deliverables)
if policy
and policy.unimplementable
and ledger.status(policy.implementation) is DeliverableStatus.SUBMITTED
else [],
# The verdict on each Check, once validation is submitted: failed Checks are
# faults in the implementation; the rest are gaps in the design.
"validation": policy.read(
validation.get("attachments", {}).get(policy.data_file, "")
)
if policy and validation
else None,
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
"allow_cloud_read": state.get("allow_cloud_read"),
"deliverables": {
str(kind): {
"phase": str(spec.phase),
"status": ledger.status(kind).value,
"revisions": ledger.revisions(kind),
"feedback": list(ledger.feedback(kind)),
}
for kind, spec in workflow.deliverables.items()
},
"phases": {
str(phase): ledger.phase_status(phase).value for phase in workflow.phases
},
"ready_work": [str(kind) for kind in ledger.ready_work()],
"phases_ready_for_approval": [
str(p) for p in ledger.phases_ready_for_approval()
],
"pending_questions": list(pending_questions),
"conversation": list(state.get("conversation", []))[-20:],
"notes": list(notes),
}
def _specialist_input(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
drafts: Mapping[str, Mapping[str, Any]],
kind: str,
attempt: int,
hints: Mapping[str, Any] | None = None,
read_first: Sequence[Mapping[str, str]] | None = None,
) -> dict[str, Any]:
workflow = ledger.workflow
spec = workflow.deliverables[kind]
given = workflow.dependencies_for(kind, deliverables)
seen = workflow.shown(kind, given)
baseline = state.get("baseline") or None
before = (baseline or {}).get("deliverables", {}).get(kind)
own, own_context = (
workflow.task_entries(state, kind) if workflow.task_entries else ({}, {})
)
return {
**own,
"execution_id": state["execution_id"],
# The Deliverables it is not shown whole, for reading on demand: those
# given as data files alone, and those not shown at all.
"library": {
name: dict(deliverables[name])
for name in given
if "content" not in seen.get(name, {})
},
# Everything it works from, whole or as data: what its submission is
# checked against, whatever part of it the Specialist is shown.
"upstream": given,
"role": spec.specialist,
"kind": kind,
"attempt": attempt,
# Nothing has been asked of this Deliverable again: no feedback on it, and
# no earlier attempt that ended without one. A lighter model may take it.
"first_attempt": not ledger.feedback(kind)
and (attempt == 1 or kind in deliverables),
"allow_cloud_read": state.get("allow_cloud_read"),
"scope_skill": (state.get("scope") or {}).get("skill"),
# The spec as supplied, for the scope analyst to complete; with the Customer
# case the Execution belongs to, which is not the customer's to restate.
"scope_working": _scope_seed(state)
if kind == workflow.spec_deliverable
else None,
"working": dict(state.get("working", {}).get(kind, {})),
# Whether the user has been shown a draft of this Deliverable. Answering some
# other question about it, such as what to do after it stopped, is not that.
"shown": kind in state.get("drafts_shown", []),
# The Baseline's version of this Deliverable, in a Change request.
"baseline": before,
"baseline_working": dict((baseline or {}).get("working", {}).get(kind, {})),
# What the Baseline's version was written from, to tell whether it still stands.
"baseline_basis": spec.written_from((baseline or {}).get("deliverables"))
if baseline and spec.written_from
else None,
"context": {
**own_context,
**dict((hints or {}).get(kind) or {}),
# Sources the caller supplied that code did not read: documents for
# the Specialist that completes the spec. What it takes from one is
# its reading, recorded with the passage it came from.
**(
{"source_documents": dict(state.get("source_documents") or {})}
if kind == workflow.spec_deliverable and state.get("source_documents")
else {}
),
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
"contract": spec.contract(),
"dependencies": seen,
"feedback": list(ledger.feedback(kind)),
"answers": list(state.get("answers", {}).get(kind, [])),
"draft": dict(drafts[kind]) if kind in drafts else None,
# What the Specialist's own tools already hold from earlier attempts.
"working_copy": workspace.summary(
state.get("working", {}).get(kind, {}), spec.graph
),
"attempt": attempt,
# Skills carrying the organisation's standards or this engagement's
# constraints. Where one disagrees with general guidance, it wins.
"overlay_skills": sorted(state.get("overlay_skills") or {}),
# False when this Execution may not read the user's cloud environment.
"cloud_read": state.get("allow_cloud_read"),
# What the customer's scope spec states, leaves to the agent, and
# leaves unknown. None when no spec was supplied.
"scope": _agreed_scope(workflow, deliverables)
or (state.get("scope") or {}).get("context"),
# The references of its Skills this task needs, when that could be
# judged: where to start reading, not all it may read.
"read_first": list(read_first) if read_first else None,
# Set only in a Change request: what this Execution changes, and from what.
"baseline": _baseline_context(
workflow, kind, baseline, before, deliverables
)
if baseline
else None,
},
}
def _requirement_changes(
workflow: Workflow,
baseline: Mapping[str, Any],
deliverables: Mapping[str, Mapping[str, Any]],
) -> str:
"""In a Change request, what the customer now asks for differently, in words."""
policy = workflow.changes
if not baseline or policy is None:
return ""
return policy.in_words(policy.summary(baseline.get("deliverables"), deliverables))
async def _task_hints(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
started: Mapping[str, int],
decider: Decider | None,
) -> Mapping[str, Mapping[str, Any]]:
"""What the Workflow judges the Specialists being started need to be told."""
workflow = ledger.workflow
if workflow.task_hints is None:
return {}
return await workflow.task_hints(
Hints(
state=state,
feedback={kind: ledger.feedback(kind) for kind in started},
deliverables=deliverables,
started=started,
decider=decider,
changed=_requirement_changes(
workflow, state.get("baseline") or {}, deliverables
),
)
)
def _scope_seed(state: ExecutionState) -> dict[str, Any]:
"""The scope spec the analyst starts from: what was supplied, and the Customer
case this Execution already belongs to."""
seed = copy.deepcopy(
(state.get("scope") or {}).get("working") or {"values": {}, "states": {}}
)
seed.setdefault("values", {}).setdefault(CASE_FIELD, _case(state))
return seed
def _agreed_scope(
workflow: Workflow,
deliverables: Mapping[str, Mapping[str, Any]],
) -> dict[str, Any] | None:
"""The scope every later Specialist reads: the spec Deliverable's data file."""
text = (
(deliverables.get(workflow.spec_deliverable or "") or {})
.get("attachments", {})
.get(SCOPE_DATA_FILE, "")
)
try:
data = json.loads(text) if text else None
except ValueError:
return None
return data if isinstance(data, dict) else None
async def _reading_hints(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
started: Mapping[str, int],
decider: Decider | None,
references: Mapping[str, Mapping[str, Mapping[str, str]]],
threshold: float,
) -> dict[str, list[dict[str, str]]]:
"""For each Specialist being started whose Skills hold many references, the ones
its task needs (iteration 5 step 8)."""
if decider is None:
return {}
workflow = ledger.workflow
graph = workflow.deliverables
asked = {
kind: references[graph[kind].specialist]
for kind in started
if references.get(graph[kind].specialist)
}
if not asked:
return {}
design = workflow.describe_design(deliverables) if workflow.describe_design else ""
changed = _requirement_changes(workflow, state.get("baseline") or {}, deliverables)
def task(kind: str) -> str:
feedback = list(ledger.feedback(kind))
doing = "Revise" if feedback or changed else "Write"
parts = [
(
f"{doing} the {str(kind).replace('_', ' ')} Deliverable of "
f"{workflow.subject}, as its {graph[kind].specialist}."
)
]
if feedback:
parts.append("Feedback to address:\n" + "\n".join(feedback))
if changed:
# In a Change request the task is the change, not the whole design.
parts.append(f"What has changed since it was delivered:\n{changed}")
parts.append(design if design else f"The request:\n{state['workflow_request']}")
return "\n".join(parts)
chosen = await asyncio.gather(
*(
read_first(
decider,
task(kind),
skills,
threshold=threshold,
scope=state["execution_id"],
)
for kind, skills in asked.items()
)
)
return {kind: found for kind, found in zip(asked, chosen, strict=True) if found}
def _baseline_context(
workflow: Workflow,
kind: str,
baseline: Mapping[str, Any],
before: Mapping[str, Any] | None,
deliverables: Mapping[str, Mapping[str, Any]],
) -> dict[str, Any]:
policy = workflow.changes
summary = (
policy.summary(baseline.get("deliverables"), deliverables) if policy else None
)
return {
"execution_id": baseline.get("execution_id"),
"original_request": baseline.get("workflow_request"),
# The final Deliverable is assembled afresh from the revised Deliverables,
# so the Baseline's, the longest document there is, is not handed back.
"deliverable": dict(before) if before and kind != workflow.final else None,
# What has changed so far, upstream of this Deliverable.
"changes": None
if summary is None or (policy and policy.is_empty(summary))
else summary,
}
def _start_from(
workflow: Workflow,
kind: str,
before: Mapping[str, Any],
working: dict[str, Any],
basis: str | None = None,
) -> None:
"""Make the Baseline's version of a Deliverable its working copy."""
attachments = dict(before.get("attachments", {}))
spec = workflow.deliverables[kind]
if spec.graph and spec.graph.attachment in attachments:
workspace.keep_graph(working, attachments[spec.graph.attachment], spec.graph)
if spec.working_copy == kind:
working[workspace.FILES] = attachments
if basis:
# Kept as it stands when what it was written from is the same as before.
working[STANDING] = {
"basis": basis,
"feedback": [],
"content": before.get("content", ""),
"attachments": attachments,
}
def _outcome_record(
role: str, kind: str, attempt: int, outcome: Outcome
) -> dict[str, Any]:
base = {"role": role, "kind": kind, "attempt": attempt}
if isinstance(outcome, Submitted):
return {
**base,
"type": "submitted",
"content": outcome.content,
"attachments": dict(outcome.attachments),
}
if isinstance(outcome, NeedsInput):
return {
**base,
"type": "needs_input",
"questions": list(outcome.questions),
"permission": outcome.permission,
"draft": outcome.draft,
"draft_attachments": dict(outcome.draft_attachments),
}
if isinstance(outcome, BudgetExhausted):
return {**base, "type": "budget_exhausted", "summary": outcome.summary}
return {**base, "type": "stopped", "summary": _describe_unfinished(outcome)}
def _describe_unfinished(outcome: Outcome) -> str:
if isinstance(outcome, BudgetExhausted):
return f"I paused: {outcome.summary} How should I proceed?"
if isinstance(outcome, Stopped):
return outcome.message
if isinstance(outcome, NeedsInput):
return " ".join(outcome.questions)
return f"Ended with an unexpected outcome: {type(outcome).__name__}."
def automatic_reply(
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
) -> dict[str, Any]:
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
questions from scripted answers where one matches.
A scripted answer is (pattern, answer) or (pattern, answer, asker). It answers a
question its pattern matches; with an asker, only when that pattern also matches
who is asking (a Deliverable kind, or the orchestrator). The first that applies
wins. So facts written for requirements discovery are not given to a later
Specialist that only wants its draft confirmed.
"""
if "approval" in prompt:
return {
"approval": {"phase": prompt["approval"]["phase"], "decision": "approve"}
}
if "permission" in prompt:
return {"permission": {prompt["permission"]["tier"]: False}}
lines = []
unanswered = False
for item in prompt.get("questions", []):
asker = str(item.get("from", ""))
answer = next(
(
scripted[1]
for scripted in scripted_answers
if re.search(scripted[0], item["question"], re.IGNORECASE)
and (len(scripted) < 3 or re.search(scripted[2], asker, re.IGNORECASE))
),
None,
)
if answer is None:
unanswered = True
else:
lines.append(f"Q: {item['question']}\nA: {answer}")
if unanswered or not lines:
lines.append(NO_USER_REPLY)
return {"reply": "\n\n".join(lines)}
def _reply_text(reply: object) -> str:
if isinstance(reply, str):
return reply
if isinstance(reply, Mapping) and isinstance(reply.get("reply"), str):
return reply["reply"]
return str(reply)
def _own_words(text: str, asked: Sequence[str]) -> str:
"""A reply without the questions it quotes, and without Q: and A: labels."""
for question in sorted(asked, key=len, reverse=True):
text = text.replace(question, "")
lines = []
for line in text.splitlines():
line = re.sub(r"^\s*(?:Q|A)\s*:\s*", "", line).strip()
if line:
lines.append(line)
return "\n".join(lines)
def _progress(ledger: Ledger) -> str:
"""The work done so far, as one comparable value: each Deliverable's status and
attempt, and each Phase's status."""
data = ledger.to_dict()
return json.dumps(
{
"deliverables": {
kind: [record.get("status"), record.get("attempt")]
for kind, record in data["deliverables"].items()
},
"phases": data.get("phases"),
},
sort_keys=True,
default=str,
)
def _render_prompt(prompt: Mapping[str, Any]) -> str:
if "approval" in prompt:
approval = prompt["approval"]
return f"Please approve {approval['phase']}: {approval['summary']}"
if "permission" in prompt:
return f"May I use {prompt['permission']['tier']} tools?"
return "\n".join(item["question"] for item in prompt.get("questions", []))
def _case(state: ExecutionState) -> str:
return state.get("customer_case") or UNASSIGNED_CASE
def _scope_with_status(text: str, status: str) -> tuple[str, dict[str, Any] | None]:
"""A completed scope spec with its status set, and what the Execution's record
says of it. Unchanged, with no record, when it cannot be read."""
try:
document = yaml.safe_load(text)
except yaml.YAMLError:
return text, None
if not isinstance(document, dict):
return text, None
document.setdefault("metadata", {})["status"] = status
return (
yaml.safe_dump(document, sort_keys=False, allow_unicode=True),
{"status": status, "readiness": document.get("readiness")},
)
def _written_before(out_dir: Path) -> set[str]:
"""The files an earlier write of this Execution's folder recorded."""
try:
manifest = json.loads((out_dir / MANIFEST_FILENAME).read_text("utf-8"))
except (OSError, ValueError):
return set()
if not isinstance(manifest, dict):
return set()
names = {manifest.get("final_document"), manifest.get("changes")}
# Every file that write put there, whatever part of the Workflow wrote it.
names.update(manifest.get("files") or [])
for entry in (manifest.get("deliverables") or {}).values():
if isinstance(entry, dict):
names.add(entry.get("document"))
names.update(entry.get("attachments") or [])
return {
name
for name in names
if isinstance(name, str) and name and ".." not in Path(name).parts
}
def _write_outputs(
output_root: Path,
state: ExecutionState,
status: str,
ledger: Ledger,
changes: Mapping[str, Any] | None = None,
assessment: Mapping[str, Any] | None = None,
) -> bool:
"""Write an Execution's files under its Customer case, with a manifest that ties
the request to what was generated for it. Returns whether anything was written.
An Execution that ended with warnings is written too. Its manifest carries the
warnings and each Deliverable's status, so nothing unapproved reads as final.
"""
customer_case = _case(state)
execution_id = state["execution_id"]
out_dir = execution_dir(output_root, execution_id, customer_case)
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
if not deliverables:
return False
files: dict[str, str] = {}
by_kind: dict[str, dict[str, Any]] = {}
workflow = ledger.workflow
guide = deliverables.get(workflow.final)
if guide:
files[workflow.final_file] = guide["content"]
for kind, deliverable in deliverables.items():
document = f"deliverables/{kind}.md"
files[document] = deliverable["content"]
attachments = dict(deliverable.get("attachments", {}))
files.update(attachments)
by_kind[kind] = {
"document": document,
"attachments": sorted(attachments),
# submitted, or back in draft after something it depends on was reopened
"status": ledger.status(workflow.kind(kind)).value,
}
kept = {
kind: {key: memory[key] for key in workflow.kept_for_changes if memory.get(key)}
for kind, memory in state.get("working", {}).items()
}
kept = {kind: memory for kind, memory in kept.items() if memory}
if kept:
# Working material a Change request starts from, such as each pillar's advice.
files[WORKING_FILENAME] = json.dumps(kept, indent=2, ensure_ascii=False)
baseline = state.get("baseline") or None
if baseline and workflow.changes:
# What this Change request changed against its Baseline, worked out from
# the ids in the two versions' data files.
files[workflow.changes.file] = json.dumps(
changes
or workflow.changes.summary(baseline.get("deliverables"), deliverables),
indent=2,
ensure_ascii=False,
)
own = workflow.outputs(state, deliverables, files) if workflow.outputs else {}
scope_record: dict[str, Any] | None = None
if SCOPE_FILE in files:
# The completed scope spec says where it stands, here and beside the Intent.
spec_phase = workflow.deliverables[workflow.spec_deliverable or ""].phase
approved = ledger.phase_status(spec_phase) is PhaseStatus.APPROVED
files[SCOPE_FILE], scope_record = _scope_with_status(
files[SCOPE_FILE],
LOCKED if status == "completed" else APPROVED if approved else UNDER_REVIEW,
)
version = state.get("intent_version")
if version:
copy_path = (
intent_dir(output_root, customer_case)
/ f"{int(version):03d}{SCOPE_COMPLETED}"
)
copy_path.parent.mkdir(parents=True, exist_ok=True)
copy_path.write_text(files[SCOPE_FILE], encoding="utf-8")
# The folder is written more than once as an Execution goes. A file an earlier
# write left that this one does not have, such as a Terraform file removed in a
# Revision, must not stay beside the files that replaced it.
for stale in _written_before(out_dir) - set(files):
(out_dir / stale).unlink(missing_ok=True)
for name, content in files.items():
path = out_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
manifest = {
"customer_case": customer_case,
"execution_id": execution_id,
"session_id": state.get("session_id"),
"intent_version": state.get("intent_version"),
"workflow_request": state["workflow_request"],
# The Execution this one changes, for a Change request.
"baseline": (state.get("baseline") or {}).get("execution_id"),
# The overlay Skills it loaded, with the version of each.
"overlay_skills": dict(state.get("overlay_skills") or {}),
# The choices made for it, by name.
"options": dict(state.get("options") or {}),
# What its spec was digested from: each source's hash, and whether code
# read it or a Specialist did.
"sources": dict(state.get("sources") or {}),
# Where the agreed scope stands and how much of it the customer stated.
"scope": scope_record,
"status": status,
"warnings": list(state.get("warnings", [])),
# Flags a person should look at; nothing was done about them.
"risks": list(state.get("risks", [])),
# How the finished work was judged when it ended, if it was.
"assessment": dict(assessment) if assessment else None,
"phases": {
str(phase): ledger.phase_status(phase).value for phase in workflow.phases
},
# The final Deliverable, as a file of its own at the top of the folder.
"final_document": workflow.final_file if guide else None,
"changes": workflow.changes.file if baseline and workflow.changes else None,
"deliverables": by_kind,
# Everything written, so a later write can remove what it no longer has.
"files": sorted(files),
# What the Workflow records of its own.
**own,
}
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / MANIFEST_FILENAME).write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
return True