From the case 02 run cancelled at 66 minutes on 2026-10-07: - The validator records verdicts a few Checks at a time (verdict_put) and they are attached on submission. It timed out five times writing them in one reply. Verdicts on the same code survive a run cut short; verdicts on changed code are dropped. Its model timeout is 300 s. - A schema review at submission, in code: every setting a Check names is looked up on its Element's resource type, and an unknown resource type is reported. Two Revisions in that run were for Checks no code could meet. On the two designs that passed validation it raises 2 of 96 Checks, both one metric name used as a value, so it is returned once and never blocks. - Reviews run together; the Jev coverage review stays off. Plans record both steps and the cancelled run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
463 lines
17 KiB
Python
463 lines
17 KiB
Python
"""Execution tools: how agents act on an Execution.
|
|
|
|
Tools enforce the Ledger's rules at the moment an agent acts, and return refusals as
|
|
messages so a ReAct agent can recover. A tool ends the agent's turn by returning an
|
|
outcome.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
FILES_SCHEMA,
|
|
NeedsInput,
|
|
Submitted,
|
|
ToolResult,
|
|
ToolSpec,
|
|
TurnEnded,
|
|
files_from_argument,
|
|
)
|
|
from app.states.ledger import Ledger, LedgerRefusal, PhaseStatus
|
|
from app.tools.workspace import (
|
|
FILES,
|
|
basis_of,
|
|
file_tools,
|
|
graph_text,
|
|
graph_tools,
|
|
keep_graph,
|
|
safe_relative,
|
|
verdict_tools,
|
|
verdicts_text,
|
|
working_files,
|
|
working_verdicts,
|
|
)
|
|
from app.workflows.chain import VERDICTS_FILE, read_graph
|
|
from app.workflows.deliverables import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
check_submission,
|
|
deliverables_in,
|
|
)
|
|
from app.workflows.target_architecture import (
|
|
DIAGRAM_ATTACHMENT,
|
|
GRAPH_ATTACHMENT,
|
|
GraphError,
|
|
TargetArchitecture,
|
|
parse_graph,
|
|
render_mermaid,
|
|
render_tables,
|
|
)
|
|
|
|
ORCHESTRATOR = "orchestrator"
|
|
logger = logging.getLogger(__name__)
|
|
_TARGET = DeliverableKind.TARGET_ARCHITECTURE
|
|
_CODE = DeliverableKind.DEPLOYMENT_GUIDANCE
|
|
CLOUD_READ = "cloud_read"
|
|
|
|
_NO_ARGS: Mapping[str, Any] = {"type": "object", "properties": {}}
|
|
|
|
|
|
def _schema(properties: Mapping[str, Any], required: Sequence[str]) -> dict[str, Any]:
|
|
return {
|
|
"type": "object",
|
|
"properties": dict(properties),
|
|
"required": list(required),
|
|
}
|
|
|
|
|
|
def _refused(reason: object) -> ToolResult:
|
|
return ToolResult(f"Refused: {reason}")
|
|
|
|
|
|
class OrchestratorToolbox:
|
|
"""The Orchestrator's tools, bound to a working copy of the Ledger for one turn."""
|
|
|
|
def __init__(
|
|
self,
|
|
ledger: Ledger,
|
|
deliverables: Mapping[str, Mapping[str, Any]],
|
|
pending_questions: Sequence[Mapping[str, Any]],
|
|
) -> None:
|
|
self.ledger = ledger
|
|
self._deliverables = deliverables
|
|
self._pending_questions = pending_questions
|
|
self.started: dict[DeliverableKind, int] = {}
|
|
self.questions: list[dict[str, str]] = []
|
|
self.approval: dict[str, Any] | None = None
|
|
self.escalation: str | None = None
|
|
|
|
def tools(self) -> list[ToolSpec]:
|
|
return [
|
|
ToolSpec(
|
|
"ready_work",
|
|
"List Deliverables ready to delegate and Phases ready for Approval.",
|
|
_NO_ARGS,
|
|
self.ready_work,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"delegate",
|
|
"Start Specialists on ready Deliverables, in parallel. Ends your turn.",
|
|
_schema(
|
|
{"kinds": {"type": "array", "items": {"type": "string"}}},
|
|
["kinds"],
|
|
),
|
|
self.delegate,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"ask_user",
|
|
"Ask the user your questions together with any waiting Specialist "
|
|
"questions. Ends your turn.",
|
|
_schema(
|
|
{"questions": {"type": "array", "items": {"type": "string"}}}, []
|
|
),
|
|
self.ask_user,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"request_approval",
|
|
"Ask the user to approve a Phase whose Deliverables are all "
|
|
"submitted. Ends your turn unless the Execution is pre-approved.",
|
|
_schema(
|
|
{"phase": {"type": "string"}, "summary": {"type": "string"}},
|
|
["phase", "summary"],
|
|
),
|
|
self.request_approval,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"reopen",
|
|
"Revise a submitted Deliverable; everything depending on it returns "
|
|
"to draft.",
|
|
_schema(
|
|
{"kind": {"type": "string"}, "reason": {"type": "string"}},
|
|
["kind", "reason"],
|
|
),
|
|
self.reopen,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"read_deliverable",
|
|
"Read a submitted Deliverable's content.",
|
|
_schema({"kind": {"type": "string"}}, ["kind"]),
|
|
self.read_deliverable,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"finish",
|
|
"Complete the Execution once packaging is approved. Ends your turn.",
|
|
_NO_ARGS,
|
|
self.finish,
|
|
tier="execution",
|
|
),
|
|
]
|
|
|
|
async def ready_work(self) -> ToolResult:
|
|
ready = ", ".join(self.ledger.ready_work()) or "none"
|
|
phases = ", ".join(self.ledger.phases_ready_for_approval()) or "none"
|
|
return ToolResult(
|
|
f"Ready to delegate: {ready}. Phases ready for approval: {phases}."
|
|
)
|
|
|
|
async def delegate(self, kinds: Sequence[str]) -> ToolResult:
|
|
try:
|
|
parsed = [DeliverableKind(kind) for kind in kinds]
|
|
if not parsed:
|
|
raise LedgerRefusal("name at least one Deliverable")
|
|
started = self.ledger.start(parsed, actor=ORCHESTRATOR)
|
|
except (ValueError, LedgerRefusal) as exc:
|
|
return _refused(exc)
|
|
self.started.update(started)
|
|
return ToolResult(f"Started: {', '.join(started)}.", TurnEnded("delegate"))
|
|
|
|
async def ask_user(self, questions: Sequence[str] = ()) -> ToolResult:
|
|
waiting = sorted(
|
|
self._pending_questions,
|
|
key=lambda item: list(DEPENDENCY_GRAPH).index(
|
|
DeliverableKind(item["kind"])
|
|
),
|
|
)
|
|
combined = [
|
|
{"from": item["kind"], "question": question}
|
|
for item in waiting
|
|
for question in item["questions"]
|
|
]
|
|
combined += [{"from": ORCHESTRATOR, "question": q} for q in questions if q]
|
|
if not combined:
|
|
return _refused("there is nothing to ask the user")
|
|
self.questions = combined
|
|
return ToolResult("Asked the user.", TurnEnded("ask_user"))
|
|
|
|
async def request_approval(self, phase: str, summary: str) -> ToolResult:
|
|
try:
|
|
phase_id = PhaseId(phase)
|
|
except ValueError as exc:
|
|
return _refused(exc)
|
|
if self.ledger.phase_status(phase_id) is not PhaseStatus.AWAITING_APPROVAL:
|
|
try:
|
|
self.ledger.request_approval(phase_id, actor=ORCHESTRATOR)
|
|
except LedgerRefusal as exc:
|
|
return _refused(exc)
|
|
if self.ledger.phase_status(phase_id) is PhaseStatus.APPROVED:
|
|
return ToolResult(
|
|
f"Phase {phase_id} approved (pre-approved Execution)."
|
|
)
|
|
self.approval = {
|
|
"phase": phase_id.value,
|
|
"summary": summary,
|
|
"deliverables": [kind.value for kind in deliverables_in(phase_id)],
|
|
}
|
|
graph = self._target_architecture()
|
|
if graph and phase_id is DEPENDENCY_GRAPH[_TARGET].phase:
|
|
# What the user is approving before any code is written.
|
|
self.approval["target_architecture"] = graph.summary()
|
|
return ToolResult(
|
|
f"Asked the user to approve {phase_id}.", TurnEnded("request_approval")
|
|
)
|
|
|
|
def _target_architecture(self) -> TargetArchitecture | None:
|
|
deliverable = self._deliverables.get(_TARGET.value)
|
|
if not deliverable:
|
|
return None
|
|
try:
|
|
return parse_graph(deliverable.get("attachments", {})[GRAPH_ATTACHMENT])
|
|
except (KeyError, GraphError):
|
|
return None
|
|
|
|
async def reopen(self, kind: str, reason: str) -> ToolResult:
|
|
try:
|
|
result = self.ledger.reopen(
|
|
DeliverableKind(kind), reason, actor=ORCHESTRATOR
|
|
)
|
|
except (ValueError, LedgerRefusal) as exc:
|
|
return _refused(exc)
|
|
if result.escalation:
|
|
if self.ledger.pre_approved:
|
|
self.escalation = result.escalation
|
|
return ToolResult(result.escalation, TurnEnded("escalate"))
|
|
return ToolResult(
|
|
f"{result.escalation} Use ask_user to put this to the user."
|
|
)
|
|
withdrawn = ", ".join(result.withdrawn_approvals) or "none"
|
|
return ToolResult(
|
|
f"Reopened {kind}. Returned to draft: {', '.join(result.reset)}. "
|
|
f"Approvals withdrawn: {withdrawn}."
|
|
)
|
|
|
|
async def read_deliverable(self, kind: str) -> ToolResult:
|
|
deliverable = self._deliverables.get(kind)
|
|
if deliverable is None:
|
|
return ToolResult(f"{kind} has not been submitted.")
|
|
return ToolResult(deliverable["content"])
|
|
|
|
async def finish(self) -> ToolResult:
|
|
if not self.ledger.can_finish():
|
|
return _refused("packaging has not been approved")
|
|
return ToolResult("Finishing the Execution.", TurnEnded("finish"))
|
|
|
|
|
|
def specialist_tools(
|
|
kind: DeliverableKind,
|
|
extra: Sequence[ToolSpec],
|
|
allow_cloud_read: bool | None,
|
|
upstream: Mapping[str, Mapping[str, Any]] | None = None,
|
|
working: MutableMapping[str, Any] | None = None,
|
|
library: Mapping[str, Mapping[str, Any]] | None = None,
|
|
review: Callable[[Mapping[str, str]], Awaitable[Sequence[str]]] | None = None,
|
|
) -> list[ToolSpec]:
|
|
"""A Specialist's tools: submit, ask, its working copy, and its role's tools with
|
|
the cloud-read gate.
|
|
|
|
`upstream` is the Deliverables this one builds on; with it, a submission's links
|
|
along the Requirements chain are checked. `working` is the Deliverable's working
|
|
memory, where its working copy is kept. `library` is the earlier Deliverables
|
|
given to the Specialist as data files alone, to read whole when it needs one.
|
|
`review` looks at the meaning of a structurally complete submission; what it
|
|
raises is returned to the Specialist once per run, and never a second time.
|
|
"""
|
|
working = {} if working is None else working
|
|
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
|
|
own_files = DEPENDENCY_GRAPH[kind].working_copy is kind
|
|
reviewed = False
|
|
is_validation = kind is DeliverableKind.VALIDATION_RESULTS
|
|
judged = ((upstream or {}).get(_CODE.value) or {}).get("attachments") or {}
|
|
# The validator's own copy of the code is the same files; either marks them.
|
|
basis = basis_of(judged or working.get(FILES) or {})
|
|
if is_validation:
|
|
working_verdicts(working, basis) # drops verdicts given on other code
|
|
graph_now = read_graph(upstream)
|
|
|
|
async def submit_deliverable(content: str, attachments: Any = None) -> ToolResult:
|
|
try:
|
|
attachments = files_from_argument(attachments)
|
|
except TypeError as exc:
|
|
return _refused(f"attachments: {exc}")
|
|
# The working copy is the submission; a file passed here replaces its copy.
|
|
if own_files:
|
|
attachments = {**working_files(working), **attachments}
|
|
if is_graph and GRAPH_ATTACHMENT not in attachments and graph_text(working):
|
|
attachments[GRAPH_ATTACHMENT] = graph_text(working)
|
|
if (
|
|
is_validation
|
|
and VERDICTS_FILE not in attachments
|
|
and verdicts_text(working)
|
|
):
|
|
attachments[VERDICTS_FILE] = verdicts_text(working)
|
|
unsafe = [name for name in attachments if not safe_relative(name)]
|
|
if unsafe:
|
|
return _refused(
|
|
"attachment names must be relative paths without '..': "
|
|
+ ", ".join(unsafe)
|
|
)
|
|
missing = check_submission(kind, content, attachments, upstream)
|
|
if missing:
|
|
return ToolResult(
|
|
"Not submitted. The Deliverable is missing: " + "; ".join(missing)
|
|
)
|
|
if own_files:
|
|
working_files(working).update(attachments)
|
|
if is_graph:
|
|
keep_graph(working, attachments[GRAPH_ATTACHMENT])
|
|
nonlocal reviewed
|
|
if review and not reviewed:
|
|
reviewed = True
|
|
try:
|
|
concerns = await review(attachments)
|
|
except Exception: # a review must never stop a submission
|
|
logger.exception("The submission review failed; accepting")
|
|
concerns = ()
|
|
if concerns:
|
|
return ToolResult(
|
|
"Not submitted. A review found: "
|
|
+ "; ".join(concerns)
|
|
+ ". Look at each one: add what is missing, or leave it when "
|
|
"it is already covered. This review runs once, so your next "
|
|
"submission is accepted."
|
|
)
|
|
if is_graph:
|
|
# The tables and the diagram are views of the graph, so the engine
|
|
# writes them: they cannot disagree with it.
|
|
graph = parse_graph(attachments[GRAPH_ATTACHMENT])
|
|
content = content.rstrip() + "\n\n" + render_tables(graph)
|
|
attachments[DIAGRAM_ATTACHMENT] = render_mermaid(graph)
|
|
return ToolResult(f"Submitted {kind}.", Submitted(content, attachments))
|
|
|
|
async def read_deliverable(kind: str) -> ToolResult:
|
|
deliverable = (library or {}).get(kind)
|
|
if deliverable is None:
|
|
available = ", ".join(library or {}) or "none"
|
|
return _refused(f"{kind} is not one you can read here: {available}")
|
|
attached = ", ".join(deliverable.get("attachments", {})) or "none"
|
|
return ToolResult(
|
|
f"{deliverable.get('content', '')}\n\nAttachments: {attached}"
|
|
)
|
|
|
|
async def needs_input(
|
|
questions: Sequence[str], draft: str | None = None, attachments: Any = None
|
|
) -> ToolResult:
|
|
questions = tuple(q for q in questions if q)
|
|
if not questions:
|
|
return _refused("ask at least one question")
|
|
try:
|
|
draft_attachments = files_from_argument(attachments)
|
|
except TypeError as exc:
|
|
return _refused(f"attachments: {exc}")
|
|
return ToolResult(
|
|
"Questions sent to the user.",
|
|
NeedsInput(
|
|
questions, draft=draft or None, draft_attachments=draft_attachments
|
|
),
|
|
)
|
|
|
|
return [
|
|
ToolSpec(
|
|
"submit_deliverable",
|
|
f"Submit the {kind} Deliverable once it meets its contract. Ends your turn.",
|
|
_schema(
|
|
{
|
|
"content": {"type": "string"},
|
|
"attachments": FILES_SCHEMA,
|
|
},
|
|
["content"],
|
|
),
|
|
submit_deliverable,
|
|
tier="execution",
|
|
),
|
|
ToolSpec(
|
|
"needs_input",
|
|
"Ask the user questions you can't proceed without. Ends your turn. When "
|
|
"you are asking them to confirm a draft, pass it as `draft` (with its "
|
|
"`attachments`): it is returned to you with their answers.",
|
|
_schema(
|
|
{
|
|
"questions": {"type": "array", "items": {"type": "string"}},
|
|
"draft": {"type": "string"},
|
|
"attachments": FILES_SCHEMA,
|
|
},
|
|
["questions"],
|
|
),
|
|
needs_input,
|
|
tier="execution",
|
|
),
|
|
*(
|
|
[
|
|
ToolSpec(
|
|
"read_deliverable",
|
|
"Read the full text of an earlier Deliverable that `dependencies` "
|
|
"gives you as its data file alone: "
|
|
f"{', '.join(library)}. Use it only when the ids are not enough.",
|
|
_schema({"kind": {"type": "string"}}, ["kind"]),
|
|
read_deliverable,
|
|
tier="execution",
|
|
)
|
|
]
|
|
if library
|
|
else ()
|
|
),
|
|
*(graph_tools(working) if is_graph else ()),
|
|
*(
|
|
verdict_tools(working, basis, graph_now.check_ids if graph_now else ())
|
|
if is_validation
|
|
else ()
|
|
),
|
|
*(file_tools(working) if own_files else ()),
|
|
*(gate_cloud_read(tool, allow_cloud_read) for tool in extra),
|
|
]
|
|
|
|
|
|
def gate_cloud_read(tool: ToolSpec, allow_cloud_read: bool | None) -> ToolSpec:
|
|
"""Apply the Execution's cloud-read permission to a tool in that tier."""
|
|
if tool.tier != CLOUD_READ or allow_cloud_read is True:
|
|
return tool
|
|
|
|
if allow_cloud_read is False:
|
|
|
|
async def refuse(**_: Any) -> ToolResult:
|
|
return _refused(
|
|
"reading live cloud resources is not permitted for this Execution. "
|
|
"Work from what the user has told you and record the gap."
|
|
)
|
|
|
|
handler = refuse
|
|
else:
|
|
|
|
async def ask_permission(**_: Any) -> ToolResult:
|
|
return ToolResult(
|
|
"Permission to read live cloud resources is needed first.",
|
|
NeedsInput(
|
|
(
|
|
f"May I read live resources in your cloud environment? ({tool.name})",
|
|
),
|
|
permission=CLOUD_READ,
|
|
),
|
|
)
|
|
|
|
handler = ask_permission
|
|
|
|
return ToolSpec(tool.name, tool.description, tool.parameters, handler, tool.tier)
|