Files
gcp_solution_architecture_a…/app/tools/execution.py
Jonathan Boniface 58e778ca7a feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07:

- The validator records verdicts a few Checks at a time (verdict_put) and
  they are attached on submission. It timed out five times writing them in
  one reply. Verdicts on the same code survive a run cut short; verdicts on
  changed code are dropped. Its model timeout is 300 s.
- A schema review at submission, in code: every setting a Check names is
  looked up on its Element's resource type, and an unknown resource type is
  reported. Two Revisions in that run were for Checks no code could meet.
  On the two designs that passed validation it raises 2 of 96 Checks, both
  one metric name used as a value, so it is returned once and never blocks.
- Reviews run together; the Jev coverage review stays off.

Plans record both steps and the cancelled run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 09:39:13 +01:00

463 lines
17 KiB
Python

"""Execution tools: how agents act on an Execution.
Tools enforce the Ledger's rules at the moment an agent acts, and return refusals as
messages so a ReAct agent can recover. A tool ends the agent's turn by returning an
outcome.
"""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from typing import Any
from app.adk.runtime import (
FILES_SCHEMA,
NeedsInput,
Submitted,
ToolResult,
ToolSpec,
TurnEnded,
files_from_argument,
)
from app.states.ledger import Ledger, LedgerRefusal, PhaseStatus
from app.tools.workspace import (
FILES,
basis_of,
file_tools,
graph_text,
graph_tools,
keep_graph,
safe_relative,
verdict_tools,
verdicts_text,
working_files,
working_verdicts,
)
from app.workflows.chain import VERDICTS_FILE, read_graph
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
DeliverableKind,
PhaseId,
check_submission,
deliverables_in,
)
from app.workflows.target_architecture import (
DIAGRAM_ATTACHMENT,
GRAPH_ATTACHMENT,
GraphError,
TargetArchitecture,
parse_graph,
render_mermaid,
render_tables,
)
ORCHESTRATOR = "orchestrator"
logger = logging.getLogger(__name__)
_TARGET = DeliverableKind.TARGET_ARCHITECTURE
_CODE = DeliverableKind.DEPLOYMENT_GUIDANCE
CLOUD_READ = "cloud_read"
_NO_ARGS: Mapping[str, Any] = {"type": "object", "properties": {}}
def _schema(properties: Mapping[str, Any], required: Sequence[str]) -> dict[str, Any]:
return {
"type": "object",
"properties": dict(properties),
"required": list(required),
}
def _refused(reason: object) -> ToolResult:
return ToolResult(f"Refused: {reason}")
class OrchestratorToolbox:
"""The Orchestrator's tools, bound to a working copy of the Ledger for one turn."""
def __init__(
self,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
pending_questions: Sequence[Mapping[str, Any]],
) -> None:
self.ledger = ledger
self._deliverables = deliverables
self._pending_questions = pending_questions
self.started: dict[DeliverableKind, int] = {}
self.questions: list[dict[str, str]] = []
self.approval: dict[str, Any] | None = None
self.escalation: str | None = None
def tools(self) -> list[ToolSpec]:
return [
ToolSpec(
"ready_work",
"List Deliverables ready to delegate and Phases ready for Approval.",
_NO_ARGS,
self.ready_work,
tier="execution",
),
ToolSpec(
"delegate",
"Start Specialists on ready Deliverables, in parallel. Ends your turn.",
_schema(
{"kinds": {"type": "array", "items": {"type": "string"}}},
["kinds"],
),
self.delegate,
tier="execution",
),
ToolSpec(
"ask_user",
"Ask the user your questions together with any waiting Specialist "
"questions. Ends your turn.",
_schema(
{"questions": {"type": "array", "items": {"type": "string"}}}, []
),
self.ask_user,
tier="execution",
),
ToolSpec(
"request_approval",
"Ask the user to approve a Phase whose Deliverables are all "
"submitted. Ends your turn unless the Execution is pre-approved.",
_schema(
{"phase": {"type": "string"}, "summary": {"type": "string"}},
["phase", "summary"],
),
self.request_approval,
tier="execution",
),
ToolSpec(
"reopen",
"Revise a submitted Deliverable; everything depending on it returns "
"to draft.",
_schema(
{"kind": {"type": "string"}, "reason": {"type": "string"}},
["kind", "reason"],
),
self.reopen,
tier="execution",
),
ToolSpec(
"read_deliverable",
"Read a submitted Deliverable's content.",
_schema({"kind": {"type": "string"}}, ["kind"]),
self.read_deliverable,
tier="execution",
),
ToolSpec(
"finish",
"Complete the Execution once packaging is approved. Ends your turn.",
_NO_ARGS,
self.finish,
tier="execution",
),
]
async def ready_work(self) -> ToolResult:
ready = ", ".join(self.ledger.ready_work()) or "none"
phases = ", ".join(self.ledger.phases_ready_for_approval()) or "none"
return ToolResult(
f"Ready to delegate: {ready}. Phases ready for approval: {phases}."
)
async def delegate(self, kinds: Sequence[str]) -> ToolResult:
try:
parsed = [DeliverableKind(kind) for kind in kinds]
if not parsed:
raise LedgerRefusal("name at least one Deliverable")
started = self.ledger.start(parsed, actor=ORCHESTRATOR)
except (ValueError, LedgerRefusal) as exc:
return _refused(exc)
self.started.update(started)
return ToolResult(f"Started: {', '.join(started)}.", TurnEnded("delegate"))
async def ask_user(self, questions: Sequence[str] = ()) -> ToolResult:
waiting = sorted(
self._pending_questions,
key=lambda item: list(DEPENDENCY_GRAPH).index(
DeliverableKind(item["kind"])
),
)
combined = [
{"from": item["kind"], "question": question}
for item in waiting
for question in item["questions"]
]
combined += [{"from": ORCHESTRATOR, "question": q} for q in questions if q]
if not combined:
return _refused("there is nothing to ask the user")
self.questions = combined
return ToolResult("Asked the user.", TurnEnded("ask_user"))
async def request_approval(self, phase: str, summary: str) -> ToolResult:
try:
phase_id = PhaseId(phase)
except ValueError as exc:
return _refused(exc)
if self.ledger.phase_status(phase_id) is not PhaseStatus.AWAITING_APPROVAL:
try:
self.ledger.request_approval(phase_id, actor=ORCHESTRATOR)
except LedgerRefusal as exc:
return _refused(exc)
if self.ledger.phase_status(phase_id) is PhaseStatus.APPROVED:
return ToolResult(
f"Phase {phase_id} approved (pre-approved Execution)."
)
self.approval = {
"phase": phase_id.value,
"summary": summary,
"deliverables": [kind.value for kind in deliverables_in(phase_id)],
}
graph = self._target_architecture()
if graph and phase_id is DEPENDENCY_GRAPH[_TARGET].phase:
# What the user is approving before any code is written.
self.approval["target_architecture"] = graph.summary()
return ToolResult(
f"Asked the user to approve {phase_id}.", TurnEnded("request_approval")
)
def _target_architecture(self) -> TargetArchitecture | None:
deliverable = self._deliverables.get(_TARGET.value)
if not deliverable:
return None
try:
return parse_graph(deliverable.get("attachments", {})[GRAPH_ATTACHMENT])
except (KeyError, GraphError):
return None
async def reopen(self, kind: str, reason: str) -> ToolResult:
try:
result = self.ledger.reopen(
DeliverableKind(kind), reason, actor=ORCHESTRATOR
)
except (ValueError, LedgerRefusal) as exc:
return _refused(exc)
if result.escalation:
if self.ledger.pre_approved:
self.escalation = result.escalation
return ToolResult(result.escalation, TurnEnded("escalate"))
return ToolResult(
f"{result.escalation} Use ask_user to put this to the user."
)
withdrawn = ", ".join(result.withdrawn_approvals) or "none"
return ToolResult(
f"Reopened {kind}. Returned to draft: {', '.join(result.reset)}. "
f"Approvals withdrawn: {withdrawn}."
)
async def read_deliverable(self, kind: str) -> ToolResult:
deliverable = self._deliverables.get(kind)
if deliverable is None:
return ToolResult(f"{kind} has not been submitted.")
return ToolResult(deliverable["content"])
async def finish(self) -> ToolResult:
if not self.ledger.can_finish():
return _refused("packaging has not been approved")
return ToolResult("Finishing the Execution.", TurnEnded("finish"))
def specialist_tools(
kind: DeliverableKind,
extra: Sequence[ToolSpec],
allow_cloud_read: bool | None,
upstream: Mapping[str, Mapping[str, Any]] | None = None,
working: MutableMapping[str, Any] | None = None,
library: Mapping[str, Mapping[str, Any]] | None = None,
review: Callable[[Mapping[str, str]], Awaitable[Sequence[str]]] | None = None,
) -> list[ToolSpec]:
"""A Specialist's tools: submit, ask, its working copy, and its role's tools with
the cloud-read gate.
`upstream` is the Deliverables this one builds on; with it, a submission's links
along the Requirements chain are checked. `working` is the Deliverable's working
memory, where its working copy is kept. `library` is the earlier Deliverables
given to the Specialist as data files alone, to read whole when it needs one.
`review` looks at the meaning of a structurally complete submission; what it
raises is returned to the Specialist once per run, and never a second time.
"""
working = {} if working is None else working
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
own_files = DEPENDENCY_GRAPH[kind].working_copy is kind
reviewed = False
is_validation = kind is DeliverableKind.VALIDATION_RESULTS
judged = ((upstream or {}).get(_CODE.value) or {}).get("attachments") or {}
# The validator's own copy of the code is the same files; either marks them.
basis = basis_of(judged or working.get(FILES) or {})
if is_validation:
working_verdicts(working, basis) # drops verdicts given on other code
graph_now = read_graph(upstream)
async def submit_deliverable(content: str, attachments: Any = None) -> ToolResult:
try:
attachments = files_from_argument(attachments)
except TypeError as exc:
return _refused(f"attachments: {exc}")
# The working copy is the submission; a file passed here replaces its copy.
if own_files:
attachments = {**working_files(working), **attachments}
if is_graph and GRAPH_ATTACHMENT not in attachments and graph_text(working):
attachments[GRAPH_ATTACHMENT] = graph_text(working)
if (
is_validation
and VERDICTS_FILE not in attachments
and verdicts_text(working)
):
attachments[VERDICTS_FILE] = verdicts_text(working)
unsafe = [name for name in attachments if not safe_relative(name)]
if unsafe:
return _refused(
"attachment names must be relative paths without '..': "
+ ", ".join(unsafe)
)
missing = check_submission(kind, content, attachments, upstream)
if missing:
return ToolResult(
"Not submitted. The Deliverable is missing: " + "; ".join(missing)
)
if own_files:
working_files(working).update(attachments)
if is_graph:
keep_graph(working, attachments[GRAPH_ATTACHMENT])
nonlocal reviewed
if review and not reviewed:
reviewed = True
try:
concerns = await review(attachments)
except Exception: # a review must never stop a submission
logger.exception("The submission review failed; accepting")
concerns = ()
if concerns:
return ToolResult(
"Not submitted. A review found: "
+ "; ".join(concerns)
+ ". Look at each one: add what is missing, or leave it when "
"it is already covered. This review runs once, so your next "
"submission is accepted."
)
if is_graph:
# The tables and the diagram are views of the graph, so the engine
# writes them: they cannot disagree with it.
graph = parse_graph(attachments[GRAPH_ATTACHMENT])
content = content.rstrip() + "\n\n" + render_tables(graph)
attachments[DIAGRAM_ATTACHMENT] = render_mermaid(graph)
return ToolResult(f"Submitted {kind}.", Submitted(content, attachments))
async def read_deliverable(kind: str) -> ToolResult:
deliverable = (library or {}).get(kind)
if deliverable is None:
available = ", ".join(library or {}) or "none"
return _refused(f"{kind} is not one you can read here: {available}")
attached = ", ".join(deliverable.get("attachments", {})) or "none"
return ToolResult(
f"{deliverable.get('content', '')}\n\nAttachments: {attached}"
)
async def needs_input(
questions: Sequence[str], draft: str | None = None, attachments: Any = None
) -> ToolResult:
questions = tuple(q for q in questions if q)
if not questions:
return _refused("ask at least one question")
try:
draft_attachments = files_from_argument(attachments)
except TypeError as exc:
return _refused(f"attachments: {exc}")
return ToolResult(
"Questions sent to the user.",
NeedsInput(
questions, draft=draft or None, draft_attachments=draft_attachments
),
)
return [
ToolSpec(
"submit_deliverable",
f"Submit the {kind} Deliverable once it meets its contract. Ends your turn.",
_schema(
{
"content": {"type": "string"},
"attachments": FILES_SCHEMA,
},
["content"],
),
submit_deliverable,
tier="execution",
),
ToolSpec(
"needs_input",
"Ask the user questions you can't proceed without. Ends your turn. When "
"you are asking them to confirm a draft, pass it as `draft` (with its "
"`attachments`): it is returned to you with their answers.",
_schema(
{
"questions": {"type": "array", "items": {"type": "string"}},
"draft": {"type": "string"},
"attachments": FILES_SCHEMA,
},
["questions"],
),
needs_input,
tier="execution",
),
*(
[
ToolSpec(
"read_deliverable",
"Read the full text of an earlier Deliverable that `dependencies` "
"gives you as its data file alone: "
f"{', '.join(library)}. Use it only when the ids are not enough.",
_schema({"kind": {"type": "string"}}, ["kind"]),
read_deliverable,
tier="execution",
)
]
if library
else ()
),
*(graph_tools(working) if is_graph else ()),
*(
verdict_tools(working, basis, graph_now.check_ids if graph_now else ())
if is_validation
else ()
),
*(file_tools(working) if own_files else ()),
*(gate_cloud_read(tool, allow_cloud_read) for tool in extra),
]
def gate_cloud_read(tool: ToolSpec, allow_cloud_read: bool | None) -> ToolSpec:
"""Apply the Execution's cloud-read permission to a tool in that tier."""
if tool.tier != CLOUD_READ or allow_cloud_read is True:
return tool
if allow_cloud_read is False:
async def refuse(**_: Any) -> ToolResult:
return _refused(
"reading live cloud resources is not permitted for this Execution. "
"Work from what the user has told you and record the gap."
)
handler = refuse
else:
async def ask_permission(**_: Any) -> ToolResult:
return ToolResult(
"Permission to read live cloud resources is needed first.",
NeedsInput(
(
f"May I read live resources in your cloud environment? ({tool.name})",
),
permission=CLOUD_READ,
),
)
handler = ask_permission
return ToolSpec(tool.name, tool.description, tool.parameters, handler, tool.tier)