The Orchestration engine keeps every completed agent Execution it relays, by tenant, Customer case and Execution, and serves its record and files at signed, expiring links. A KAB workflow step, which takes one output per step, is given one JSON document of links (`deliver: links`); other callers still get each file, and what they are given is kept either way. - store.py, links.py, delivery.py, serving.py; the relay keeps and links - the card declares `deliver`, `sources` and an output schema - image: STORE_DIR on a /data volume; PUBLIC_URL and LINK_SECRET settings - option 1's tests replaced by tests of the links document Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
619 lines
23 KiB
Python
619 lines
23 KiB
Python
"""Dispatching a request to agents and relaying everything back (ADR-0003).
|
|
|
|
Each of the Orchestration engine's tasks runs a plan: one agent, or a sequence of
|
|
them. At any moment it stands for one task of one agent. What the agent says, a
|
|
status, a question, an Approval it asks for, a file, goes to the caller unchanged;
|
|
what the caller replies goes to the agent unchanged. When an agent of a sequence
|
|
completes, its files go to the next as `sources` (step F4).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import uuid
|
|
from collections.abc import Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from a2a.client import Client, ClientConfig, ClientFactory
|
|
from a2a.helpers.proto_helpers import (
|
|
get_data_parts,
|
|
new_data_part,
|
|
new_task,
|
|
new_text_part,
|
|
)
|
|
from a2a.server.agent_execution import AgentExecutor, RequestContext
|
|
from a2a.server.events import EventQueue
|
|
from a2a.server.tasks import TaskUpdater
|
|
from a2a.types import (
|
|
Artifact,
|
|
CancelTaskRequest,
|
|
Message,
|
|
Part,
|
|
Role,
|
|
SendMessageRequest,
|
|
Task,
|
|
TaskState,
|
|
TaskStatus,
|
|
)
|
|
from google.protobuf import json_format
|
|
|
|
from orchestration.agents import Agent
|
|
from orchestration.agents import Sequence as AgentSequence
|
|
from orchestration.delivery import Delivery
|
|
from orchestration.routing import WORK, route
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The states after which the agent waits for the caller, or is done.
|
|
_ENDS_A_TURN = {
|
|
TaskState.TASK_STATE_INPUT_REQUIRED,
|
|
TaskState.TASK_STATE_AUTH_REQUIRED,
|
|
TaskState.TASK_STATE_COMPLETED,
|
|
TaskState.TASK_STATE_FAILED,
|
|
TaskState.TASK_STATE_CANCELED,
|
|
TaskState.TASK_STATE_REJECTED,
|
|
}
|
|
_PAUSES = {TaskState.TASK_STATE_INPUT_REQUIRED, TaskState.TASK_STATE_AUTH_REQUIRED}
|
|
# What every agent of a sequence is sent of the caller's fields. The rest go to the
|
|
# first agent only, or to the agent whose work names them: `operating_model: {...}`.
|
|
_CARRIED = ("customer_case", "pre_approved", "request")
|
|
# The Orchestration engine's own record that an agent's Execution completed, sent
|
|
# with its files to the next agent.
|
|
RECORD = "execution.json"
|
|
# How a caller is given what agents delivered (ADR-0004): one document of links to
|
|
# what was kept, or each file as the agent sent it.
|
|
DELIVER, LINKS, FILES = "deliver", "links", "files"
|
|
# The one artifact a caller given links receives.
|
|
DOCUMENT = "result.json"
|
|
# What a KAB workflow step sends with every call, by which it is known: it takes
|
|
# one output per step, so it is given links unless it asks otherwise.
|
|
_PLATFORM_STEP = ("workflow_id", "step_number")
|
|
|
|
|
|
@dataclass
|
|
class Plan:
|
|
"""What one of our tasks runs: the agents, in turn, and the caller's fields."""
|
|
|
|
steps: tuple[Agent, ...]
|
|
fields: dict[str, Any] # the request's metadata, but for `work` and `deliver`
|
|
text: str # the request, as the agents are sent it
|
|
other_parts: list[Part] = field(default_factory=list) # data and files sent with it
|
|
work: str = "" # the work the caller named, an agent's or a sequence's
|
|
deliver: str = FILES
|
|
# Each agent that completed, by its work: its Execution, how it ended, links.
|
|
results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
|
|
def metadata(self, step: int, sources: Mapping[str, str] | None) -> dict[str, Any]:
|
|
"""The metadata for the agent at `step`."""
|
|
works = {agent.work for agent in self.steps}
|
|
if step == 0:
|
|
sent = {k: v for k, v in self.fields.items() if k not in works}
|
|
else:
|
|
sent = {k: self.fields[k] for k in _CARRIED if k in self.fields}
|
|
own = self.fields.get(self.steps[step].work)
|
|
if isinstance(own, Mapping):
|
|
sent.update(own)
|
|
if sources is not None:
|
|
sent["sources"] = dict(sources)
|
|
if isinstance(sent.get("sources"), Mapping):
|
|
sent["sources"] = _as_files(sent["sources"])
|
|
return sent
|
|
|
|
|
|
@dataclass
|
|
class Waiting:
|
|
"""The agent's task one of our tasks stands for, and where it is in its plan."""
|
|
|
|
plan: Plan
|
|
step: int
|
|
task_id: str
|
|
context_id: str
|
|
|
|
@property
|
|
def agent(self) -> Agent:
|
|
return self.plan.steps[self.step]
|
|
|
|
|
|
class AgentClients:
|
|
"""One A2A client per agent, made from its card the first time it is called."""
|
|
|
|
def __init__(self, httpx_client: httpx.AsyncClient) -> None:
|
|
self._factory = ClientFactory(
|
|
ClientConfig(streaming=True, httpx_client=httpx_client)
|
|
)
|
|
self._clients: dict[str, Client] = {}
|
|
|
|
async def for_agent(self, agent: Agent) -> Client:
|
|
if agent.work not in self._clients:
|
|
self._clients[agent.work] = await self._factory.create_from_url(agent.url)
|
|
return self._clients[agent.work]
|
|
|
|
def forget(self, agent: Agent) -> None:
|
|
"""Read the agent's card again next time: it may have moved or changed."""
|
|
self._clients.pop(agent.work, None)
|
|
|
|
|
|
class RelayExecutor(AgentExecutor):
|
|
def __init__(
|
|
self,
|
|
agents: Sequence[Agent],
|
|
clients: AgentClients,
|
|
heartbeat_seconds: float,
|
|
sequences: Sequence[AgentSequence] = (),
|
|
delivery: Delivery | None = None,
|
|
) -> None:
|
|
self._delivery = delivery
|
|
self._agents = list(agents)
|
|
self._sequences = list(sequences)
|
|
self._clients = clients
|
|
self._heartbeat_seconds = heartbeat_seconds
|
|
self._waiting: dict[str, Waiting] = {}
|
|
|
|
async def execute(self, context: RequestContext, event_queue: EventQueue) -> None:
|
|
task_id = context.task_id or ""
|
|
updater = TaskUpdater(event_queue, task_id, context.context_id or "")
|
|
parts = list(context.message.parts) if context.message is not None else []
|
|
if context.current_task is not None:
|
|
await self._resume(task_id, context, parts, updater)
|
|
return
|
|
await event_queue.enqueue_event(
|
|
new_task(
|
|
task_id,
|
|
context.context_id or "",
|
|
TaskState.TASK_STATE_SUBMITTED,
|
|
history=[context.message] if context.message is not None else [],
|
|
)
|
|
)
|
|
await updater.start_work()
|
|
options = _options(context)
|
|
decision = route(options, self._agents, self._sequences)
|
|
logger.info("Task %s: %s", task_id, decision.reason)
|
|
if not decision.agents:
|
|
await updater.complete(
|
|
updater.new_agent_message(
|
|
[
|
|
new_text_part(decision.reason),
|
|
new_data_part(
|
|
{
|
|
"status": "not_started",
|
|
"work": [
|
|
*(a.work for a in self._agents),
|
|
*(s.work for s in self._sequences),
|
|
],
|
|
}
|
|
),
|
|
]
|
|
)
|
|
)
|
|
return
|
|
fields = {k: v for k, v in options.items() if k not in (WORK, DELIVER)}
|
|
texts = [part.text for part in parts if part.HasField("text")]
|
|
# A KAB step sends its fields as metadata and, as the text, all of them
|
|
# again as one JSON document. The agents are sent the request as their text.
|
|
plan = Plan(
|
|
steps=decision.agents,
|
|
fields=fields,
|
|
text=str(fields["request"]) if fields.get("request") else "\n".join(texts),
|
|
other_parts=[part for part in parts if not part.HasField("text")],
|
|
work=str(options.get(WORK) or decision.agents[0].work),
|
|
deliver=_deliver(options),
|
|
)
|
|
await self._run(task_id, plan, 0, self._opening(plan, 0, None), updater)
|
|
|
|
async def _resume(
|
|
self,
|
|
task_id: str,
|
|
context: RequestContext,
|
|
parts: list[Part],
|
|
updater: TaskUpdater,
|
|
) -> None:
|
|
waiting = self._waiting.get(task_id)
|
|
if waiting is None:
|
|
await updater.failed(
|
|
updater.new_agent_message(
|
|
[
|
|
new_text_part(
|
|
"This task's place in the agent's work is no longer "
|
|
"held here, as after a restart. Start the request again."
|
|
)
|
|
]
|
|
)
|
|
)
|
|
return
|
|
await updater.start_work()
|
|
message = Message(
|
|
message_id=str(uuid.uuid4()),
|
|
role=Role.ROLE_USER,
|
|
parts=parts,
|
|
task_id=waiting.task_id,
|
|
context_id=waiting.context_id,
|
|
)
|
|
metadata = _options(context)
|
|
if metadata:
|
|
message.metadata.update(metadata)
|
|
await self._run(task_id, waiting.plan, waiting.step, message, updater)
|
|
|
|
@staticmethod
|
|
def _opening(plan: Plan, step: int, sources: Mapping[str, str] | None) -> Message:
|
|
"""The first message to the agent at `step`."""
|
|
message = Message(
|
|
message_id=str(uuid.uuid4()),
|
|
role=Role.ROLE_USER,
|
|
parts=[new_text_part(plan.text), *plan.other_parts]
|
|
if plan.text
|
|
else list(plan.other_parts),
|
|
)
|
|
metadata = plan.metadata(step, sources)
|
|
if metadata:
|
|
message.metadata.update(metadata)
|
|
return message
|
|
|
|
async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None:
|
|
task_id = context.task_id or ""
|
|
waiting = self._waiting.pop(task_id, None)
|
|
if waiting is not None:
|
|
with contextlib.suppress(Exception):
|
|
client = await self._clients.for_agent(waiting.agent)
|
|
await client.cancel_task(CancelTaskRequest(id=waiting.task_id))
|
|
await TaskUpdater(event_queue, task_id, context.context_id or "").cancel()
|
|
|
|
async def _run(
|
|
self,
|
|
task_id: str,
|
|
plan: Plan,
|
|
step: int,
|
|
message: Message,
|
|
updater: TaskUpdater,
|
|
) -> None:
|
|
"""Relay one turn of the agent at `step`; when it completes and another
|
|
agent follows, start that one with its files."""
|
|
agent = plan.steps[step]
|
|
heartbeat = asyncio.create_task(self._beat(agent, updater))
|
|
relayed: set[str] = set() # artifact ids already passed on
|
|
files: dict[str, str] = {} # what it delivered, by name, for the next agent
|
|
guide: list[str] = [] # the file it sent as its document, if any
|
|
relay_files = plan.deliver == FILES
|
|
try:
|
|
client = await self._clients.for_agent(agent)
|
|
async for event in client.send_message(SendMessageRequest(message=message)):
|
|
status: TaskStatus | None = None
|
|
if event.HasField("task"):
|
|
self._hold(task_id, plan, step, event.task)
|
|
for artifact in event.task.artifacts:
|
|
await self._artifact(
|
|
agent, artifact, relayed, files, guide, updater, relay_files
|
|
)
|
|
status = event.task.status
|
|
elif event.HasField("artifact_update"):
|
|
await self._artifact(
|
|
agent,
|
|
event.artifact_update.artifact,
|
|
relayed,
|
|
files,
|
|
guide,
|
|
updater,
|
|
relay_files,
|
|
)
|
|
elif event.HasField("status_update"):
|
|
status = event.status_update.status
|
|
elif event.HasField("message"):
|
|
# A reply with no task: the agent's whole answer.
|
|
heartbeat.cancel()
|
|
self._waiting.pop(task_id, None)
|
|
await updater.complete(
|
|
updater.new_agent_message(list(event.message.parts))
|
|
)
|
|
return
|
|
if status is not None and status.state in _ENDS_A_TURN:
|
|
heartbeat.cancel()
|
|
if status.state == TaskState.TASK_STATE_COMPLETED:
|
|
waiting = self._waiting.get(task_id)
|
|
self._kept(plan, agent, waiting, status, files, guide)
|
|
await self._completed(
|
|
task_id, plan, step, status, files, updater
|
|
)
|
|
else:
|
|
await self._end_turn(task_id, status, updater)
|
|
return
|
|
if status is not None and status.HasField("message"):
|
|
await updater.update_status(
|
|
TaskState.TASK_STATE_WORKING,
|
|
message=updater.new_agent_message(list(status.message.parts)),
|
|
)
|
|
heartbeat.cancel()
|
|
self._waiting.pop(task_id, None)
|
|
await updater.failed(
|
|
updater.new_agent_message(
|
|
[
|
|
new_text_part(
|
|
f"The {agent.work} agent stopped before its task ended."
|
|
)
|
|
]
|
|
)
|
|
)
|
|
except Exception as exc:
|
|
heartbeat.cancel()
|
|
self._clients.forget(agent)
|
|
self._waiting.pop(task_id, None)
|
|
logger.exception("Task %s: relaying to %s failed", task_id, agent.work)
|
|
await updater.failed(
|
|
updater.new_agent_message(
|
|
[
|
|
new_text_part(
|
|
f"Could not relay to the {agent.work} agent at {agent.url}: "
|
|
f"{type(exc).__name__}: {exc}"
|
|
)
|
|
]
|
|
)
|
|
)
|
|
finally:
|
|
heartbeat.cancel()
|
|
|
|
def _kept(
|
|
self,
|
|
plan: Plan,
|
|
agent: Agent,
|
|
waiting: Waiting | None,
|
|
status: TaskStatus,
|
|
files: Mapping[str, str],
|
|
guide: list[str],
|
|
) -> None:
|
|
"""Keep what the agent delivered and note it in the plan's results."""
|
|
execution_id = waiting.task_id if waiting else ""
|
|
ended = _ended(status)
|
|
result: dict[str, Any] = {"execution_id": execution_id, "status": ended}
|
|
record = (
|
|
self._delivery.keep(
|
|
plan.fields,
|
|
work=agent.work,
|
|
execution_id=execution_id or str(uuid.uuid4()),
|
|
status=ended,
|
|
request=plan.text,
|
|
files=files,
|
|
guide=guide[0] if guide else None,
|
|
)
|
|
if self._delivery is not None
|
|
else None
|
|
)
|
|
if record is not None and self._delivery is not None:
|
|
result.update(self._delivery.links(record))
|
|
else:
|
|
result["files"] = {name: None for name in sorted(files)}
|
|
plan.results[agent.work] = result
|
|
|
|
async def _completed(
|
|
self,
|
|
task_id: str,
|
|
plan: Plan,
|
|
step: int,
|
|
status: TaskStatus,
|
|
files: dict[str, str],
|
|
updater: TaskUpdater,
|
|
) -> None:
|
|
"""The agent at `step` completed: start the next with its files, unless it
|
|
ended with warnings, when nothing after it starts; or, after the last,
|
|
finish."""
|
|
agent = plan.steps[step]
|
|
parts = list(status.message.parts) if status.HasField("message") else []
|
|
ended = _ended(status)
|
|
if step + 1 == len(plan.steps):
|
|
await self._finish(task_id, plan, parts, ended, updater)
|
|
return
|
|
if ended != "completed":
|
|
not_started = [later.work for later in plan.steps[step + 1 :]]
|
|
await self._finish(
|
|
task_id,
|
|
plan,
|
|
[
|
|
new_text_part(
|
|
f"The {agent.work} agent ended with warnings, so "
|
|
f"{', '.join(not_started)} was not started. Settle them "
|
|
"and ask again."
|
|
),
|
|
new_data_part(
|
|
{
|
|
"status": "stopped",
|
|
"after": agent.work,
|
|
"not_started": not_started,
|
|
}
|
|
),
|
|
*parts, # what the agent itself said
|
|
],
|
|
"stopped",
|
|
updater,
|
|
not_started,
|
|
)
|
|
return
|
|
record = {
|
|
"status": "completed",
|
|
"customer_case": plan.fields.get("customer_case"),
|
|
"execution_id": plan.results[agent.work]["execution_id"] or None,
|
|
"workflow_request": plan.text,
|
|
"work": agent.work,
|
|
}
|
|
sources = {
|
|
**files,
|
|
RECORD: json.dumps({k: v for k, v in record.items() if v is not None}),
|
|
}
|
|
following = plan.steps[step + 1]
|
|
await updater.update_status(
|
|
TaskState.TASK_STATE_WORKING,
|
|
message=updater.new_agent_message(
|
|
[
|
|
new_text_part(
|
|
f"The {agent.work} agent completed. Its files go to the "
|
|
f"{following.work} agent."
|
|
)
|
|
]
|
|
),
|
|
)
|
|
await self._run(
|
|
task_id, plan, step + 1, self._opening(plan, step + 1, sources), updater
|
|
)
|
|
|
|
async def _finish(
|
|
self,
|
|
task_id: str,
|
|
plan: Plan,
|
|
parts: list[Part],
|
|
ended: str,
|
|
updater: TaskUpdater,
|
|
not_started: list[str] | None = None,
|
|
) -> None:
|
|
"""Complete our task; a caller given links has one document of them."""
|
|
self._waiting.pop(task_id, None)
|
|
if plan.deliver == LINKS:
|
|
document = self._document(plan, ended, not_started)
|
|
await updater.add_artifact(
|
|
[new_text_part(json.dumps(document), media_type="application/json")],
|
|
name=DOCUMENT,
|
|
)
|
|
await updater.complete(updater.new_agent_message(parts))
|
|
|
|
def _document(
|
|
self, plan: Plan, ended: str, not_started: list[str] | None
|
|
) -> dict[str, Any]:
|
|
"""One document of links to what each agent delivered; at its top level,
|
|
the last agent's."""
|
|
last = list(plan.results.values())[-1] if plan.results else {}
|
|
_, case = Delivery.place(plan.fields)
|
|
document: dict[str, Any] = {
|
|
"status": ended,
|
|
"work": plan.work,
|
|
"customer_case": case,
|
|
"execution_id": last.get("execution_id"),
|
|
"record": last.get("record"),
|
|
"guide": last.get("guide"),
|
|
"files": last.get("files", {}),
|
|
"latest": last.get("latest"),
|
|
"case": last.get("case"),
|
|
"results": plan.results,
|
|
}
|
|
if not_started:
|
|
document["not_started"] = not_started
|
|
return document
|
|
|
|
def _hold(self, task_id: str, plan: Plan, step: int, task: Task) -> None:
|
|
if task.id:
|
|
self._waiting[task_id] = Waiting(plan, step, task.id, task.context_id)
|
|
|
|
async def _end_turn(
|
|
self, task_id: str, status: TaskStatus, updater: TaskUpdater
|
|
) -> None:
|
|
parts = list(status.message.parts) if status.HasField("message") else []
|
|
message = updater.new_agent_message(parts) if parts else None
|
|
if status.state not in _PAUSES:
|
|
self._waiting.pop(task_id, None)
|
|
if status.state == TaskState.TASK_STATE_INPUT_REQUIRED:
|
|
await updater.requires_input(message)
|
|
elif status.state == TaskState.TASK_STATE_AUTH_REQUIRED:
|
|
await updater.requires_auth(message)
|
|
elif status.state == TaskState.TASK_STATE_COMPLETED:
|
|
await updater.complete(message)
|
|
elif status.state == TaskState.TASK_STATE_REJECTED:
|
|
await updater.reject(message)
|
|
elif status.state == TaskState.TASK_STATE_CANCELED:
|
|
await updater.cancel(message)
|
|
else:
|
|
await updater.failed(message)
|
|
|
|
@staticmethod
|
|
async def _artifact(
|
|
agent: Agent,
|
|
artifact: Artifact,
|
|
relayed: set[str],
|
|
files: dict[str, str],
|
|
guide: list[str],
|
|
updater: TaskUpdater,
|
|
relay: bool,
|
|
) -> None:
|
|
if artifact.artifact_id in relayed:
|
|
return
|
|
relayed.add(artifact.artifact_id)
|
|
text = "".join(part.text for part in artifact.parts if part.HasField("text"))
|
|
if artifact.name and text:
|
|
files[artifact.name] = text
|
|
# The Execution engine sends its final Deliverable as markdown.
|
|
if not guide and any(
|
|
part.media_type == "text/markdown" for part in artifact.parts
|
|
):
|
|
guide.append(artifact.name)
|
|
if not relay:
|
|
return
|
|
metadata = (
|
|
json_format.MessageToDict(artifact.metadata)
|
|
if artifact.HasField("metadata")
|
|
else {}
|
|
)
|
|
await updater.add_artifact(
|
|
list(artifact.parts),
|
|
artifact_id=artifact.artifact_id or None,
|
|
name=artifact.name or None,
|
|
metadata={**metadata, "work": agent.work},
|
|
)
|
|
|
|
async def _beat(self, agent: Agent, updater: TaskUpdater) -> None:
|
|
while True:
|
|
await asyncio.sleep(self._heartbeat_seconds)
|
|
await updater.update_status(
|
|
TaskState.TASK_STATE_WORKING,
|
|
message=updater.new_agent_message(
|
|
[new_text_part(f"The {agent.work} agent is still working.")]
|
|
),
|
|
)
|
|
|
|
|
|
def _deliver(options: Mapping[str, Any]) -> str:
|
|
"""How the caller is given what agents delivered: as it asked, or links for a
|
|
KAB workflow step, which takes one output per step, and files for anyone else."""
|
|
asked = options.get(DELIVER)
|
|
if asked in (LINKS, FILES):
|
|
return str(asked)
|
|
return LINKS if all(key in options for key in _PLATFORM_STEP) else FILES
|
|
|
|
|
|
def _ended(status: TaskStatus) -> str:
|
|
"""How an agent says its work ended, from its last word: `completed` unless it
|
|
says otherwise, such as `completed_with_warnings`."""
|
|
parts = list(status.message.parts) if status.HasField("message") else []
|
|
said = next(
|
|
(data for data in get_data_parts(parts) if isinstance(data, Mapping)), {}
|
|
)
|
|
return str(said.get("status", "completed"))
|
|
|
|
|
|
def _as_files(sources: Mapping[str, Any]) -> dict[str, str]:
|
|
"""Sources as files by name: what was parsed on the way, as a platform step
|
|
does with an earlier step's output, is sent on as JSON text."""
|
|
return {
|
|
str(name): content if isinstance(content, str) else json.dumps(content)
|
|
for name, content in sources.items()
|
|
}
|
|
|
|
|
|
def _options(context: RequestContext) -> dict[str, Any]:
|
|
"""The request's metadata: the message's, then the request's own."""
|
|
options: dict[str, Any] = {}
|
|
if context.message is not None and context.message.HasField("metadata"):
|
|
options.update(
|
|
_whole_numbers(json_format.MessageToDict(context.message.metadata))
|
|
)
|
|
options.update(context.metadata)
|
|
return options
|
|
|
|
|
|
def _whole_numbers(value: Any) -> Any:
|
|
"""Metadata holds every number as a double; a caller's 9 is sent on as 9."""
|
|
if isinstance(value, float) and value.is_integer():
|
|
return int(value)
|
|
if isinstance(value, Mapping):
|
|
return {key: _whole_numbers(item) for key, item in value.items()}
|
|
if isinstance(value, list):
|
|
return [_whole_numbers(item) for item in value]
|
|
return value
|