Files
orchestration_engine/tests/orchestration/test_relay.py

301 lines
11 KiB
Python

"""The Orchestration engine in front of a real agent: the Execution engine on the toy
Workflow, scripted. Both run in this process, joined by httpx's ASGI transport."""
from __future__ import annotations
import asyncio
import json
import uuid
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
import httpx
from execution_engine.adk.fake_runtime import Call
from execution_engine.adk.roles import load_roles
from execution_engine.agent import Assembly
from execution_engine.config import Settings
from execution_engine.main import create_app as create_agent
from execution_engine.providers import load_provider
from orchestration.agents import Agent
from orchestration.config import OrchestrationSettings
from orchestration.main import create_app
from tests.toy.support import make_engine, once_each
from tests.toy.workflow import PLANNING, ROOT, TOY
PICNIC_URL = "http://picnic"
PICNIC = Agent(work="picnic", url=PICNIC_URL, description="Plans picnics.")
PARTY = Agent(work="party", url="http://party", description="Plans parties.")
SETTINGS = OrchestrationSettings(_env_file=None, HEARTBEAT_SECONDS=3600) # type: ignore[call-arg]
@asynccontextmanager
async def picnic_agent(
tmp_path: Path, turns: dict[str, list[Any]]
) -> AsyncIterator[Any]:
settings = Settings( # type: ignore[call-arg]
_env_file=None,
CLOUD_PROVIDER="toy",
AGENT_NAME="picnic-agent",
A2A_PUBLIC_URL=PICNIC_URL,
)
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, runtime = make_engine(tmp_path, turns, roles=roles)
@asynccontextmanager
async def assembly(given: Settings) -> AsyncIterator[Assembly]:
yield Assembly(engine, bundle, roles, {})
app = create_agent(TOY, settings, assembly)
async with app.router.lifespan_context(app):
yield app, runtime
@asynccontextmanager
async def orchestration(
agent_app: Any, agents: list[Agent]
) -> AsyncIterator[httpx.AsyncClient]:
"""A caller's client to the Orchestration engine, whose agents are served by
`agent_app` at the picnic agent's address."""
to_agents = httpx.AsyncClient(
mounts={PICNIC_URL: httpx.ASGITransport(app=agent_app)}
)
app = create_app(agents, SETTINGS, httpx_client=to_agents)
async with (
to_agents,
httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
) as caller,
):
yield caller
async def send(
caller: httpx.AsyncClient,
parts: list[dict],
task: dict | None = None,
metadata: dict | None = None,
path: str = "/a2a",
) -> dict:
message: dict[str, Any] = {
"messageId": str(uuid.uuid4()),
"role": "ROLE_USER",
"parts": parts,
}
if task:
message.update(taskId=task["id"], contextId=task["contextId"])
if metadata:
message["metadata"] = metadata
response = await caller.post(
path,
json={
"jsonrpc": "2.0",
"id": 1,
"method": "SendMessage",
"params": {"message": message},
},
headers={"A2A-Version": "1.0"},
)
assert response.status_code == 200, response.text
body = response.json()
assert "error" not in body, body
return body["result"]["task"]
def status_parts(task: dict) -> tuple[str, dict | None]:
parts = task["status"]["message"]["parts"]
text = "\n".join(part["text"] for part in parts if "text" in part)
data = next((part["data"] for part in parts if "data" in part), None)
return text, data
async def test_the_agents_questions_and_approvals_are_relayed(tmp_path):
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
*turns["menu-writer"],
]
async with (
picnic_agent(tmp_path, turns) as (agent_app, runtime),
orchestration(agent_app, [PICNIC, PARTY]) as caller,
):
task = await send(
caller, [{"text": "A picnic for six"}], metadata={"work": "picnic"}
)
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
text, data = status_parts(task)
assert "Any allergies?" in text
assert data == {
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
}
task = await send(caller, [{"text": "No nuts"}], task)
_, data = status_parts(task)
assert data is not None
assert data["prompt"]["approval"]["phase"] == PLANNING
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
task = await send(caller, [{"data": approval}], task)
_, data = status_parts(task)
assert data is not None
assert data["prompt"]["approval"]["phase"] == "checking"
async def test_a_pre_approved_request_completes_with_the_agents_files(tmp_path):
async with (
picnic_agent(tmp_path, once_each()) as (agent_app, _),
orchestration(agent_app, [PICNIC, PARTY]) as caller,
):
task = await send(
caller,
[{"text": "A picnic"}],
metadata={"work": "picnic", "pre_approved": True, "customer_case": "park"},
path="/",
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
text, data = status_parts(task)
assert text == "The picnic is planned."
assert data == {"status": "completed", "warnings": []}
names = {artifact["name"] for artifact in task["artifacts"]}
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
# The metadata went to the agent as it came, the Customer case with it.
assert (tmp_path / "deliverables" / "cases" / "park").is_dir()
async def test_a_request_naming_no_work_starts_nothing_and_says_how(tmp_path):
async with (
picnic_agent(tmp_path, once_each()) as (agent_app, runtime),
orchestration(agent_app, [PICNIC, PARTY]) as caller,
):
task = await send(caller, [{"text": "Something nice on Saturday"}])
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
text, data = status_parts(task)
assert "picnic" in text and "party" in text
assert data == {"status": "not_started", "work": ["picnic", "party"]}
assert runtime.runs == []
async def test_the_card_offers_the_work_each_agent_does():
app = create_app([PICNIC, PARTY], SETTINGS)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
) as caller:
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
card = (await caller.get(path)).json()
skill = card["skills"][0]
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
assert skill["inputArrives"] == "as_fields"
work = skill["inputSchema"]["properties"]["work"]
assert work["enum"] == ["picnic", "party"]
assert "Plans picnics." in work["description"]
assert (await caller.get("/health")).json()["agents"] == ["picnic", "party"]
class SlowAgent:
"""An agent that works for a while, then completes: what the operating model
agent does for half an hour without a word."""
delay = 0.3
async def for_agent(self, agent: Agent) -> Any:
return self
def forget(self, agent: Agent) -> None:
pass
async def send_message(self, request: Any) -> AsyncIterator[Any]:
from a2a.helpers.proto_helpers import new_text_part
from a2a.types import Message, Role, StreamResponse, Task, TaskState, TaskStatus
await asyncio.sleep(self.delay)
done = Message(
message_id="done", role=Role.ROLE_AGENT, parts=[new_text_part("Done.")]
)
yield StreamResponse(
task=Task(
id="theirs",
context_id="ctx",
status=TaskStatus(state=TaskState.TASK_STATE_COMPLETED, message=done),
)
)
async def test_a_silent_agent_is_reported_as_still_working():
settings = OrchestrationSettings(_env_file=None, HEARTBEAT_SECONDS=0.05) # type: ignore[call-arg]
app = create_app([PICNIC], settings, clients=SlowAgent()) # type: ignore[arg-type]
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
) as caller:
response = await caller.post(
"/a2a",
json={
"jsonrpc": "2.0",
"id": 1,
"method": "SendStreamingMessage",
"params": {
"message": {
"messageId": "m1",
"role": "ROLE_USER",
"parts": [{"text": "A picnic"}],
}
},
},
headers={"A2A-Version": "1.0"},
)
events = [
json.loads(line.removeprefix("data:"))
for line in response.text.splitlines()
if line.startswith("data:")
]
states = [
event["result"]["statusUpdate"]["status"]
for event in events
if "statusUpdate" in event.get("result", {})
]
beats = [s for s in states if "still working" in str(s.get("message", ""))]
assert len(beats) >= 2
assert states[-1]["state"] == "TASK_STATE_COMPLETED"
class RecordingAgent(SlowAgent):
"""An agent that records the message it was sent and completes at once."""
delay = 0.0
def __init__(self) -> None:
self.received: list[Any] = []
async def send_message(self, request: Any) -> AsyncIterator[Any]:
self.received.append(request.message)
async for event in super().send_message(request):
yield event
async def test_the_request_field_is_the_text_the_agent_is_sent():
# A KAB step sends every field as metadata and, as the message text, all of
# them again as one JSON document. The agent is sent the request as its text,
# whichever agent it is; the fields stay in the metadata.
agent = RecordingAgent()
app = create_app([PICNIC], SETTINGS, clients=agent) # type: ignore[arg-type]
fields = {"work": "picnic", "request": "A picnic for nine", "pre_approved": True}
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
) as caller:
await send(caller, [{"text": json.dumps(fields)}], metadata=fields)
(message,) = agent.received
assert [part.text for part in message.parts] == ["A picnic for nine"]
assert message.metadata["request"] == "A picnic for nine"
assert message.metadata["pre_approved"] is True
assert "work" not in message.metadata