feat: working orhcestration engine
This commit is contained in:
300
tests/orchestration/test_relay.py
Normal file
300
tests/orchestration/test_relay.py
Normal file
@@ -0,0 +1,300 @@
|
||||
"""The Orchestration engine in front of a real agent: the Execution engine on the toy
|
||||
Workflow, scripted. Both run in this process, joined by httpx's ASGI transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call
|
||||
from execution_engine.adk.roles import load_roles
|
||||
from execution_engine.agent import Assembly
|
||||
from execution_engine.config import Settings
|
||||
from execution_engine.main import create_app as create_agent
|
||||
from execution_engine.providers import load_provider
|
||||
from orchestration.agents import Agent
|
||||
from orchestration.config import OrchestrationSettings
|
||||
from orchestration.main import create_app
|
||||
from tests.toy.support import make_engine, once_each
|
||||
from tests.toy.workflow import PLANNING, ROOT, TOY
|
||||
|
||||
PICNIC_URL = "http://picnic"
|
||||
PICNIC = Agent(work="picnic", url=PICNIC_URL, description="Plans picnics.")
|
||||
PARTY = Agent(work="party", url="http://party", description="Plans parties.")
|
||||
SETTINGS = OrchestrationSettings(_env_file=None, HEARTBEAT_SECONDS=3600) # type: ignore[call-arg]
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def picnic_agent(
|
||||
tmp_path: Path, turns: dict[str, list[Any]]
|
||||
) -> AsyncIterator[Any]:
|
||||
settings = Settings( # type: ignore[call-arg]
|
||||
_env_file=None,
|
||||
CLOUD_PROVIDER="toy",
|
||||
AGENT_NAME="picnic-agent",
|
||||
A2A_PUBLIC_URL=PICNIC_URL,
|
||||
)
|
||||
bundle = load_provider("toy", ROOT)
|
||||
roles = load_roles(ROOT / "agents", bundle, TOY)
|
||||
engine, runtime = make_engine(tmp_path, turns, roles=roles)
|
||||
|
||||
@asynccontextmanager
|
||||
async def assembly(given: Settings) -> AsyncIterator[Assembly]:
|
||||
yield Assembly(engine, bundle, roles, {})
|
||||
|
||||
app = create_agent(TOY, settings, assembly)
|
||||
async with app.router.lifespan_context(app):
|
||||
yield app, runtime
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def orchestration(
|
||||
agent_app: Any, agents: list[Agent]
|
||||
) -> AsyncIterator[httpx.AsyncClient]:
|
||||
"""A caller's client to the Orchestration engine, whose agents are served by
|
||||
`agent_app` at the picnic agent's address."""
|
||||
to_agents = httpx.AsyncClient(
|
||||
mounts={PICNIC_URL: httpx.ASGITransport(app=agent_app)}
|
||||
)
|
||||
app = create_app(agents, SETTINGS, httpx_client=to_agents)
|
||||
async with (
|
||||
to_agents,
|
||||
httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
|
||||
) as caller,
|
||||
):
|
||||
yield caller
|
||||
|
||||
|
||||
async def send(
|
||||
caller: httpx.AsyncClient,
|
||||
parts: list[dict],
|
||||
task: dict | None = None,
|
||||
metadata: dict | None = None,
|
||||
path: str = "/a2a",
|
||||
) -> dict:
|
||||
message: dict[str, Any] = {
|
||||
"messageId": str(uuid.uuid4()),
|
||||
"role": "ROLE_USER",
|
||||
"parts": parts,
|
||||
}
|
||||
if task:
|
||||
message.update(taskId=task["id"], contextId=task["contextId"])
|
||||
if metadata:
|
||||
message["metadata"] = metadata
|
||||
response = await caller.post(
|
||||
path,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "SendMessage",
|
||||
"params": {"message": message},
|
||||
},
|
||||
headers={"A2A-Version": "1.0"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert "error" not in body, body
|
||||
return body["result"]["task"]
|
||||
|
||||
|
||||
def status_parts(task: dict) -> tuple[str, dict | None]:
|
||||
parts = task["status"]["message"]["parts"]
|
||||
text = "\n".join(part["text"] for part in parts if "text" in part)
|
||||
data = next((part["data"] for part in parts if "data" in part), None)
|
||||
return text, data
|
||||
|
||||
|
||||
async def test_the_agents_questions_and_approvals_are_relayed(tmp_path):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [
|
||||
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
||||
*turns["menu-writer"],
|
||||
]
|
||||
async with (
|
||||
picnic_agent(tmp_path, turns) as (agent_app, runtime),
|
||||
orchestration(agent_app, [PICNIC, PARTY]) as caller,
|
||||
):
|
||||
task = await send(
|
||||
caller, [{"text": "A picnic for six"}], metadata={"work": "picnic"}
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
text, data = status_parts(task)
|
||||
assert "Any allergies?" in text
|
||||
assert data == {
|
||||
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
|
||||
}
|
||||
|
||||
task = await send(caller, [{"text": "No nuts"}], task)
|
||||
|
||||
_, data = status_parts(task)
|
||||
assert data is not None
|
||||
assert data["prompt"]["approval"]["phase"] == PLANNING
|
||||
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
|
||||
|
||||
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
|
||||
task = await send(caller, [{"data": approval}], task)
|
||||
|
||||
_, data = status_parts(task)
|
||||
assert data is not None
|
||||
assert data["prompt"]["approval"]["phase"] == "checking"
|
||||
|
||||
|
||||
async def test_a_pre_approved_request_completes_with_the_agents_files(tmp_path):
|
||||
async with (
|
||||
picnic_agent(tmp_path, once_each()) as (agent_app, _),
|
||||
orchestration(agent_app, [PICNIC, PARTY]) as caller,
|
||||
):
|
||||
task = await send(
|
||||
caller,
|
||||
[{"text": "A picnic"}],
|
||||
metadata={"work": "picnic", "pre_approved": True, "customer_case": "park"},
|
||||
path="/",
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
text, data = status_parts(task)
|
||||
assert text == "The picnic is planned."
|
||||
assert data == {"status": "completed", "warnings": []}
|
||||
names = {artifact["name"] for artifact in task["artifacts"]}
|
||||
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
|
||||
# The metadata went to the agent as it came, the Customer case with it.
|
||||
assert (tmp_path / "deliverables" / "cases" / "park").is_dir()
|
||||
|
||||
|
||||
async def test_a_request_naming_no_work_starts_nothing_and_says_how(tmp_path):
|
||||
async with (
|
||||
picnic_agent(tmp_path, once_each()) as (agent_app, runtime),
|
||||
orchestration(agent_app, [PICNIC, PARTY]) as caller,
|
||||
):
|
||||
task = await send(caller, [{"text": "Something nice on Saturday"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
text, data = status_parts(task)
|
||||
assert "picnic" in text and "party" in text
|
||||
assert data == {"status": "not_started", "work": ["picnic", "party"]}
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_the_card_offers_the_work_each_agent_does():
|
||||
app = create_app([PICNIC, PARTY], SETTINGS)
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
|
||||
) as caller:
|
||||
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
||||
card = (await caller.get(path)).json()
|
||||
skill = card["skills"][0]
|
||||
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
|
||||
assert skill["inputArrives"] == "as_fields"
|
||||
work = skill["inputSchema"]["properties"]["work"]
|
||||
assert work["enum"] == ["picnic", "party"]
|
||||
assert "Plans picnics." in work["description"]
|
||||
assert (await caller.get("/health")).json()["agents"] == ["picnic", "party"]
|
||||
|
||||
|
||||
class SlowAgent:
|
||||
"""An agent that works for a while, then completes: what the operating model
|
||||
agent does for half an hour without a word."""
|
||||
|
||||
delay = 0.3
|
||||
|
||||
async def for_agent(self, agent: Agent) -> Any:
|
||||
return self
|
||||
|
||||
def forget(self, agent: Agent) -> None:
|
||||
pass
|
||||
|
||||
async def send_message(self, request: Any) -> AsyncIterator[Any]:
|
||||
from a2a.helpers.proto_helpers import new_text_part
|
||||
from a2a.types import Message, Role, StreamResponse, Task, TaskState, TaskStatus
|
||||
|
||||
await asyncio.sleep(self.delay)
|
||||
done = Message(
|
||||
message_id="done", role=Role.ROLE_AGENT, parts=[new_text_part("Done.")]
|
||||
)
|
||||
yield StreamResponse(
|
||||
task=Task(
|
||||
id="theirs",
|
||||
context_id="ctx",
|
||||
status=TaskStatus(state=TaskState.TASK_STATE_COMPLETED, message=done),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def test_a_silent_agent_is_reported_as_still_working():
|
||||
settings = OrchestrationSettings(_env_file=None, HEARTBEAT_SECONDS=0.05) # type: ignore[call-arg]
|
||||
app = create_app([PICNIC], settings, clients=SlowAgent()) # type: ignore[arg-type]
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
|
||||
) as caller:
|
||||
response = await caller.post(
|
||||
"/a2a",
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "SendStreamingMessage",
|
||||
"params": {
|
||||
"message": {
|
||||
"messageId": "m1",
|
||||
"role": "ROLE_USER",
|
||||
"parts": [{"text": "A picnic"}],
|
||||
}
|
||||
},
|
||||
},
|
||||
headers={"A2A-Version": "1.0"},
|
||||
)
|
||||
|
||||
events = [
|
||||
json.loads(line.removeprefix("data:"))
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data:")
|
||||
]
|
||||
states = [
|
||||
event["result"]["statusUpdate"]["status"]
|
||||
for event in events
|
||||
if "statusUpdate" in event.get("result", {})
|
||||
]
|
||||
beats = [s for s in states if "still working" in str(s.get("message", ""))]
|
||||
assert len(beats) >= 2
|
||||
assert states[-1]["state"] == "TASK_STATE_COMPLETED"
|
||||
|
||||
|
||||
class RecordingAgent(SlowAgent):
|
||||
"""An agent that records the message it was sent and completes at once."""
|
||||
|
||||
delay = 0.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.received: list[Any] = []
|
||||
|
||||
async def send_message(self, request: Any) -> AsyncIterator[Any]:
|
||||
self.received.append(request.message)
|
||||
async for event in super().send_message(request):
|
||||
yield event
|
||||
|
||||
|
||||
async def test_the_request_field_is_the_text_the_agent_is_sent():
|
||||
# A KAB step sends every field as metadata and, as the message text, all of
|
||||
# them again as one JSON document. The agent is sent the request as its text,
|
||||
# whichever agent it is; the fields stay in the metadata.
|
||||
agent = RecordingAgent()
|
||||
app = create_app([PICNIC], SETTINGS, clients=agent) # type: ignore[arg-type]
|
||||
fields = {"work": "picnic", "request": "A picnic for nine", "pre_approved": True}
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://orchestration"
|
||||
) as caller:
|
||||
await send(caller, [{"text": json.dumps(fields)}], metadata=fields)
|
||||
|
||||
(message,) = agent.received
|
||||
assert [part.text for part in message.parts] == ["A picnic for nine"]
|
||||
assert message.metadata["request"] == "A picnic for nine"
|
||||
assert message.metadata["pre_approved"] is True
|
||||
assert "work" not in message.metadata
|
||||
Reference in New Issue
Block a user