Files
orchestration_engine/tests/toy/test_server.py

375 lines
14 KiB
Python

"""An agent's server on the toy Workflow: the card, A2A tasks and the one-shot route."""
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
import pytest
from starlette.testclient import TestClient
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.roles import load_roles
from execution_engine.agent import Assembly
from execution_engine.config import Settings
from execution_engine.main import create_app
from execution_engine.providers import load_provider
from tests.toy.support import make_engine, once_each
from tests.toy.workflow import PLANNING, ROOT, TOY
SETTINGS = Settings( # type: ignore[call-arg]
_env_file=None, CLOUD_PROVIDER="toy", AGENT_NAME="picnic-agent"
)
def client_for(
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
) -> tuple[TestClient, FakeAgentRuntime]:
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, runtime = make_engine(tmp_path, turns, roles=roles, **overrides)
@asynccontextmanager
async def assembly(settings):
yield Assembly(engine, bundle, roles, {"larder": "not reachable"})
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
def send(
client: TestClient,
parts: list[dict],
task: dict | None = None,
metadata=None,
path: str = "/a2a",
):
message = {"messageId": str(uuid.uuid4()), "role": "ROLE_USER", "parts": parts}
if task:
message.update(taskId=task["id"], contextId=task["contextId"])
if metadata:
message["metadata"] = metadata
response = client.post(
path,
json={
"jsonrpc": "2.0",
"id": 1,
"method": "SendMessage",
"params": {"message": message},
},
headers={"A2A-Version": "1.0"},
)
assert response.status_code == 200, response.text
body = response.json()
assert "error" not in body, body
return body["result"]["task"]
def status_parts(task: dict) -> tuple[str, dict | None]:
parts = task["status"]["message"]["parts"]
text = "\n".join(part["text"] for part in parts if "text" in part)
data = next((part["data"] for part in parts if "data" in part), None)
return text, data
def test_the_card_describes_the_workflow_it_serves(tmp_path):
client, _ = client_for(tmp_path, {})
with client:
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
card = client.get(path).json()
skill = card["skills"][0]
assert card["name"] == "picnic-agent"
assert card["description"].startswith("Plans a picnic with you")
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
assert (skill["id"], skill["name"]) == ("picnic", "Picnic planning")
assert "planning: menu, shopping_list" in skill["description"]
assert "packing: plan" in skill["description"]
assert skill["tags"] == ["picnic", "toy"]
assert "architecture" not in str(card).lower()
def test_the_card_declares_the_fields_a_platform_step_sends(tmp_path):
# KAB's workflow builder draws a step's inputs from skills[0].inputSchema.
client, _ = client_for(tmp_path, {})
with client:
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
skill = client.get(path).json()["skills"][0]
schema = skill["inputSchema"]
assert skill["inputArrives"] == "as_fields"
assert set(schema["properties"]) == {
"request",
"customer_case",
"scope",
"pre_approved",
}
assert schema["properties"]["scope"]["uiFieldType"] == "textarea"
assert schema["properties"]["pre_approved"]["type"] == "boolean"
assert schema["required"] == ["customer_case"]
def test_a_request_in_the_metadata_is_the_executions_request(tmp_path):
# A platform step sends every input as metadata and, as the message text,
# all of them again as one JSON document. The request is the field.
client, _ = client_for(tmp_path, once_each())
with client:
task = send(
client,
[{"text": '{"request": "A picnic for nine", "pre_approved": true}'}],
metadata={
"request": "A picnic for nine",
"pre_approved": True,
"customer_case": "park",
},
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
record = (out / "execution.json").read_text()
assert "A picnic for nine" in record
assert '\\"pre_approved\\"' not in record
def test_a_caller_that_posts_to_the_agents_own_address_is_served(tmp_path):
# The KAB platform calls the URL an agent was registered with, not the
# interface its card names, so the same handler answers at the root.
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
*turns["menu-writer"],
]
client, _ = client_for(tmp_path, turns)
with client:
task = send(client, [{"text": "A picnic for six"}], path="/")
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
text, _ = status_parts(task)
assert "Any allergies?" in text
def test_health_names_the_provider_and_what_could_not_be_reached(tmp_path):
client, _ = client_for(tmp_path, {})
with client:
health = client.get("/health").json()
assert health["status"] == "healthy"
assert health["agent"] == "picnic-agent"
assert health["cloud_provider"] == "toy"
assert health["mcp_discovery_gaps"] == {"larder": "not reachable"}
def test_a_question_pauses_the_task_and_a_text_reply_resumes_it(tmp_path):
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
*turns["menu-writer"],
]
client, runtime = client_for(tmp_path, turns)
with client:
task = send(client, [{"text": "A picnic for six"}])
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
text, data = status_parts(task)
assert "Any allergies?" in text
assert data == {
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
}
task = send(client, [{"text": "No nuts"}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
def test_an_approval_counts_only_as_structured_data(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
task = send(client, [{"text": "A picnic"}])
text, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
assert "Please review the planning Phase" in text
assert '"decision": "approve"' in text
task = send(client, [{"text": "approve"}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
task = send(client, [{"data": approval}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == "checking"
def test_a_pre_approved_task_completes_with_its_files_as_artefacts(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
task = send(
client,
[{"text": "A picnic"}],
metadata={"pre_approved": True, "guests": 9, "customer_case": "park"},
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
text, data = status_parts(task)
assert text == "The picnic is planned."
assert data == {"status": "completed", "warnings": []}
names = {artifact["name"] for artifact in task["artifacts"]}
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
# An option the Workflow has is taken from the message's metadata.
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
assert '"guests": 9' in (out / "execution.json").read_text()
def test_an_engine_failure_fails_the_task(tmp_path):
# An Orchestrator on a model, with nothing scripted for it to do.
client, _ = client_for(tmp_path, {}, orchestrator=None)
with client:
task = send(client, [{"text": "A picnic"}])
assert task["status"]["state"] == "TASK_STATE_FAILED"
assert "ScriptExhausted" in status_parts(task)[0]
def test_generate_runs_pre_approved_and_replies_with_the_deliverables(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
response = client.post("/generate", json={"request": "A picnic", "guests": 2})
body = response.json()
assert response.status_code == 200
assert body["status"] == "completed"
assert set(body["deliverables"]) == {"menu", "shopping_list", "check", "plan"}
assert body["warnings"] == []
@pytest.mark.parametrize(
("payload", "error"),
[
({}, "'request' or a 'scope' spec is required"),
({"request": "A picnic", "guests": 0}, "guests must be a whole number"),
(
{"request": "A picnic", "customer_case": "Not A Slug"},
"customer_case must be",
),
],
)
def test_generate_refuses_what_cannot_start(tmp_path, payload, error):
client, runtime = client_for(tmp_path, once_each())
with client:
response = client.post("/generate", json=payload)
assert response.status_code == 400
assert error in response.json()["error"]
assert runtime.runs == []
def test_an_app_needs_the_agents_folder_or_a_way_to_assemble_it():
with pytest.raises(ValueError, match="needs the agent's root"):
create_app(TOY, SETTINGS)
# --- messages that start nothing ---------------------------------------------------------
def client_with_intake(tmp_path, decider, turns=None):
from execution_engine.adk.runtime import RoleSpec
from execution_engine.nodes.intake import DeliveredAnswers, Intake
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, runtime = make_engine(tmp_path, turns or once_each(), roles=roles)
@asynccontextmanager
async def assembly(settings):
yield Assembly(
engine,
bundle,
roles,
{},
intake=Intake(decider, tmp_path),
answers=DeliveredAnswers(
runtime,
RoleSpec("reply-writer", "Write.", budget=3),
tmp_path,
TOY.final,
),
)
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
def test_a_question_about_the_agent_is_answered_in_the_workflows_own_words(tmp_path):
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
decider = FakeDecider(
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.05, 0.9)}
)
client, runtime = client_with_intake(tmp_path, decider)
with client:
task = send(client, [{"text": "What can you do?"}])
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert status_parts(task)[0] == TOY.about
assert runtime.runs == []
def test_a_question_about_delivered_work_is_answered_from_its_final_document(tmp_path):
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
seen: dict = {}
def writer(task):
seen.update(task.context)
return [Call("write_reply", {"text": "Lemonade, because it is summer."})]
decider = FakeDecider(
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.95, 0.9)}
)
turns = {**once_each(), "reply-writer": [writer]}
client, _ = client_with_intake(tmp_path, decider, turns)
with client:
send(
client,
[{"text": "A picnic"}],
metadata={"pre_approved": True, "customer_case": "park"},
)
task = send(
client, [{"text": "Why lemonade?"}], metadata={"customer_case": "park"}
)
assert status_parts(task)[0] == "Lemonade, because it is summer."
# Written from the plan, the Workflow's final Deliverable.
assert seen["delivered"]["final_document"].startswith("# plan")
def test_a_workflow_built_from_the_settings_and_bundle_is_the_one_assembled(
tmp_path, monkeypatch
):
# An agent whose Workflow depends on its settings and provider bundle gives a
# function to build it; the card still describes the Workflow it was given.
import execution_engine.main as server
seen: dict[str, Any] = {}
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, _ = make_engine(tmp_path, {}, roles=roles)
@asynccontextmanager
async def fake_open_assembly(settings, workflow, *, root):
seen.update(workflow=workflow, root=root)
yield Assembly(engine, bundle, roles, {})
def build(settings, bundle):
return TOY
monkeypatch.setattr(server, "open_assembly", fake_open_assembly)
app = server.create_app(TOY, SETTINGS, root=ROOT, build=build)
with TestClient(app) as client:
assert client.get("/health").json()["status"] == "healthy"
assert (
client.get("/.well-known/agent-card.json").json()["name"] == "picnic-agent"
)
assert seen == {"workflow": build, "root": ROOT}