375 lines
14 KiB
Python
375 lines
14 KiB
Python
"""An agent's server on the toy Workflow: the card, A2A tasks and the one-shot route."""
|
|
|
|
import uuid
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from starlette.testclient import TestClient
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.roles import load_roles
|
|
from execution_engine.agent import Assembly
|
|
from execution_engine.config import Settings
|
|
from execution_engine.main import create_app
|
|
from execution_engine.providers import load_provider
|
|
from tests.toy.support import make_engine, once_each
|
|
from tests.toy.workflow import PLANNING, ROOT, TOY
|
|
|
|
SETTINGS = Settings( # type: ignore[call-arg]
|
|
_env_file=None, CLOUD_PROVIDER="toy", AGENT_NAME="picnic-agent"
|
|
)
|
|
|
|
|
|
def client_for(
|
|
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
|
|
) -> tuple[TestClient, FakeAgentRuntime]:
|
|
bundle = load_provider("toy", ROOT)
|
|
roles = load_roles(ROOT / "agents", bundle, TOY)
|
|
engine, runtime = make_engine(tmp_path, turns, roles=roles, **overrides)
|
|
|
|
@asynccontextmanager
|
|
async def assembly(settings):
|
|
yield Assembly(engine, bundle, roles, {"larder": "not reachable"})
|
|
|
|
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
|
|
|
|
|
|
def send(
|
|
client: TestClient,
|
|
parts: list[dict],
|
|
task: dict | None = None,
|
|
metadata=None,
|
|
path: str = "/a2a",
|
|
):
|
|
message = {"messageId": str(uuid.uuid4()), "role": "ROLE_USER", "parts": parts}
|
|
if task:
|
|
message.update(taskId=task["id"], contextId=task["contextId"])
|
|
if metadata:
|
|
message["metadata"] = metadata
|
|
response = client.post(
|
|
path,
|
|
json={
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "SendMessage",
|
|
"params": {"message": message},
|
|
},
|
|
headers={"A2A-Version": "1.0"},
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
body = response.json()
|
|
assert "error" not in body, body
|
|
return body["result"]["task"]
|
|
|
|
|
|
def status_parts(task: dict) -> tuple[str, dict | None]:
|
|
parts = task["status"]["message"]["parts"]
|
|
text = "\n".join(part["text"] for part in parts if "text" in part)
|
|
data = next((part["data"] for part in parts if "data" in part), None)
|
|
return text, data
|
|
|
|
|
|
def test_the_card_describes_the_workflow_it_serves(tmp_path):
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
|
card = client.get(path).json()
|
|
skill = card["skills"][0]
|
|
assert card["name"] == "picnic-agent"
|
|
assert card["description"].startswith("Plans a picnic with you")
|
|
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
|
|
assert (skill["id"], skill["name"]) == ("picnic", "Picnic planning")
|
|
assert "planning: menu, shopping_list" in skill["description"]
|
|
assert "packing: plan" in skill["description"]
|
|
assert skill["tags"] == ["picnic", "toy"]
|
|
assert "architecture" not in str(card).lower()
|
|
|
|
|
|
def test_the_card_declares_the_fields_a_platform_step_sends(tmp_path):
|
|
# KAB's workflow builder draws a step's inputs from skills[0].inputSchema.
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
|
skill = client.get(path).json()["skills"][0]
|
|
schema = skill["inputSchema"]
|
|
assert skill["inputArrives"] == "as_fields"
|
|
assert set(schema["properties"]) == {
|
|
"request",
|
|
"customer_case",
|
|
"scope",
|
|
"pre_approved",
|
|
}
|
|
assert schema["properties"]["scope"]["uiFieldType"] == "textarea"
|
|
assert schema["properties"]["pre_approved"]["type"] == "boolean"
|
|
assert schema["required"] == ["customer_case"]
|
|
|
|
|
|
def test_a_request_in_the_metadata_is_the_executions_request(tmp_path):
|
|
# A platform step sends every input as metadata and, as the message text,
|
|
# all of them again as one JSON document. The request is the field.
|
|
client, _ = client_for(tmp_path, once_each())
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": '{"request": "A picnic for nine", "pre_approved": true}'}],
|
|
metadata={
|
|
"request": "A picnic for nine",
|
|
"pre_approved": True,
|
|
"customer_case": "park",
|
|
},
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
|
|
record = (out / "execution.json").read_text()
|
|
assert "A picnic for nine" in record
|
|
assert '\\"pre_approved\\"' not in record
|
|
|
|
|
|
def test_a_caller_that_posts_to_the_agents_own_address_is_served(tmp_path):
|
|
# The KAB platform calls the URL an agent was registered with, not the
|
|
# interface its card names, so the same handler answers at the root.
|
|
turns = once_each()
|
|
turns["menu-writer"] = [
|
|
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
|
*turns["menu-writer"],
|
|
]
|
|
client, _ = client_for(tmp_path, turns)
|
|
with client:
|
|
task = send(client, [{"text": "A picnic for six"}], path="/")
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
text, _ = status_parts(task)
|
|
assert "Any allergies?" in text
|
|
|
|
|
|
def test_health_names_the_provider_and_what_could_not_be_reached(tmp_path):
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
health = client.get("/health").json()
|
|
|
|
assert health["status"] == "healthy"
|
|
assert health["agent"] == "picnic-agent"
|
|
assert health["cloud_provider"] == "toy"
|
|
assert health["mcp_discovery_gaps"] == {"larder": "not reachable"}
|
|
|
|
|
|
def test_a_question_pauses_the_task_and_a_text_reply_resumes_it(tmp_path):
|
|
turns = once_each()
|
|
turns["menu-writer"] = [
|
|
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
|
*turns["menu-writer"],
|
|
]
|
|
client, runtime = client_for(tmp_path, turns)
|
|
with client:
|
|
task = send(client, [{"text": "A picnic for six"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
text, data = status_parts(task)
|
|
assert "Any allergies?" in text
|
|
assert data == {
|
|
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
|
|
}
|
|
|
|
task = send(client, [{"text": "No nuts"}], task)
|
|
|
|
_, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == PLANNING
|
|
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
|
|
|
|
|
|
def test_an_approval_counts_only_as_structured_data(tmp_path):
|
|
client, _ = client_for(tmp_path, once_each())
|
|
with client:
|
|
task = send(client, [{"text": "A picnic"}])
|
|
text, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == PLANNING
|
|
assert "Please review the planning Phase" in text
|
|
assert '"decision": "approve"' in text
|
|
|
|
task = send(client, [{"text": "approve"}], task)
|
|
_, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == PLANNING
|
|
|
|
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
|
|
task = send(client, [{"data": approval}], task)
|
|
_, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == "checking"
|
|
|
|
|
|
def test_a_pre_approved_task_completes_with_its_files_as_artefacts(tmp_path):
|
|
client, _ = client_for(tmp_path, once_each())
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": "A picnic"}],
|
|
metadata={"pre_approved": True, "guests": 9, "customer_case": "park"},
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
text, data = status_parts(task)
|
|
assert text == "The picnic is planned."
|
|
assert data == {"status": "completed", "warnings": []}
|
|
names = {artifact["name"] for artifact in task["artifacts"]}
|
|
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
|
|
# An option the Workflow has is taken from the message's metadata.
|
|
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
|
|
assert '"guests": 9' in (out / "execution.json").read_text()
|
|
|
|
|
|
def test_an_engine_failure_fails_the_task(tmp_path):
|
|
# An Orchestrator on a model, with nothing scripted for it to do.
|
|
client, _ = client_for(tmp_path, {}, orchestrator=None)
|
|
with client:
|
|
task = send(client, [{"text": "A picnic"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_FAILED"
|
|
assert "ScriptExhausted" in status_parts(task)[0]
|
|
|
|
|
|
def test_generate_runs_pre_approved_and_replies_with_the_deliverables(tmp_path):
|
|
client, _ = client_for(tmp_path, once_each())
|
|
with client:
|
|
response = client.post("/generate", json={"request": "A picnic", "guests": 2})
|
|
|
|
body = response.json()
|
|
assert response.status_code == 200
|
|
assert body["status"] == "completed"
|
|
assert set(body["deliverables"]) == {"menu", "shopping_list", "check", "plan"}
|
|
assert body["warnings"] == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("payload", "error"),
|
|
[
|
|
({}, "'request' or a 'scope' spec is required"),
|
|
({"request": "A picnic", "guests": 0}, "guests must be a whole number"),
|
|
(
|
|
{"request": "A picnic", "customer_case": "Not A Slug"},
|
|
"customer_case must be",
|
|
),
|
|
],
|
|
)
|
|
def test_generate_refuses_what_cannot_start(tmp_path, payload, error):
|
|
client, runtime = client_for(tmp_path, once_each())
|
|
with client:
|
|
response = client.post("/generate", json=payload)
|
|
|
|
assert response.status_code == 400
|
|
assert error in response.json()["error"]
|
|
assert runtime.runs == []
|
|
|
|
|
|
def test_an_app_needs_the_agents_folder_or_a_way_to_assemble_it():
|
|
with pytest.raises(ValueError, match="needs the agent's root"):
|
|
create_app(TOY, SETTINGS)
|
|
|
|
|
|
# --- messages that start nothing ---------------------------------------------------------
|
|
|
|
|
|
def client_with_intake(tmp_path, decider, turns=None):
|
|
from execution_engine.adk.runtime import RoleSpec
|
|
from execution_engine.nodes.intake import DeliveredAnswers, Intake
|
|
|
|
bundle = load_provider("toy", ROOT)
|
|
roles = load_roles(ROOT / "agents", bundle, TOY)
|
|
engine, runtime = make_engine(tmp_path, turns or once_each(), roles=roles)
|
|
|
|
@asynccontextmanager
|
|
async def assembly(settings):
|
|
yield Assembly(
|
|
engine,
|
|
bundle,
|
|
roles,
|
|
{},
|
|
intake=Intake(decider, tmp_path),
|
|
answers=DeliveredAnswers(
|
|
runtime,
|
|
RoleSpec("reply-writer", "Write.", budget=3),
|
|
tmp_path,
|
|
TOY.final,
|
|
),
|
|
)
|
|
|
|
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
|
|
|
|
|
|
def test_a_question_about_the_agent_is_answered_in_the_workflows_own_words(tmp_path):
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer
|
|
|
|
decider = FakeDecider(
|
|
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.05, 0.9)}
|
|
)
|
|
client, runtime = client_with_intake(tmp_path, decider)
|
|
with client:
|
|
task = send(client, [{"text": "What can you do?"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
assert status_parts(task)[0] == TOY.about
|
|
assert runtime.runs == []
|
|
|
|
|
|
def test_a_question_about_delivered_work_is_answered_from_its_final_document(tmp_path):
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer
|
|
|
|
seen: dict = {}
|
|
|
|
def writer(task):
|
|
seen.update(task.context)
|
|
return [Call("write_reply", {"text": "Lemonade, because it is summer."})]
|
|
|
|
decider = FakeDecider(
|
|
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.95, 0.9)}
|
|
)
|
|
turns = {**once_each(), "reply-writer": [writer]}
|
|
client, _ = client_with_intake(tmp_path, decider, turns)
|
|
with client:
|
|
send(
|
|
client,
|
|
[{"text": "A picnic"}],
|
|
metadata={"pre_approved": True, "customer_case": "park"},
|
|
)
|
|
task = send(
|
|
client, [{"text": "Why lemonade?"}], metadata={"customer_case": "park"}
|
|
)
|
|
|
|
assert status_parts(task)[0] == "Lemonade, because it is summer."
|
|
# Written from the plan, the Workflow's final Deliverable.
|
|
assert seen["delivered"]["final_document"].startswith("# plan")
|
|
|
|
|
|
def test_a_workflow_built_from_the_settings_and_bundle_is_the_one_assembled(
|
|
tmp_path, monkeypatch
|
|
):
|
|
# An agent whose Workflow depends on its settings and provider bundle gives a
|
|
# function to build it; the card still describes the Workflow it was given.
|
|
import execution_engine.main as server
|
|
|
|
seen: dict[str, Any] = {}
|
|
bundle = load_provider("toy", ROOT)
|
|
roles = load_roles(ROOT / "agents", bundle, TOY)
|
|
engine, _ = make_engine(tmp_path, {}, roles=roles)
|
|
|
|
@asynccontextmanager
|
|
async def fake_open_assembly(settings, workflow, *, root):
|
|
seen.update(workflow=workflow, root=root)
|
|
yield Assembly(engine, bundle, roles, {})
|
|
|
|
def build(settings, bundle):
|
|
return TOY
|
|
|
|
monkeypatch.setattr(server, "open_assembly", fake_open_assembly)
|
|
app = server.create_app(TOY, SETTINGS, root=ROOT, build=build)
|
|
with TestClient(app) as client:
|
|
assert client.get("/health").json()["status"] == "healthy"
|
|
assert (
|
|
client.get("/.well-known/agent-card.json").json()["name"] == "picnic-agent"
|
|
)
|
|
|
|
assert seen == {"workflow": build, "root": ROOT}
|