Some checks failed
validation / verify (push) Failing after 10s
Every Execution now begins by agreeing the scope, before requirements. - A new Phase and Deliverable, produced by a scope analyst with two tools: scope_read shows what is settled and what is not; scope_set records fields with where each value came from (the request, the customer's answer, or an assumption with its reason). Code checks each value against the catalogue. - With someone present the analyst asks for what is missing; with nobody present it assumes it, visibly. With no spec it drafts one from the request. A value the customer stated in the file is changed only by their own answer. - A submission cannot leave an essential field unknown. Code writes the completed spec, the data every later Specialist reads, and the information register, from what was recorded. - The spec's status follows the Execution: under review, approved, then locked as a Baseline. A Change request starts from the delivered scope. - The requirements analyst derives from the agreed scope and no longer asks for what it settles. About 80 tests assumed requirements came first; tests/scoped.py runs scoping ahead of a test's own script, and tests about order and counts are updated. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
524 lines
18 KiB
Python
524 lines
18 KiB
Python
"""The A2A server: tasks map onto Executions, driven over JSON-RPC with a scripted runtime."""
|
|
|
|
import uuid
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
from starlette.testclient import TestClient
|
|
|
|
from app.adk.fake_runtime import (
|
|
Call,
|
|
FakeAgentRuntime,
|
|
FakeDecider,
|
|
sample_submission,
|
|
)
|
|
from app.adk.roles import load_roles
|
|
from app.adk.runtime import Answer
|
|
from app.agent import Assembly, ExecutionEngine
|
|
from app.config import get_settings
|
|
from app.main import create_app
|
|
from app.nodes.intake import Intake
|
|
from app.providers import load_provider
|
|
from app.recorder import InMemoryRecorder
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
|
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
|
|
|
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
|
K = DeliverableKind
|
|
LEGACY_GENERATE_KEYS = {
|
|
"execution_id",
|
|
"session_id",
|
|
"status",
|
|
"validation_passed",
|
|
"total_loop_iterations",
|
|
"current_phase",
|
|
"artifacts",
|
|
"active_skills",
|
|
}
|
|
|
|
|
|
def submission(kind: DeliverableKind) -> list[Call]:
|
|
return [sample_submission(kind)]
|
|
|
|
|
|
def complete_run_turns() -> dict:
|
|
def delegate(*kinds):
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
def approve(phase):
|
|
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
|
|
|
return {
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(PhaseId.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
[
|
|
approve(PhaseId.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[
|
|
approve(PhaseId.SOLUTION_IMPLEMENTATION),
|
|
delegate(K.VALIDATION_RESULTS),
|
|
],
|
|
[approve(PhaseId.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
|
[approve(PhaseId.PACKAGING), Call("finish")],
|
|
],
|
|
**{
|
|
spec.specialist: [submission(kind)]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
},
|
|
}
|
|
|
|
|
|
def client_for(
|
|
tmp_path: Path, turns: dict, decider=None, answers=None
|
|
) -> tuple[TestClient, FakeAgentRuntime]:
|
|
bundle = load_provider("gcp")
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
runtime = FakeAgentRuntime(turns)
|
|
|
|
@asynccontextmanager
|
|
async def fake_assembly(settings):
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
)
|
|
yield Assembly(
|
|
engine,
|
|
bundle,
|
|
roles,
|
|
{"compute": "API not enabled"},
|
|
intake=Intake(decider, tmp_path) if decider else None,
|
|
answers=answers,
|
|
)
|
|
|
|
return TestClient(create_app(get_settings(), fake_assembly)), runtime
|
|
|
|
|
|
def send(
|
|
client: TestClient, parts: list[dict], task: dict | None = None, metadata=None
|
|
):
|
|
message = {"messageId": str(uuid.uuid4()), "role": "ROLE_USER", "parts": parts}
|
|
if task:
|
|
message.update(taskId=task["id"], contextId=task["contextId"])
|
|
if metadata:
|
|
message["metadata"] = metadata
|
|
response = client.post(
|
|
"/a2a",
|
|
json={
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "SendMessage",
|
|
"params": {"message": message},
|
|
},
|
|
headers={"A2A-Version": "1.0"},
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
body = response.json()
|
|
assert "error" not in body, body
|
|
return body["result"]["task"]
|
|
|
|
|
|
def status_parts(task: dict) -> tuple[str, dict | None]:
|
|
parts = task["status"]["message"]["parts"]
|
|
text = "\n".join(part["text"] for part in parts if "text" in part)
|
|
data = next((part["data"] for part in parts if "data" in part), None)
|
|
return text, data
|
|
|
|
|
|
def test_agent_card_is_served_at_current_and_legacy_paths(tmp_path):
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
|
card = client.get(path).json()
|
|
assert card["name"] == get_settings().AGENT_NAME
|
|
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
|
|
assert "solution_architecture" in card["skills"][0]["description"]
|
|
|
|
|
|
def test_question_pauses_the_task_and_a_text_reply_resumes_it(tmp_path):
|
|
turns = {
|
|
"orchestrator": [
|
|
[Call("ask_user", {"questions": ["What is the RTO?"]})],
|
|
lambda task: [
|
|
Call(
|
|
"ask_user",
|
|
{
|
|
"questions": [
|
|
f"Heard: {task.context['conversation'][-1]['text']}"
|
|
]
|
|
},
|
|
)
|
|
],
|
|
]
|
|
}
|
|
client, _ = client_for(tmp_path, turns)
|
|
with client:
|
|
task = send(client, [{"text": "Design regional ingestion"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
text, data = status_parts(task)
|
|
assert "What is the RTO?" in text
|
|
assert data == {
|
|
"prompt": {
|
|
"questions": [{"from": "orchestrator", "question": "What is the RTO?"}]
|
|
}
|
|
}
|
|
|
|
task = send(client, [{"text": "One hour"}], task)
|
|
|
|
text, _ = status_parts(task)
|
|
assert "Heard: One hour" in text
|
|
|
|
|
|
def test_approval_counts_only_as_structured_data(tmp_path):
|
|
turns = complete_run_turns()
|
|
turns["orchestrator"][2] = [
|
|
Call("request_approval", {"phase": "requirements_discovery", "summary": "done"})
|
|
]
|
|
turns["orchestrator"][3:] = [
|
|
[
|
|
Call(
|
|
"request_approval",
|
|
{"phase": "requirements_discovery", "summary": "again"},
|
|
)
|
|
],
|
|
[Call("ask_user", {"questions": ["Approved; what next?"]})],
|
|
]
|
|
client, _ = client_for(tmp_path, turns)
|
|
with client:
|
|
task = send(client, [{"text": "Design ingestion"}])
|
|
text, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == "requirements_discovery"
|
|
assert '"decision": "approve"' in text
|
|
|
|
task = send(client, [{"text": "approve"}], task)
|
|
_, data = status_parts(task)
|
|
assert data["prompt"]["approval"]["phase"] == "requirements_discovery"
|
|
|
|
task = send(
|
|
client,
|
|
[
|
|
{
|
|
"data": {
|
|
"approval": {
|
|
"phase": "requirements_discovery",
|
|
"decision": "approve",
|
|
}
|
|
}
|
|
}
|
|
],
|
|
task,
|
|
)
|
|
text, _ = status_parts(task)
|
|
assert "Approved; what next?" in text
|
|
|
|
|
|
def test_pre_approved_task_completes_with_guide_and_code_artifacts(tmp_path):
|
|
client, _ = client_for(tmp_path, complete_run_turns())
|
|
with client:
|
|
task = send(
|
|
client, [{"text": "Design ingestion"}], metadata={"pre_approved": True}
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
names = {artifact["name"] for artifact in task["artifacts"]}
|
|
assert {
|
|
"solution-architecture-guide.md",
|
|
"terraform/main.tf",
|
|
"architecture.mmd",
|
|
} <= names
|
|
|
|
|
|
def test_engine_failure_fails_the_task(tmp_path):
|
|
client, _ = client_for(tmp_path, {"orchestrator": []})
|
|
with client:
|
|
task = send(client, [{"text": "Design ingestion"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_FAILED"
|
|
assert "ScriptExhausted" in status_parts(task)[0]
|
|
|
|
|
|
def test_v0_3_clients_are_accepted(tmp_path):
|
|
client, _ = client_for(
|
|
tmp_path, {"orchestrator": [[Call("ask_user", {"questions": ["Region?"]})]]}
|
|
)
|
|
with client:
|
|
response = client.post(
|
|
"/a2a",
|
|
json={
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "message/send",
|
|
"params": {
|
|
"message": {
|
|
"messageId": "m1",
|
|
"role": "user",
|
|
"kind": "message",
|
|
"parts": [{"kind": "text", "text": "Design ingestion"}],
|
|
}
|
|
},
|
|
},
|
|
)
|
|
|
|
result = response.json()["result"]
|
|
assert result["status"]["state"] == "input-required"
|
|
|
|
|
|
def test_generate_runs_pre_approved_and_keeps_the_legacy_response_keys(tmp_path):
|
|
client, _ = client_for(tmp_path, complete_run_turns())
|
|
with client:
|
|
response = client.post("/generate", json={"request": "Design ingestion"})
|
|
|
|
body = response.json()
|
|
assert response.status_code == 200, body
|
|
assert LEGACY_GENERATE_KEYS <= set(body)
|
|
assert body["status"] == "success"
|
|
assert body["validation_passed"] is True
|
|
assert body["artifacts"]["terraform_code"] == "# terraform/main.tf\n"
|
|
assert body["artifacts"]["source_discovery_doc"] == "Details."
|
|
assert "google-cloud-solution-architecture" in body["active_skills"]
|
|
assert (
|
|
tmp_path
|
|
/ "deliverables"
|
|
/ "cases"
|
|
/ "unassigned"
|
|
/ "executions"
|
|
/ body["execution_id"]
|
|
).is_dir()
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{}, {"request": " "}])
|
|
def test_generate_requires_a_request(tmp_path, payload):
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
response = client.post("/generate", json=payload)
|
|
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_health_reports_mcp_discovery_gaps(tmp_path):
|
|
client, _ = client_for(tmp_path, {})
|
|
with client:
|
|
body = client.get("/health").json()
|
|
|
|
assert body["status"] == "healthy"
|
|
assert body["mcp_discovery_gaps"] == {"compute": "API not enabled"}
|
|
|
|
|
|
def test_generate_groups_the_execution_under_the_callers_customer_case(tmp_path):
|
|
turns = {
|
|
"orchestrator": [
|
|
[Call("delegate", {"kinds": ["requirements"]})],
|
|
[],
|
|
],
|
|
"requirements-analyst": [submission(DeliverableKind.REQUIREMENTS)],
|
|
}
|
|
client, _ = client_for(tmp_path, turns)
|
|
with client:
|
|
refused = client.post(
|
|
"/generate", json={"request": "Design ingestion", "customer_case": "../x"}
|
|
)
|
|
accepted = client.post(
|
|
"/generate",
|
|
json={"request": "Design ingestion", "customer_case": "acme-webhooks"},
|
|
)
|
|
|
|
assert refused.status_code == 400
|
|
assert "customer_case" in refused.json()["error"]
|
|
assert accepted.status_code == 200, accepted.json()
|
|
assert (tmp_path / "lessons" / "candidates" / "acme-webhooks").is_dir()
|
|
|
|
|
|
# --- a message that starts something is read first (intake) ----------------------------
|
|
|
|
|
|
def test_a_question_is_answered_and_starts_no_execution(tmp_path):
|
|
decider = FakeDecider({"intake": Answer("question", 0.97)})
|
|
client, runtime = client_for(tmp_path, {}, decider)
|
|
with client:
|
|
task = send(client, [{"text": "What can you do?"}])
|
|
|
|
text, data = status_parts(task)
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
assert "Nothing was started" in text
|
|
assert data == {"status": "answered", "intake": "question"}
|
|
assert runtime.runs == []
|
|
assert not (tmp_path / "deliverables").exists()
|
|
|
|
|
|
def test_a_new_request_starts_an_execution_as_before(tmp_path):
|
|
decider = FakeDecider({"intake": Answer("new_request", 0.99)})
|
|
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
|
client, runtime = client_for(tmp_path, turns, decider)
|
|
with client:
|
|
task = send(client, [{"text": "Design an ingestion pipeline."}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
assert "Which region?" in status_parts(task)[0]
|
|
assert len(runtime.runs) == 1
|
|
|
|
|
|
def test_an_unsure_reading_starts_an_execution(tmp_path):
|
|
decider = FakeDecider({"intake": Answer("question", 0.5)})
|
|
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
|
client, runtime = client_for(tmp_path, turns, decider)
|
|
with client:
|
|
task = send(client, [{"text": "hello"}])
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
assert len(runtime.runs) == 1
|
|
|
|
|
|
def test_a_pre_approved_task_is_not_read_as_a_question(tmp_path):
|
|
decider = FakeDecider({"intake": Answer("question", 0.99)})
|
|
client, runtime = client_for(tmp_path, complete_run_turns(), decider)
|
|
with client:
|
|
task = send(
|
|
client, [{"text": "What can you do?"}], metadata={"pre_approved": True}
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
assert decider.asked == []
|
|
assert runtime.runs
|
|
|
|
|
|
def delivered_case(tmp_path: Path, case: str = "acme") -> None:
|
|
folder = tmp_path / "deliverables" / "cases" / case / "intent"
|
|
folder.mkdir(parents=True)
|
|
(folder / "history.json").write_text(
|
|
'{"customer_case": "acme", "intents": [{"version": 1, "workflow_request": '
|
|
'"Design ingestion.", "execution_id": "v1", "status": "completed"}]}'
|
|
)
|
|
|
|
|
|
class Answers:
|
|
"""Stands in for the reply writer answering from delivered files."""
|
|
|
|
def __init__(self, reply):
|
|
self.reply = reply
|
|
self.asked: list[tuple[str, str, str]] = []
|
|
|
|
async def answer(self, question, customer_case, execution_id):
|
|
self.asked.append((question, customer_case, execution_id))
|
|
return self.reply
|
|
|
|
|
|
ABOUT_DELIVERED = {
|
|
"intake": Answer("question", 0.98),
|
|
"about_delivered": Answer(0.92, 0.9),
|
|
}
|
|
|
|
|
|
def test_a_question_about_delivered_work_is_answered_from_it(tmp_path):
|
|
delivered_case(tmp_path)
|
|
answers = Answers("Cloud SQL, because bookings are relational.")
|
|
client, runtime = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), answers)
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": "Why did you choose Cloud SQL?"}],
|
|
metadata={"customer_case": "acme"},
|
|
)
|
|
|
|
text, data = status_parts(task)
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
assert text == "Cloud SQL, because bookings are relational."
|
|
assert data == {"status": "answered", "intake": "question", "about": "v1"}
|
|
assert answers.asked == [("Why did you choose Cloud SQL?", "acme", "v1")]
|
|
assert runtime.runs == []
|
|
|
|
|
|
def test_a_question_that_could_not_be_answered_says_where_to_look(tmp_path):
|
|
delivered_case(tmp_path)
|
|
client, _ = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), Answers(None))
|
|
with client:
|
|
task = send(
|
|
client, [{"text": "Why Cloud SQL?"}], metadata={"customer_case": "acme"}
|
|
)
|
|
|
|
text, data = status_parts(task)
|
|
assert "could not answer that from the documents delivered in Execution v1" in text
|
|
assert data["about"] == "v1"
|
|
|
|
|
|
def test_a_question_about_the_service_is_still_answered_in_general(tmp_path):
|
|
delivered_case(tmp_path)
|
|
decider = FakeDecider(
|
|
{"intake": Answer("question", 0.98), "about_delivered": Answer(0.08, 0.9)}
|
|
)
|
|
answers = Answers("unused")
|
|
client, _ = client_for(tmp_path, {}, decider, answers)
|
|
with client:
|
|
task = send(
|
|
client, [{"text": "What can you do?"}], metadata={"customer_case": "acme"}
|
|
)
|
|
|
|
text, data = status_parts(task)
|
|
assert text.startswith("I design cloud solution architectures.")
|
|
assert data == {"status": "answered", "intake": "question"}
|
|
assert answers.asked == []
|
|
|
|
|
|
def test_a_change_to_delivered_work_names_the_baseline_and_starts_nothing(tmp_path):
|
|
delivered_case(tmp_path)
|
|
decider = FakeDecider({"intake": Answer("change_request", 0.98)})
|
|
client, runtime = client_for(tmp_path, {}, decider)
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": "Availability must now be 99.99%."}],
|
|
metadata={"customer_case": "acme"},
|
|
)
|
|
|
|
text, data = status_parts(task)
|
|
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
|
assert 'send this again with "baseline" set to "v1"' in text
|
|
assert data == {"status": "answered", "intake": "change_request", "baseline": "v1"}
|
|
assert runtime.runs == []
|
|
|
|
|
|
def test_a_caller_who_says_it_is_new_work_is_not_second_guessed(tmp_path):
|
|
delivered_case(tmp_path)
|
|
decider = FakeDecider({"intake": Answer("change_request", 0.98)})
|
|
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
|
client, runtime = client_for(tmp_path, turns, decider)
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": "Availability must now be 99.99%."}],
|
|
metadata={"customer_case": "acme", "new_request": True},
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
|
assert decider.asked == [] and len(runtime.runs) == 1
|
|
|
|
|
|
def test_a_baseline_that_cannot_be_one_fails_the_task_and_generate(tmp_path):
|
|
client, runtime = client_for(tmp_path, {})
|
|
with client:
|
|
task = send(
|
|
client,
|
|
[{"text": "A change."}],
|
|
metadata={"customer_case": "acme", "baseline": "nope"},
|
|
)
|
|
response = client.post(
|
|
"/generate",
|
|
json={"request": "A change.", "customer_case": "acme", "baseline": "nope"},
|
|
)
|
|
|
|
assert task["status"]["state"] == "TASK_STATE_FAILED"
|
|
assert "is not an Execution of Customer case acme" in status_parts(task)[0]
|
|
assert response.status_code == 400
|
|
assert "is not an Execution of Customer case acme" in response.json()["error"]
|
|
assert runtime.runs == []
|