Files
gcp_solution_architecture_a…/tests/execution/test_engine.py
Jonathan Boniface f040493583 feat: run role files on the ADK runtime with scoped Skills
Increment 5 of .scratch/solution-architect-agent/spec.md, test-first.

- agents/*.md: provider-neutral role files for the Orchestrator, nine
  Specialists and the pillar advisor, with a shared Specialist protocol.
- app/agents/roles.py: load and validate role files; Skills, MCP
  servers and models come from the provider bundle.
- app/agents/adk_runtime.py: AgentRuntime on google-adk 2.9 with a
  per-run LlmAgent, role-scoped SkillToolset (no run_skill_script),
  step budgets, early stop on turn-ending tools, LiteLLM models.
- app/execution/pillars.py + toolkit.py: design advisor fans out to one
  advisor per WAF pillar in parallel.
- Specialists receive every earlier-Phase Deliverable as input.
- Bundle gains role_skills, pillar_skills and -latest Gemini aliases;
  pin google-adk[extensions]==2.9.0.
- 37 new tests; full suite 158 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 09:51:02 +01:00

634 lines
22 KiB
Python

"""The Execution engine: LangGraph state, Specialist fan-out, interrupts and recording.
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
"""
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from app.execution.engine import ExecutionEngine
from app.execution.recorder import InMemoryRecorder
K = DeliverableKind
P = PhaseId
pytestmark = pytest.mark.asyncio
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, f"Produce {spec.kind}.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
# --- scripting helpers ---------------------------------------------------------------
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def request_approval(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": f"{phase} done"})
def ask_user(*questions: str) -> Call:
return Call("ask_user", {"questions": list(questions)})
def reopen(kind: DeliverableKind, reason: str) -> Call:
return Call("reopen", {"kind": kind.value, "reason": reason})
FINISH = Call("finish")
def submission(kind: DeliverableKind, note: str = "") -> Call:
spec = DEPENDENCY_GRAPH[kind]
body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections)
content = f"# {kind}\n{note or 'Summary.'}\n{body}\n"
attachments = {name: f"# {name}\n" for name in spec.attachments}
return Call("submit_deliverable", {"content": content, "attachments": attachments})
def submits(kind: DeliverableKind) -> list[Call]:
return [submission(kind)]
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine:
options = {
"runtime": runtime,
"roles": ROLES,
"recorder": InMemoryRecorder(),
"checkpointer": InMemorySaver(),
"output_root": tmp_path,
}
options.update(overrides)
return ExecutionEngine(**options)
def happy_path_turns(**extra_specialist_turns) -> dict:
turns = {
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
],
**{spec.specialist: [submits(kind)] for kind, spec in DEPENDENCY_GRAPH.items()},
}
turns.update(extra_specialist_turns)
return turns
# --- happy path and parallel start -----------------------------------------------------
async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
recorder = InMemoryRecorder()
engine = make_engine(tmp_path, runtime, recorder=recorder)
result = await engine.start("Event-driven ingestion", pre_approved=True)
assert result.status == "completed"
assert result.prompt is None
ledger = result.state["ledger"]
assert set(ledger["phases"].values()) == {"approved"}
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
out = tmp_path / "deliverables" / "executions" / result.execution_id
assert (out / "solution-architecture-guide.md").is_file()
for kind in K:
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
assert (out / "terraform" / "main.tf").is_file()
assert (out / "architecture.mmd").is_file()
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
recorded_types = [e.type for e in recorder.events[result.execution_id]]
assert recorded_types.count("approved") == 4
async def test_independent_specialists_run_together_between_orchestrator_turns(
tmp_path,
):
runtime = FakeAgentRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
roles = [run.role for run in runtime.runs]
fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3]
assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == {
specialist(K.ARCHITECTURE_DIAGRAM),
specialist(K.DESIGN_RECOMMENDATIONS),
}
assert roles[fourth_turn + 3] == "orchestrator"
async def test_specialist_receives_its_dependencies_content(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
describer_run = next(
r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION)
)
dependencies = describer_run.task.context["dependencies"]
assert list(dependencies) == [
"requirements",
"technical_decomposition",
"product_selection",
"architecture_diagram",
]
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
assert describer_run.task.deliverable == "architecture_description"
# --- refusals reach the agent ------------------------------------------------------------
async def test_orchestrator_cannot_delegate_work_whose_dependencies_are_unmet(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.PRODUCT_SELECTION), delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
}
)
result = await make_engine(tmp_path, runtime).start("Ingestion")
first_turn = runtime.runs[0]
assert first_turn.results[0][0] == "delegate"
assert "Refused" in first_turn.results[0][1]
assert "product_selection" in first_turn.results[0][1]
assert result.status == "awaiting_user"
async def test_incomplete_submission_is_returned_to_the_specialist(tmp_path):
incomplete = Call(
"submit_deliverable", {"content": "## Functional requirements\nx"}
)
runtime = FakeAgentRuntime(
{
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]],
specialist(K.REQUIREMENTS): [[incomplete, submission(K.REQUIREMENTS)]],
}
)
await make_engine(tmp_path, runtime).start("Ingestion")
analyst = runtime.runs[1]
assert "Non-functional requirements" in analyst.results[0][1]
assert analyst.results[1] == ("submit_deliverable", "Submitted requirements.")
# --- interactive approvals ----------------------------------------------------------------
async def test_approval_pauses_the_execution_until_the_user_decides(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[ask_user("Pause here")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingestion")
assert paused.status == "awaiting_user"
assert paused.prompt["approval"]["phase"] == "requirements_discovery"
assert set(paused.prompt["approval"]["deliverables"]) == {
"requirements",
"technical_decomposition",
}
assert (
paused.state["ledger"]["phases"]["requirements_discovery"]
== "awaiting_approval"
)
resumed = await engine.resume(
paused.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved"
assert (
resumed.state["ledger"]["deliverables"]["product_selection"]["status"]
== "submitted"
)
assert resumed.prompt == {
"questions": [{"from": "orchestrator", "question": "Pause here"}]
}
async def test_free_text_reply_never_grants_an_approval(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingestion")
again = await engine.resume(paused.execution_id, {"reply": "approve, looks good"})
assert (
again.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval"
)
assert again.prompt["approval"]["phase"] == "requirements_discovery"
assert again.state["conversation"][-2:] == [
{"from": "user", "text": "approve, looks good"},
{
"from": "orchestrator",
"text": "Please approve requirements_discovery: requirements_discovery done",
},
]
async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[
reopen(K.REQUIREMENTS, "Add the RTO target"),
delegate(K.REQUIREMENTS),
],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
],
specialist(K.REQUIREMENTS): [
submits(K.REQUIREMENTS),
submits(K.REQUIREMENTS),
],
specialist(K.TECHNICAL_DECOMPOSITION): [
submits(K.TECHNICAL_DECOMPOSITION),
submits(K.TECHNICAL_DECOMPOSITION),
],
}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingestion")
second = await engine.resume(
paused.execution_id,
{
"approval": {
"phase": "requirements_discovery",
"decision": "changes",
"feedback": "Add the RTO target",
}
},
)
assert second.prompt["approval"]["phase"] == "requirements_discovery"
ledger = second.state["ledger"]
assert ledger["deliverables"]["requirements"]["revisions"] == 1
second_analyst_run = [
r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)
][1]
assert second_analyst_run.task.context["feedback"] == ["Add the RTO target"]
assert "changes_requested" in [e["type"] for e in ledger["events"]]
# --- one voice ---------------------------------------------------------------------------------
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
tmp_path,
):
def diagrammer_turn(task):
if task.context["answers"]:
return submits(K.ARCHITECTURE_DIAGRAM)
return [Call("needs_input", {"questions": ["Single region or multi-region?"]})]
def advisor_turn(task):
if task.context["answers"]:
return submits(K.DESIGN_RECOMMENDATIONS)
return [Call("needs_input", {"questions": ["What is the RTO?"]})]
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[ask_user()],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[ask_user("Continue?")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn],
specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn],
}
)
engine = make_engine(tmp_path, runtime)
approval = await engine.start("Ingestion")
first = await engine.resume(
approval.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert first.prompt == {
"questions": [
{
"from": "architecture_diagram",
"question": "Single region or multi-region?",
},
{"from": "design_recommendations", "question": "What is the RTO?"},
]
}
second = await engine.resume(
first.execution_id, {"reply": "Single region, RTO 1 hour"}
)
deliverables = second.state["ledger"]["deliverables"]
assert deliverables["architecture_diagram"]["status"] == "submitted"
assert deliverables["design_recommendations"]["status"] == "submitted"
advisor_retry = [
r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS)
][1]
assert advisor_retry.task.context["answers"] == [
{"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"}
]
# --- revision after validation ------------------------------------------------------------
async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp_path):
turns = happy_path_turns()
turns["orchestrator"] = [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
]
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
submits(K.DEPLOYMENT_GUIDANCE),
submits(K.DEPLOYMENT_GUIDANCE),
]
turns[specialist(K.VALIDATION_RESULTS)] = [
[submission(K.VALIDATION_RESULTS, "FAILED: missing IAM binding")],
submits(K.VALIDATION_RESULTS),
]
runtime = FakeAgentRuntime(turns)
result = await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
assert result.status == "completed"
events = result.state["ledger"]["events"]
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
assert withdrawn == ["solution_architecture"]
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
assert requests.count("solution_architecture") == 2
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
assert iac_runs[1].task.context["feedback"] == [
"terraform plan: missing IAM binding"
]
async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[reopen(K.REQUIREMENTS, "not good enough")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
}
)
result = await make_engine(tmp_path, runtime, max_revisions=0).start(
"Ingestion", pre_approved=True
)
assert result.status == "completed_with_warnings"
assert "revision limit" in result.state["warnings"][0]
assert not (tmp_path / "deliverables").exists()
# --- budgets -------------------------------------------------------------------------------------
async def test_orchestrator_budget_exhaustion_hands_control_to_the_user(tmp_path):
roles = {**ROLES, "orchestrator": RoleSpec("orchestrator", "", budget=2)}
wandering = [Call("ready_work"), Call("ready_work"), Call("ready_work")]
runtime = FakeAgentRuntime({"orchestrator": [wandering]})
result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion")
assert result.status == "awaiting_user"
question = result.prompt["questions"][0]
assert question["from"] == "orchestrator"
assert "2 tool calls" in question["question"]
async def test_specialist_budget_exhaustion_is_relayed_as_a_question(tmp_path):
roles = {
**ROLES,
specialist(K.REQUIREMENTS): RoleSpec("requirements-analyst", "", budget=1),
}
runtime = FakeAgentRuntime(
{
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]],
specialist(K.REQUIREMENTS): [
[
Call("submit_deliverable", {"content": "## Current state\nx"}),
submission(K.REQUIREMENTS),
]
],
}
)
result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion")
assert result.prompt["questions"][0]["from"] == "requirements"
assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
# --- cloud read permission -------------------------------------------------------------------
def cloud_read_factory(calls: list[str]):
async def list_projects() -> ToolResult:
calls.append("list_projects")
return ToolResult("projects: demo-project")
tool = ToolSpec(
"list_projects",
"List projects.",
{"type": "object", "properties": {}},
list_projects,
tier="cloud_read",
)
def factory(role: RoleSpec, deliverable: str | None):
return [tool] if role.name == "requirements-analyst" else []
return factory
async def test_cloud_read_asks_permission_once_then_runs_the_tool(tmp_path):
calls: list[str] = []
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.REQUIREMENTS)],
[ask_user("Next?")],
],
specialist(K.REQUIREMENTS): [
[Call("list_projects"), submission(K.REQUIREMENTS)],
[Call("list_projects"), submission(K.REQUIREMENTS)],
],
}
)
engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls))
asked = await engine.start("Ingestion")
assert asked.prompt == {
"permission": {"tier": "cloud_read", "requested_by": ["requirements"]}
}
assert calls == []
granted = await engine.resume(
asked.execution_id, {"permission": {"cloud_read": True}}
)
assert calls == ["list_projects"]
assert granted.state["allow_cloud_read"] is True
assert (
granted.state["ledger"]["deliverables"]["requirements"]["status"] == "submitted"
)
async def test_denied_cloud_read_is_refused_to_the_agent(tmp_path):
calls: list[str] = []
runtime = FakeAgentRuntime(
{
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]],
specialist(K.REQUIREMENTS): [
[Call("list_projects"), submission(K.REQUIREMENTS)]
],
}
)
engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls))
await engine.start("Ingestion", allow_cloud_read=False)
analyst = runtime.runs[1]
assert "not permitted" in analyst.results[0][1]
assert calls == []
# --- isolation and durability -------------------------------------------------------------------
async def test_executions_in_one_session_share_no_state(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user("?")],
[ask_user("fresh?")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
}
)
engine = make_engine(tmp_path, runtime)
first = await engine.start("First", session_id="s-1")
second = await engine.start("Second", session_id="s-1")
assert first.execution_id != second.execution_id
assert second.state["deliverables"] == {}
assert second.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
assert second.state["workflow_request"] == "Second"
assert second.state["session_id"] == "s-1"
async def test_paused_execution_resumes_from_a_durable_checkpoint(tmp_path):
turns = {
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[ask_user("Resumed")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
}
runtime = FakeAgentRuntime(turns)
db_path = str(tmp_path / "checkpoints.db")
async with AsyncSqliteSaver.from_conn_string(db_path) as saver:
paused = await make_engine(tmp_path, runtime, checkpointer=saver).start(
"Ingestion"
)
assert paused.prompt["approval"]["phase"] == "requirements_discovery"
async with AsyncSqliteSaver.from_conn_string(db_path) as saver:
resumed = await make_engine(tmp_path, runtime, checkpointer=saver).resume(
paused.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert resumed.prompt == {
"questions": [{"from": "orchestrator", "question": "Resumed"}]
}
assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved"