feat: run role files on the ADK runtime with scoped Skills

Increment 5 of .scratch/solution-architect-agent/spec.md, test-first.

- agents/*.md: provider-neutral role files for the Orchestrator, nine
  Specialists and the pillar advisor, with a shared Specialist protocol.
- app/agents/roles.py: load and validate role files; Skills, MCP
  servers and models come from the provider bundle.
- app/agents/adk_runtime.py: AgentRuntime on google-adk 2.9 with a
  per-run LlmAgent, role-scoped SkillToolset (no run_skill_script),
  step budgets, early stop on turn-ending tools, LiteLLM models.
- app/execution/pillars.py + toolkit.py: design advisor fans out to one
  advisor per WAF pillar in parallel.
- Specialists receive every earlier-Phase Deliverable as input.
- Bundle gains role_skills, pillar_skills and -latest Gemini aliases;
  pin google-adk[extensions]==2.9.0.
- 37 new tests; full suite 158 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-16 09:51:02 +01:00
parent d9214c671a
commit f040493583
31 changed files with 1431 additions and 17 deletions

0
tests/agents/__init__.py Normal file
View File

View File

@@ -0,0 +1,121 @@
"""The Execution engine running real role files on the ADK runtime, with scripted models."""
import json
from collections.abc import AsyncGenerator
from pathlib import Path
import pytest
from google.adk.models.base_llm import BaseLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.genai import types
from langgraph.checkpoint.memory import InMemorySaver
from pydantic import Field
from app.agents.adk_runtime import AdkAgentRuntime
from app.agents.roles import load_roles
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from app.execution.engine import ExecutionEngine
from app.execution.recorder import InMemoryRecorder
from app.execution.toolkit import make_tool_factory
from app.providers import load_provider
pytestmark = pytest.mark.asyncio
AGENTS_DIR = Path(__file__).parents[2] / "agents"
class RoleScriptedLlm(BaseLlm):
"""One scripted response per model call, keyed by the agent's role."""
model: str = "scripted"
script: list = Field(default_factory=list)
requests: list = Field(default_factory=list)
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
self.requests.append(llm_request)
parts = self.script.pop(0)
yield LlmResponse(content=types.Content(role="model", parts=parts))
def call(tool: str, **args) -> list[types.Part]:
return [types.Part(function_call=types.FunctionCall(name=tool, args=args))]
def requirements_markdown() -> str:
sections = DEPENDENCY_GRAPH[DeliverableKind.REQUIREMENTS].sections
return "\n".join(f"## {section}\nDetails." for section in sections)
def task_context(request: LlmRequest) -> dict:
text = request.contents[0].parts[0].text
return json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0])
async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
models = {
"orchestrator": RoleScriptedLlm(
script=[
call("delegate", kinds=["requirements"]),
call("ask_user", questions=[]),
call("delegate", kinds=["requirements"]),
call("ask_user", questions=["Anything to add?"]),
]
),
"requirements-analyst": RoleScriptedLlm(
script=[
call("load_skill", skill_name="google-cloud-solution-architecture"),
call(
"needs_input", questions=["What availability target do you need?"]
),
call("submit_deliverable", content=requirements_markdown()),
]
),
}
runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_factory=lambda role: models[role.name],
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
tool_factory=make_tool_factory(runtime, roles, bundle.pillar_skills),
)
asked = await engine.start("Regional event ingestion")
assert asked.prompt == {
"questions": [
{
"from": "requirements",
"question": "What availability target do you need?",
}
]
}
answered = await engine.resume(asked.execution_id, {"reply": "99.9% in one region"})
assert (
answered.state["ledger"]["deliverables"]["requirements"]["status"]
== "submitted"
)
analyst_second_run = models["requirements-analyst"].requests[-1]
assert task_context(analyst_second_run)["answers"] == [
{
"questions": ["What availability target do you need?"],
"reply": "99.9% in one region",
}
]
orchestrator_prompt = models["orchestrator"].requests[0].config.system_instruction
assert "You coordinate one solution-architecture Execution" in orchestrator_prompt
analyst_prompt = (
models["requirements-analyst"].requests[0].config.system_instruction
)
assert "## How you work" in analyst_prompt

View File

@@ -0,0 +1,256 @@
"""The ADK implementation of AgentRuntime, driven by a scripted model (no network)."""
from collections.abc import AsyncGenerator
from pathlib import Path
import pytest
from google.adk.models.base_llm import BaseLlm
from google.adk.models.lite_llm import LiteLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.genai import types
from pydantic import Field
from app.agents.adk_runtime import AdkAgentRuntime, default_model_factory
from app.agents.runtime import (
AgentTask,
BudgetExhausted,
NeedsInput,
RoleSpec,
Stopped,
Submitted,
ToolResult,
ToolSpec,
TurnEnded,
)
from app.providers import load_provider
pytestmark = pytest.mark.asyncio
SKILLS = {name: meta.path for name, meta in load_provider("gcp").skills.items()}
class ScriptedLlm(BaseLlm):
"""Returns one scripted response per model call and records every request."""
model: str = "scripted"
script: list[list[types.Part]] = Field(default_factory=list)
requests: list[LlmRequest] = Field(default_factory=list)
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
self.requests.append(llm_request)
parts = self.script.pop(0) if self.script else [types.Part(text="(no script)")]
yield LlmResponse(content=types.Content(role="model", parts=parts))
def call(tool: str, **args) -> types.Part:
return types.Part(function_call=types.FunctionCall(name=tool, args=args))
def text(value: str) -> types.Part:
return types.Part(text=value)
def runtime_with(model: ScriptedLlm) -> AdkAgentRuntime:
return AdkAgentRuntime(skill_dirs=SKILLS, model_factory=lambda role: model)
class Recorder:
"""ToolSpecs whose handlers record their calls."""
def __init__(self) -> None:
self.calls: list[tuple[str, dict]] = []
def tool(self, name: str, outcome=None, message: str = "ok") -> ToolSpec:
async def handler(**kwargs) -> ToolResult:
self.calls.append((name, kwargs))
return ToolResult(f"{message}: {kwargs}", outcome)
return ToolSpec(
name,
f"The {name} tool.",
{"type": "object", "properties": {"note": {"type": "string"}}},
handler,
)
ROLE = RoleSpec(
"iac-author",
"Write Terraform. Keep {braces} and {{doubles}} verbatim.",
budget=5,
skills=("google-cloud-solution-architecture",),
)
TASK = AgentTask("iac-author", "Produce deployment_guidance.", {"feedback": ["x"]})
async def test_specialist_loads_its_skill_and_submits():
recorder = Recorder()
submitted = Submitted("# Guidance", {"terraform/main.tf": "resource {}"})
model = ScriptedLlm(
script=[
[call("load_skill", skill_name="google-cloud-solution-architecture")],
[call("submit_deliverable", note="done")],
]
)
outcome = await runtime_with(model).run(
ROLE, TASK, [recorder.tool("submit_deliverable", submitted)]
)
assert outcome == submitted
assert recorder.calls == [("submit_deliverable", {"note": "done"})]
skill_response = _function_responses(model.requests[1])["load_skill"]
assert "Phase 1: Requirements discovery" in str(skill_response)
async def test_agent_is_offered_its_tools_and_skill_tools_without_scripts():
model = ScriptedLlm(script=[[text("nothing to do")]])
await runtime_with(model).run(ROLE, TASK, [Recorder().tool("submit_deliverable")])
request = model.requests[0]
assert set(request.tools_dict) == {
"submit_deliverable",
"list_skills",
"load_skill",
"load_skill_resource",
}
declaration = request.tools_dict["submit_deliverable"]._get_declaration()
assert declaration.parameters_json_schema["properties"] == {
"note": {"type": "string"}
}
async def test_role_without_skills_gets_no_skill_tools():
model = ScriptedLlm(script=[[text("done")]])
role = RoleSpec("decomposer", "Decompose.", budget=3)
await runtime_with(model).run(role, TASK, [Recorder().tool("needs_input")])
assert set(model.requests[0].tools_dict) == {"needs_input"}
async def test_only_the_roles_skills_can_be_loaded():
model = ScriptedLlm(
script=[
[call("load_skill", skill_name="google-cloud-waf-security")],
[text("ok")],
]
)
await runtime_with(model).run(ROLE, TASK, [])
response = _function_responses(model.requests[1])["load_skill"]
assert "error" in response
assert "Well-Architected" not in str(response)
async def test_instructions_and_task_context_reach_the_model_verbatim():
model = ScriptedLlm(script=[[text("done")]])
await runtime_with(model).run(ROLE, TASK, [])
request = model.requests[0]
assert (
"Keep {braces} and {{doubles}} verbatim." in request.config.system_instruction
)
user_text = request.contents[0].parts[0].text
assert "Produce deployment_guidance." in user_text
assert '"feedback": [\n "x"\n ]' in user_text
async def test_turn_ends_as_soon_as_a_tool_returns_an_outcome():
recorder = Recorder()
model = ScriptedLlm(
script=[
[call("delegate"), call("ask_user")],
[call("ask_user")],
]
)
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=10),
TASK,
[
recorder.tool("delegate", TurnEnded("delegate")),
recorder.tool("ask_user", TurnEnded("ask_user")),
],
)
assert outcome == TurnEnded("delegate")
assert [name for name, _ in recorder.calls] == ["delegate"]
assert len(model.requests) == 1
async def test_budget_counts_every_tool_call_and_stops_the_run():
recorder = Recorder()
model = ScriptedLlm(script=[[call("ready_work")] for _ in range(5)])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=2),
TASK,
[recorder.tool("ready_work")],
)
assert isinstance(outcome, BudgetExhausted)
assert "2 tool calls" in outcome.summary
assert len(recorder.calls) == 2
async def test_model_reply_without_a_turn_ending_tool_is_stopped():
model = ScriptedLlm(script=[[text("I need the RTO before I can continue.")]])
outcome = await runtime_with(model).run(ROLE, TASK, [])
assert outcome == Stopped("I need the RTO before I can continue.")
async def test_needs_input_outcome_passes_through():
needs = NeedsInput(("What region?",))
model = ScriptedLlm(script=[[call("needs_input")]])
outcome = await runtime_with(model).run(
ROLE, TASK, [Recorder().tool("needs_input", needs)]
)
assert outcome == needs
async def test_each_run_starts_with_a_fresh_session():
model = ScriptedLlm(script=[[text("first")], [text("second")]])
runtime = runtime_with(model)
await runtime.run(ROLE, TASK, [])
await runtime.run(ROLE, TASK, [])
assert len(model.requests[1].contents) == 1
def test_default_model_factory_wraps_the_role_model_in_litellm():
model = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-flash-latest")
)
assert isinstance(model, LiteLlm)
assert model.model == "gemini/gemini-flash-latest"
with pytest.raises(ValueError, match="model"):
default_model_factory(RoleSpec("diagrammer", "", budget=1))
def test_unknown_skill_is_rejected_when_the_runtime_is_built():
with pytest.raises(ValueError, match="no-such-skill"):
AdkAgentRuntime(
skill_dirs={"no-such-skill": Path("/nonexistent")},
model_factory=lambda role: ScriptedLlm(),
)
def _function_responses(request: LlmRequest) -> dict[str, dict]:
responses = {}
for content in request.contents:
for part in content.parts or []:
if part.function_response:
responses[part.function_response.name] = part.function_response.response
return responses

138
tests/agents/test_roles.py Normal file
View File

@@ -0,0 +1,138 @@
"""Role files (agents/*.md) resolved against a Cloud provider bundle."""
from pathlib import Path
import pytest
from app.agents.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
from app.execution.deliverables import DEPENDENCY_GRAPH
from app.providers import load_provider
AGENTS_DIR = Path(__file__).parents[2] / "agents"
@pytest.fixture(scope="module")
def bundle():
return load_provider("gcp")
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
roles = load_roles(AGENTS_DIR, bundle)
expected = {"orchestrator", PILLAR_ADVISOR}
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
assert set(roles) == expected
for role in roles.values():
assert role.instructions.strip()
def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["orchestrator"].skills == ("google-cloud-solution-architecture",)
assert roles[PILLAR_ADVISOR].skills == ()
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
assert roles["diagrammer"].model == bundle.models.for_role("diagrammer")
assert roles["orchestrator"].model == bundle.models.default
def test_preview_mcp_servers_can_be_excluded(bundle):
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
assert "asset_inventory" not in roles["requirements-analyst"].mcp_servers
def test_budgets_default_by_role_kind(bundle):
roles = load_roles(AGENTS_DIR, bundle, orchestrator_budget=7, specialist_budget=3)
assert roles["orchestrator"].budget == 7
assert roles["packager"].budget == 3
def test_specialists_include_the_shared_protocol_once(bundle):
roles = load_roles(AGENTS_DIR, bundle)
protocol = (AGENTS_DIR / "partials" / "specialist-protocol.md").read_text(
encoding="utf-8"
)
for spec in DEPENDENCY_GRAPH.values():
assert roles[spec.specialist].instructions.count(protocol.strip()) == 1
assert protocol.strip() not in roles["orchestrator"].instructions
def test_role_files_name_no_provider_specific_skills():
for path in AGENTS_DIR.rglob("*.md"):
text = path.read_text(encoding="utf-8").lower()
assert "google-cloud-" not in text, path.name
# --- validation -----------------------------------------------------------------------
def write_agents(tmp_path: Path, bundle, overrides: dict[str, str]) -> Path:
"""Copy the real role files, then overwrite some."""
target = tmp_path / "agents"
for path in AGENTS_DIR.rglob("*.md"):
destination = target / path.relative_to(AGENTS_DIR)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8")
for name, text in overrides.items():
(target / name).write_text(text, encoding="utf-8")
return target
@pytest.mark.parametrize(
("overrides", "message"),
[
({"decomposer.md": "---\nrole: other\n---\nBody"}, "match"),
(
{
"decomposer.md": "---\nrole: decomposer\ndeliverable: requirements\n---\nB"
},
"requirements",
),
({"decomposer.md": "---\nrole: decomposer\n---\nBody"}, "deliverable"),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\ntool_tiers: [write]\n---\nB"
},
"write",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\nincludes: [missing]\n---\nB"
},
"missing",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\nskills: [x]\n---\nB"
},
"skills",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\n---\n "
},
"instructions",
),
],
)
def test_invalid_role_files_are_rejected(tmp_path, bundle, overrides, message):
agents_dir = write_agents(tmp_path, bundle, overrides)
with pytest.raises(RoleConfigError, match=message):
load_roles(agents_dir, bundle)
def test_missing_role_file_is_rejected(tmp_path, bundle):
agents_dir = write_agents(tmp_path, bundle, {})
(agents_dir / "packager.md").unlink()
with pytest.raises(RoleConfigError, match="packager"):
load_roles(agents_dir, bundle)

View File

@@ -9,6 +9,7 @@ from app.execution.deliverables import (
PhaseId,
check_submission,
deliverables_in,
inputs_of,
)
@@ -78,6 +79,25 @@ def test_graph_matches_the_upstream_skill_dependencies():
)
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == (
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.DESIGN_RECOMMENDATIONS,
)
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == (
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.ARCHITECTURE_DIAGRAM,
)
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
DeliverableKind.SOLUTION_GUIDE
}
def test_submission_meeting_the_contract_has_nothing_missing():
content = (
"# Deployment guidance\n"

View File

@@ -151,7 +151,12 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION)
)
dependencies = describer_run.task.context["dependencies"]
assert set(dependencies) == {"architecture_diagram", "product_selection"}
assert list(dependencies) == [
"requirements",
"technical_decomposition",
"product_selection",
"architecture_diagram",
]
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
assert describer_run.task.deliverable == "architecture_description"

View File

@@ -0,0 +1,110 @@
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
import pytest
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.roles import PILLAR_ADVISOR
from app.agents.runtime import RoleSpec, ToolSpec
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.execution.toolkit import make_tool_factory
pytestmark = pytest.mark.asyncio
PILLARS = {
"security": "waf-security",
"reliability": "waf-reliability",
"sustainability": "waf-sustainability",
}
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
def advice(pillar: str) -> list[Call]:
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
def turn_for_pillar(task):
return advice(task.context["pillar"])
async def consult(tool: ToolSpec, **args):
return await tool.handler(**args)
async def test_consults_every_pillar_with_only_its_skill():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="Event ingestion on serverless compute.")
assert result.outcome is None
for pillar in PILLARS:
assert f"## {pillar}" in result.message
assert f"Use strong defaults for {pillar}." in result.message
runs = {run.task.context["pillar"]: run for run in runtime.runs}
assert set(runs) == set(PILLARS)
assert (
runs["security"].task.context["brief"]
== "Event ingestion on serverless compute."
)
assert runtime.runs[0].tools == ("submit_advice",)
async def test_advisors_are_scoped_to_their_pillar_skill():
seen: dict[str, tuple[str, ...]] = {}
class RecordingRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
seen[task.context["pillar"]] = role.skills
return await super().run(role, task, tools)
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
async def test_a_subset_of_pillars_can_be_consulted():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["reliability"])
assert "## reliability" in result.message
assert "## security" not in result.message
async def test_unknown_pillars_are_refused():
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["speed"])
assert result.message.startswith("Refused")
assert "security" in result.message
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
def maybe_fail(task):
if task.context["pillar"] == "reliability":
return [Call("submit_advice", {"content": ""})]
return advice(task.context["pillar"])
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
result = await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
)
assert "Use strong defaults for security." in result.message
assert "## reliability\n\nNo guidance:" in result.message
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
assert [tool.name for tool in design] == [CONSULT_TOOL]
assert list(other) == []

View File

@@ -60,6 +60,20 @@ def test_role_mcp_servers_can_exclude_preview_servers():
assert bundle.mcp_servers_for_role("no-such-role") == ()
def test_gcp_bundle_maps_roles_and_pillars_to_vendored_skills():
bundle = load_provider("gcp")
assert bundle.role_skills["orchestrator"] == ("google-cloud-solution-architecture",)
assert bundle.pillar_skills == {
"security": "google-cloud-waf-security",
"reliability": "google-cloud-waf-reliability",
"operational-excellence": "google-cloud-waf-operational-excellence",
"cost-optimization": "google-cloud-waf-cost-optimization",
"performance-optimization": "google-cloud-waf-performance-optimization",
"sustainability": "google-cloud-waf-sustainability",
}
def test_model_for_role_falls_back_to_default():
bundle = load_provider("gcp")
@@ -154,6 +168,8 @@ def test_minimal_bundle_loads(tmp_path):
"allow",
),
({"role_mcp_servers": {"describer": ["missing"]}}, "missing"),
({"role_skills": {"describer": ["no-such-skill"]}}, "no-such-skill"),
({"pillar_skills": {"security": "no-such-skill"}}, "no-such-skill"),
],
)
def test_invalid_bundles_are_rejected(tmp_path, override, message):