feat: run role files on the ADK runtime with scoped Skills
Increment 5 of .scratch/solution-architect-agent/spec.md, test-first. - agents/*.md: provider-neutral role files for the Orchestrator, nine Specialists and the pillar advisor, with a shared Specialist protocol. - app/agents/roles.py: load and validate role files; Skills, MCP servers and models come from the provider bundle. - app/agents/adk_runtime.py: AgentRuntime on google-adk 2.9 with a per-run LlmAgent, role-scoped SkillToolset (no run_skill_script), step budgets, early stop on turn-ending tools, LiteLLM models. - app/execution/pillars.py + toolkit.py: design advisor fans out to one advisor per WAF pillar in parallel. - Specialists receive every earlier-Phase Deliverable as input. - Bundle gains role_skills, pillar_skills and -latest Gemini aliases; pin google-adk[extensions]==2.9.0. - 37 new tests; full suite 158 passed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
0
tests/agents/__init__.py
Normal file
0
tests/agents/__init__.py
Normal file
121
tests/agents/test_adk_engine.py
Normal file
121
tests/agents/test_adk_engine.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""The Execution engine running real role files on the ADK runtime, with scripted models."""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from google.adk.models.base_llm import BaseLlm
|
||||
from google.adk.models.llm_request import LlmRequest
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from pydantic import Field
|
||||
|
||||
from app.agents.adk_runtime import AdkAgentRuntime
|
||||
from app.agents.roles import load_roles
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
from app.execution.engine import ExecutionEngine
|
||||
from app.execution.recorder import InMemoryRecorder
|
||||
from app.execution.toolkit import make_tool_factory
|
||||
from app.providers import load_provider
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
||||
|
||||
|
||||
class RoleScriptedLlm(BaseLlm):
|
||||
"""One scripted response per model call, keyed by the agent's role."""
|
||||
|
||||
model: str = "scripted"
|
||||
script: list = Field(default_factory=list)
|
||||
requests: list = Field(default_factory=list)
|
||||
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
self.requests.append(llm_request)
|
||||
parts = self.script.pop(0)
|
||||
yield LlmResponse(content=types.Content(role="model", parts=parts))
|
||||
|
||||
|
||||
def call(tool: str, **args) -> list[types.Part]:
|
||||
return [types.Part(function_call=types.FunctionCall(name=tool, args=args))]
|
||||
|
||||
|
||||
def requirements_markdown() -> str:
|
||||
sections = DEPENDENCY_GRAPH[DeliverableKind.REQUIREMENTS].sections
|
||||
return "\n".join(f"## {section}\nDetails." for section in sections)
|
||||
|
||||
|
||||
def task_context(request: LlmRequest) -> dict:
|
||||
text = request.contents[0].parts[0].text
|
||||
return json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0])
|
||||
|
||||
|
||||
async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
models = {
|
||||
"orchestrator": RoleScriptedLlm(
|
||||
script=[
|
||||
call("delegate", kinds=["requirements"]),
|
||||
call("ask_user", questions=[]),
|
||||
call("delegate", kinds=["requirements"]),
|
||||
call("ask_user", questions=["Anything to add?"]),
|
||||
]
|
||||
),
|
||||
"requirements-analyst": RoleScriptedLlm(
|
||||
script=[
|
||||
call("load_skill", skill_name="google-cloud-solution-architecture"),
|
||||
call(
|
||||
"needs_input", questions=["What availability target do you need?"]
|
||||
),
|
||||
call("submit_deliverable", content=requirements_markdown()),
|
||||
]
|
||||
),
|
||||
}
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
||||
model_factory=lambda role: models[role.name],
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
tool_factory=make_tool_factory(runtime, roles, bundle.pillar_skills),
|
||||
)
|
||||
|
||||
asked = await engine.start("Regional event ingestion")
|
||||
|
||||
assert asked.prompt == {
|
||||
"questions": [
|
||||
{
|
||||
"from": "requirements",
|
||||
"question": "What availability target do you need?",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
answered = await engine.resume(asked.execution_id, {"reply": "99.9% in one region"})
|
||||
|
||||
assert (
|
||||
answered.state["ledger"]["deliverables"]["requirements"]["status"]
|
||||
== "submitted"
|
||||
)
|
||||
analyst_second_run = models["requirements-analyst"].requests[-1]
|
||||
assert task_context(analyst_second_run)["answers"] == [
|
||||
{
|
||||
"questions": ["What availability target do you need?"],
|
||||
"reply": "99.9% in one region",
|
||||
}
|
||||
]
|
||||
orchestrator_prompt = models["orchestrator"].requests[0].config.system_instruction
|
||||
assert "You coordinate one solution-architecture Execution" in orchestrator_prompt
|
||||
analyst_prompt = (
|
||||
models["requirements-analyst"].requests[0].config.system_instruction
|
||||
)
|
||||
assert "## How you work" in analyst_prompt
|
||||
256
tests/agents/test_adk_runtime.py
Normal file
256
tests/agents/test_adk_runtime.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""The ADK implementation of AgentRuntime, driven by a scripted model (no network)."""
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from google.adk.models.base_llm import BaseLlm
|
||||
from google.adk.models.lite_llm import LiteLlm
|
||||
from google.adk.models.llm_request import LlmRequest
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types
|
||||
from pydantic import Field
|
||||
|
||||
from app.agents.adk_runtime import AdkAgentRuntime, default_model_factory
|
||||
from app.agents.runtime import (
|
||||
AgentTask,
|
||||
BudgetExhausted,
|
||||
NeedsInput,
|
||||
RoleSpec,
|
||||
Stopped,
|
||||
Submitted,
|
||||
ToolResult,
|
||||
ToolSpec,
|
||||
TurnEnded,
|
||||
)
|
||||
from app.providers import load_provider
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
SKILLS = {name: meta.path for name, meta in load_provider("gcp").skills.items()}
|
||||
|
||||
|
||||
class ScriptedLlm(BaseLlm):
|
||||
"""Returns one scripted response per model call and records every request."""
|
||||
|
||||
model: str = "scripted"
|
||||
script: list[list[types.Part]] = Field(default_factory=list)
|
||||
requests: list[LlmRequest] = Field(default_factory=list)
|
||||
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
self.requests.append(llm_request)
|
||||
parts = self.script.pop(0) if self.script else [types.Part(text="(no script)")]
|
||||
yield LlmResponse(content=types.Content(role="model", parts=parts))
|
||||
|
||||
|
||||
def call(tool: str, **args) -> types.Part:
|
||||
return types.Part(function_call=types.FunctionCall(name=tool, args=args))
|
||||
|
||||
|
||||
def text(value: str) -> types.Part:
|
||||
return types.Part(text=value)
|
||||
|
||||
|
||||
def runtime_with(model: ScriptedLlm) -> AdkAgentRuntime:
|
||||
return AdkAgentRuntime(skill_dirs=SKILLS, model_factory=lambda role: model)
|
||||
|
||||
|
||||
class Recorder:
|
||||
"""ToolSpecs whose handlers record their calls."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
def tool(self, name: str, outcome=None, message: str = "ok") -> ToolSpec:
|
||||
async def handler(**kwargs) -> ToolResult:
|
||||
self.calls.append((name, kwargs))
|
||||
return ToolResult(f"{message}: {kwargs}", outcome)
|
||||
|
||||
return ToolSpec(
|
||||
name,
|
||||
f"The {name} tool.",
|
||||
{"type": "object", "properties": {"note": {"type": "string"}}},
|
||||
handler,
|
||||
)
|
||||
|
||||
|
||||
ROLE = RoleSpec(
|
||||
"iac-author",
|
||||
"Write Terraform. Keep {braces} and {{doubles}} verbatim.",
|
||||
budget=5,
|
||||
skills=("google-cloud-solution-architecture",),
|
||||
)
|
||||
TASK = AgentTask("iac-author", "Produce deployment_guidance.", {"feedback": ["x"]})
|
||||
|
||||
|
||||
async def test_specialist_loads_its_skill_and_submits():
|
||||
recorder = Recorder()
|
||||
submitted = Submitted("# Guidance", {"terraform/main.tf": "resource {}"})
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("load_skill", skill_name="google-cloud-solution-architecture")],
|
||||
[call("submit_deliverable", note="done")],
|
||||
]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
ROLE, TASK, [recorder.tool("submit_deliverable", submitted)]
|
||||
)
|
||||
|
||||
assert outcome == submitted
|
||||
assert recorder.calls == [("submit_deliverable", {"note": "done"})]
|
||||
skill_response = _function_responses(model.requests[1])["load_skill"]
|
||||
assert "Phase 1: Requirements discovery" in str(skill_response)
|
||||
|
||||
|
||||
async def test_agent_is_offered_its_tools_and_skill_tools_without_scripts():
|
||||
model = ScriptedLlm(script=[[text("nothing to do")]])
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [Recorder().tool("submit_deliverable")])
|
||||
|
||||
request = model.requests[0]
|
||||
assert set(request.tools_dict) == {
|
||||
"submit_deliverable",
|
||||
"list_skills",
|
||||
"load_skill",
|
||||
"load_skill_resource",
|
||||
}
|
||||
declaration = request.tools_dict["submit_deliverable"]._get_declaration()
|
||||
assert declaration.parameters_json_schema["properties"] == {
|
||||
"note": {"type": "string"}
|
||||
}
|
||||
|
||||
|
||||
async def test_role_without_skills_gets_no_skill_tools():
|
||||
model = ScriptedLlm(script=[[text("done")]])
|
||||
role = RoleSpec("decomposer", "Decompose.", budget=3)
|
||||
|
||||
await runtime_with(model).run(role, TASK, [Recorder().tool("needs_input")])
|
||||
|
||||
assert set(model.requests[0].tools_dict) == {"needs_input"}
|
||||
|
||||
|
||||
async def test_only_the_roles_skills_can_be_loaded():
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("load_skill", skill_name="google-cloud-waf-security")],
|
||||
[text("ok")],
|
||||
]
|
||||
)
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
response = _function_responses(model.requests[1])["load_skill"]
|
||||
assert "error" in response
|
||||
assert "Well-Architected" not in str(response)
|
||||
|
||||
|
||||
async def test_instructions_and_task_context_reach_the_model_verbatim():
|
||||
model = ScriptedLlm(script=[[text("done")]])
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
request = model.requests[0]
|
||||
assert (
|
||||
"Keep {braces} and {{doubles}} verbatim." in request.config.system_instruction
|
||||
)
|
||||
user_text = request.contents[0].parts[0].text
|
||||
assert "Produce deployment_guidance." in user_text
|
||||
assert '"feedback": [\n "x"\n ]' in user_text
|
||||
|
||||
|
||||
async def test_turn_ends_as_soon_as_a_tool_returns_an_outcome():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("delegate"), call("ask_user")],
|
||||
[call("ask_user")],
|
||||
]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=10),
|
||||
TASK,
|
||||
[
|
||||
recorder.tool("delegate", TurnEnded("delegate")),
|
||||
recorder.tool("ask_user", TurnEnded("ask_user")),
|
||||
],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert [name for name, _ in recorder.calls] == ["delegate"]
|
||||
assert len(model.requests) == 1
|
||||
|
||||
|
||||
async def test_budget_counts_every_tool_call_and_stops_the_run():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(script=[[call("ready_work")] for _ in range(5)])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=2),
|
||||
TASK,
|
||||
[recorder.tool("ready_work")],
|
||||
)
|
||||
|
||||
assert isinstance(outcome, BudgetExhausted)
|
||||
assert "2 tool calls" in outcome.summary
|
||||
assert len(recorder.calls) == 2
|
||||
|
||||
|
||||
async def test_model_reply_without_a_turn_ending_tool_is_stopped():
|
||||
model = ScriptedLlm(script=[[text("I need the RTO before I can continue.")]])
|
||||
|
||||
outcome = await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
assert outcome == Stopped("I need the RTO before I can continue.")
|
||||
|
||||
|
||||
async def test_needs_input_outcome_passes_through():
|
||||
needs = NeedsInput(("What region?",))
|
||||
model = ScriptedLlm(script=[[call("needs_input")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
ROLE, TASK, [Recorder().tool("needs_input", needs)]
|
||||
)
|
||||
|
||||
assert outcome == needs
|
||||
|
||||
|
||||
async def test_each_run_starts_with_a_fresh_session():
|
||||
model = ScriptedLlm(script=[[text("first")], [text("second")]])
|
||||
runtime = runtime_with(model)
|
||||
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
|
||||
assert len(model.requests[1].contents) == 1
|
||||
|
||||
|
||||
def test_default_model_factory_wraps_the_role_model_in_litellm():
|
||||
model = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-flash-latest")
|
||||
)
|
||||
|
||||
assert isinstance(model, LiteLlm)
|
||||
assert model.model == "gemini/gemini-flash-latest"
|
||||
with pytest.raises(ValueError, match="model"):
|
||||
default_model_factory(RoleSpec("diagrammer", "", budget=1))
|
||||
|
||||
|
||||
def test_unknown_skill_is_rejected_when_the_runtime_is_built():
|
||||
with pytest.raises(ValueError, match="no-such-skill"):
|
||||
AdkAgentRuntime(
|
||||
skill_dirs={"no-such-skill": Path("/nonexistent")},
|
||||
model_factory=lambda role: ScriptedLlm(),
|
||||
)
|
||||
|
||||
|
||||
def _function_responses(request: LlmRequest) -> dict[str, dict]:
|
||||
responses = {}
|
||||
for content in request.contents:
|
||||
for part in content.parts or []:
|
||||
if part.function_response:
|
||||
responses[part.function_response.name] = part.function_response.response
|
||||
return responses
|
||||
138
tests/agents/test_roles.py
Normal file
138
tests/agents/test_roles.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""Role files (agents/*.md) resolved against a Cloud provider bundle."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agents.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH
|
||||
from app.providers import load_provider
|
||||
|
||||
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bundle():
|
||||
return load_provider("gcp")
|
||||
|
||||
|
||||
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
expected = {"orchestrator", PILLAR_ADVISOR}
|
||||
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
|
||||
assert set(roles) == expected
|
||||
for role in roles.values():
|
||||
assert role.instructions.strip()
|
||||
|
||||
|
||||
def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert roles["orchestrator"].skills == ("google-cloud-solution-architecture",)
|
||||
assert roles[PILLAR_ADVISOR].skills == ()
|
||||
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
|
||||
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
|
||||
assert roles["diagrammer"].model == bundle.models.for_role("diagrammer")
|
||||
assert roles["orchestrator"].model == bundle.models.default
|
||||
|
||||
|
||||
def test_preview_mcp_servers_can_be_excluded(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
|
||||
|
||||
assert "asset_inventory" not in roles["requirements-analyst"].mcp_servers
|
||||
|
||||
|
||||
def test_budgets_default_by_role_kind(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle, orchestrator_budget=7, specialist_budget=3)
|
||||
|
||||
assert roles["orchestrator"].budget == 7
|
||||
assert roles["packager"].budget == 3
|
||||
|
||||
|
||||
def test_specialists_include_the_shared_protocol_once(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
protocol = (AGENTS_DIR / "partials" / "specialist-protocol.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
for spec in DEPENDENCY_GRAPH.values():
|
||||
assert roles[spec.specialist].instructions.count(protocol.strip()) == 1
|
||||
assert protocol.strip() not in roles["orchestrator"].instructions
|
||||
|
||||
|
||||
def test_role_files_name_no_provider_specific_skills():
|
||||
for path in AGENTS_DIR.rglob("*.md"):
|
||||
text = path.read_text(encoding="utf-8").lower()
|
||||
assert "google-cloud-" not in text, path.name
|
||||
|
||||
|
||||
# --- validation -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_agents(tmp_path: Path, bundle, overrides: dict[str, str]) -> Path:
|
||||
"""Copy the real role files, then overwrite some."""
|
||||
target = tmp_path / "agents"
|
||||
for path in AGENTS_DIR.rglob("*.md"):
|
||||
destination = target / path.relative_to(AGENTS_DIR)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
for name, text in overrides.items():
|
||||
(target / name).write_text(text, encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "message"),
|
||||
[
|
||||
({"decomposer.md": "---\nrole: other\n---\nBody"}, "match"),
|
||||
(
|
||||
{
|
||||
"decomposer.md": "---\nrole: decomposer\ndeliverable: requirements\n---\nB"
|
||||
},
|
||||
"requirements",
|
||||
),
|
||||
({"decomposer.md": "---\nrole: decomposer\n---\nBody"}, "deliverable"),
|
||||
(
|
||||
{
|
||||
"decomposer.md": "---\nrole: decomposer\n"
|
||||
"deliverable: technical_decomposition\ntool_tiers: [write]\n---\nB"
|
||||
},
|
||||
"write",
|
||||
),
|
||||
(
|
||||
{
|
||||
"decomposer.md": "---\nrole: decomposer\n"
|
||||
"deliverable: technical_decomposition\nincludes: [missing]\n---\nB"
|
||||
},
|
||||
"missing",
|
||||
),
|
||||
(
|
||||
{
|
||||
"decomposer.md": "---\nrole: decomposer\n"
|
||||
"deliverable: technical_decomposition\nskills: [x]\n---\nB"
|
||||
},
|
||||
"skills",
|
||||
),
|
||||
(
|
||||
{
|
||||
"decomposer.md": "---\nrole: decomposer\n"
|
||||
"deliverable: technical_decomposition\n---\n "
|
||||
},
|
||||
"instructions",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_invalid_role_files_are_rejected(tmp_path, bundle, overrides, message):
|
||||
agents_dir = write_agents(tmp_path, bundle, overrides)
|
||||
|
||||
with pytest.raises(RoleConfigError, match=message):
|
||||
load_roles(agents_dir, bundle)
|
||||
|
||||
|
||||
def test_missing_role_file_is_rejected(tmp_path, bundle):
|
||||
agents_dir = write_agents(tmp_path, bundle, {})
|
||||
(agents_dir / "packager.md").unlink()
|
||||
|
||||
with pytest.raises(RoleConfigError, match="packager"):
|
||||
load_roles(agents_dir, bundle)
|
||||
@@ -9,6 +9,7 @@ from app.execution.deliverables import (
|
||||
PhaseId,
|
||||
check_submission,
|
||||
deliverables_in,
|
||||
inputs_of,
|
||||
)
|
||||
|
||||
|
||||
@@ -78,6 +79,25 @@ def test_graph_matches_the_upstream_skill_dependencies():
|
||||
)
|
||||
|
||||
|
||||
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
|
||||
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
|
||||
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == (
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.DESIGN_RECOMMENDATIONS,
|
||||
)
|
||||
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == (
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.ARCHITECTURE_DIAGRAM,
|
||||
)
|
||||
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
|
||||
DeliverableKind.SOLUTION_GUIDE
|
||||
}
|
||||
|
||||
|
||||
def test_submission_meeting_the_contract_has_nothing_missing():
|
||||
content = (
|
||||
"# Deployment guidance\n"
|
||||
|
||||
@@ -151,7 +151,12 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
|
||||
r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION)
|
||||
)
|
||||
dependencies = describer_run.task.context["dependencies"]
|
||||
assert set(dependencies) == {"architecture_diagram", "product_selection"}
|
||||
assert list(dependencies) == [
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"architecture_diagram",
|
||||
]
|
||||
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
|
||||
assert describer_run.task.deliverable == "architecture_description"
|
||||
|
||||
|
||||
110
tests/execution/test_pillars.py
Normal file
110
tests/execution/test_pillars.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agents.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.agents.roles import PILLAR_ADVISOR
|
||||
from app.agents.runtime import RoleSpec, ToolSpec
|
||||
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
|
||||
from app.execution.toolkit import make_tool_factory
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
PILLARS = {
|
||||
"security": "waf-security",
|
||||
"reliability": "waf-reliability",
|
||||
"sustainability": "waf-sustainability",
|
||||
}
|
||||
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
|
||||
|
||||
|
||||
def advice(pillar: str) -> list[Call]:
|
||||
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
|
||||
|
||||
|
||||
def turn_for_pillar(task):
|
||||
return advice(task.context["pillar"])
|
||||
|
||||
|
||||
async def consult(tool: ToolSpec, **args):
|
||||
return await tool.handler(**args)
|
||||
|
||||
|
||||
async def test_consults_every_pillar_with_only_its_skill():
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="Event ingestion on serverless compute.")
|
||||
|
||||
assert result.outcome is None
|
||||
for pillar in PILLARS:
|
||||
assert f"## {pillar}" in result.message
|
||||
assert f"Use strong defaults for {pillar}." in result.message
|
||||
runs = {run.task.context["pillar"]: run for run in runtime.runs}
|
||||
assert set(runs) == set(PILLARS)
|
||||
assert (
|
||||
runs["security"].task.context["brief"]
|
||||
== "Event ingestion on serverless compute."
|
||||
)
|
||||
assert runtime.runs[0].tools == ("submit_advice",)
|
||||
|
||||
|
||||
async def test_advisors_are_scoped_to_their_pillar_skill():
|
||||
seen: dict[str, tuple[str, ...]] = {}
|
||||
|
||||
class RecordingRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
seen[task.context["pillar"]] = role.skills
|
||||
return await super().run(role, task, tools)
|
||||
|
||||
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
|
||||
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
|
||||
|
||||
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
|
||||
|
||||
|
||||
async def test_a_subset_of_pillars_can_be_consulted():
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="b", pillars=["reliability"])
|
||||
|
||||
assert "## reliability" in result.message
|
||||
assert "## security" not in result.message
|
||||
|
||||
|
||||
async def test_unknown_pillars_are_refused():
|
||||
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="b", pillars=["speed"])
|
||||
|
||||
assert result.message.startswith("Refused")
|
||||
assert "security" in result.message
|
||||
|
||||
|
||||
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
|
||||
def maybe_fail(task):
|
||||
if task.context["pillar"] == "reliability":
|
||||
return [Call("submit_advice", {"content": ""})]
|
||||
return advice(task.context["pillar"])
|
||||
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
|
||||
|
||||
result = await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
|
||||
)
|
||||
|
||||
assert "Use strong defaults for security." in result.message
|
||||
assert "## reliability\n\nNo guidance:" in result.message
|
||||
|
||||
|
||||
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
||||
roles = {PILLAR_ADVISOR: ADVISOR}
|
||||
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
|
||||
|
||||
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
|
||||
|
||||
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
||||
assert list(other) == []
|
||||
@@ -60,6 +60,20 @@ def test_role_mcp_servers_can_exclude_preview_servers():
|
||||
assert bundle.mcp_servers_for_role("no-such-role") == ()
|
||||
|
||||
|
||||
def test_gcp_bundle_maps_roles_and_pillars_to_vendored_skills():
|
||||
bundle = load_provider("gcp")
|
||||
|
||||
assert bundle.role_skills["orchestrator"] == ("google-cloud-solution-architecture",)
|
||||
assert bundle.pillar_skills == {
|
||||
"security": "google-cloud-waf-security",
|
||||
"reliability": "google-cloud-waf-reliability",
|
||||
"operational-excellence": "google-cloud-waf-operational-excellence",
|
||||
"cost-optimization": "google-cloud-waf-cost-optimization",
|
||||
"performance-optimization": "google-cloud-waf-performance-optimization",
|
||||
"sustainability": "google-cloud-waf-sustainability",
|
||||
}
|
||||
|
||||
|
||||
def test_model_for_role_falls_back_to_default():
|
||||
bundle = load_provider("gcp")
|
||||
|
||||
@@ -154,6 +168,8 @@ def test_minimal_bundle_loads(tmp_path):
|
||||
"allow",
|
||||
),
|
||||
({"role_mcp_servers": {"describer": ["missing"]}}, "missing"),
|
||||
({"role_skills": {"describer": ["no-such-skill"]}}, "no-such-skill"),
|
||||
({"pillar_skills": {"security": "no-such-skill"}}, "no-such-skill"),
|
||||
],
|
||||
)
|
||||
def test_invalid_bundles_are_rejected(tmp_path, override, message):
|
||||
|
||||
Reference in New Issue
Block a user