148 lines
5.2 KiB
Python
148 lines
5.2 KiB
Python
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
|
|
|
|
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from app.adk.roles import PILLAR_ADVISOR
|
|
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
|
from app.tools.pillars import CONSULT_TOOL, pillar_consultation_tool
|
|
from app.tools.toolkit import make_tool_factory
|
|
|
|
PILLARS = {
|
|
"security": "waf-security",
|
|
"reliability": "waf-reliability",
|
|
"sustainability": "waf-sustainability",
|
|
}
|
|
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
|
|
|
|
|
|
def advice(pillar: str) -> list[Call]:
|
|
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
|
|
|
|
|
|
def turn_for_pillar(task):
|
|
return advice(task.context["pillar"])
|
|
|
|
|
|
async def consult(tool: ToolSpec, **args):
|
|
return await tool.handler(**args)
|
|
|
|
|
|
async def test_consults_every_pillar_with_only_its_skill():
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="Event ingestion on serverless compute.")
|
|
|
|
assert result.outcome is None
|
|
for pillar in PILLARS:
|
|
assert f"## {pillar}" in result.message
|
|
assert f"Use strong defaults for {pillar}." in result.message
|
|
runs = {run.task.context["pillar"]: run for run in runtime.runs}
|
|
assert set(runs) == set(PILLARS)
|
|
assert (
|
|
runs["security"].task.context["brief"]
|
|
== "Event ingestion on serverless compute."
|
|
)
|
|
assert runtime.runs[0].tools == ("submit_advice",)
|
|
|
|
|
|
async def test_advisors_are_scoped_to_their_pillar_skill():
|
|
seen: dict[str, tuple[str, ...]] = {}
|
|
|
|
class RecordingRuntime(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
seen[task.context["pillar"]] = role.skills
|
|
return await super().run(role, task, tools)
|
|
|
|
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
|
|
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
|
|
|
|
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
|
|
|
|
|
|
async def test_a_subset_of_pillars_can_be_consulted():
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="b", pillars=["reliability"])
|
|
|
|
assert "## reliability" in result.message
|
|
assert "## security" not in result.message
|
|
|
|
|
|
async def test_unknown_pillars_are_refused():
|
|
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="b", pillars=["speed"])
|
|
|
|
assert result.message.startswith("Refused")
|
|
assert "security" in result.message
|
|
|
|
|
|
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
|
|
def maybe_fail(task):
|
|
if task.context["pillar"] == "reliability":
|
|
return [Call("submit_advice", {"content": ""})]
|
|
return advice(task.context["pillar"])
|
|
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
|
|
|
|
result = await consult(
|
|
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
|
|
)
|
|
|
|
assert "Use strong defaults for security." in result.message
|
|
assert "## reliability\n\nNo guidance:" in result.message
|
|
|
|
|
|
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
|
|
async def noop(**_):
|
|
return ToolResult("ok")
|
|
|
|
def tool(name, tier="general"):
|
|
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
|
|
|
|
class Catalog:
|
|
def tools_for(self, server_ids):
|
|
available = {
|
|
"docs": [tool("docs__search")],
|
|
"compute": [tool("compute__list_instances", "cloud_read")],
|
|
}
|
|
return [t for sid in server_ids for t in available.get(sid, [])]
|
|
|
|
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
|
factory = make_tool_factory(
|
|
runtime,
|
|
{PILLAR_ADVISOR: advisor},
|
|
{"security": "waf-security"},
|
|
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
|
|
catalog=Catalog(),
|
|
)
|
|
|
|
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
|
|
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
|
|
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
|
|
"mermaid_check"
|
|
]
|
|
assert [t.name for t in factory(analyst, "requirements")] == [
|
|
"compute__list_instances"
|
|
]
|
|
|
|
[consult_tool] = factory(
|
|
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
|
|
)
|
|
await consult_tool.handler(brief="b")
|
|
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
|
|
|
|
|
|
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
|
roles = {PILLAR_ADVISOR: ADVISOR}
|
|
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
|
|
|
|
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
|
|
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
|
|
|
|
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
|
assert list(other) == []
|