Files
gcp_solution_architecture_a…/tests/execution/test_pillars.py
Jonathan Boniface ed104e5697 feat: add local tools and read-only remote MCP servers
Increment 6 of .scratch/solution-architect-agent/spec.md, test-first.

- app/tools/mcp.py: MCP client and catalog exposing only allowlisted,
  readOnlyHint tools as framework-neutral ToolSpecs; unreachable servers
  and failed calls become discovery gaps; ADC auth with quota project.
- app/tools/local.py: Mermaid structure check and Terraform fmt,
  validate and plan in a scratch copy; apply is never constructed.
- Tool factory gives roles their named local tools and bundle MCP
  servers; pillar advisors get general-tier documentation tools.
- providers/gcp.yaml: 42 allowlisted tools reviewed against live
  tools/list of 12 Google Cloud MCP servers (snapshot in
  providers/reviews/), excluding data-reading and secret-capable tools.
- scripts/list-mcp-tools.py, docs/operations/mcp-access.md (IAM deny
  policy, service account roles), cloud-discovery and iac-validation
  Skills, MCP and Terraform settings, pytest asyncio auto mode.
- 31 new tests; full suite 189 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 10:14:05 +01:00

148 lines
5.2 KiB
Python

"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.roles import PILLAR_ADVISOR
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.execution.toolkit import make_tool_factory
PILLARS = {
"security": "waf-security",
"reliability": "waf-reliability",
"sustainability": "waf-sustainability",
}
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
def advice(pillar: str) -> list[Call]:
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
def turn_for_pillar(task):
return advice(task.context["pillar"])
async def consult(tool: ToolSpec, **args):
return await tool.handler(**args)
async def test_consults_every_pillar_with_only_its_skill():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="Event ingestion on serverless compute.")
assert result.outcome is None
for pillar in PILLARS:
assert f"## {pillar}" in result.message
assert f"Use strong defaults for {pillar}." in result.message
runs = {run.task.context["pillar"]: run for run in runtime.runs}
assert set(runs) == set(PILLARS)
assert (
runs["security"].task.context["brief"]
== "Event ingestion on serverless compute."
)
assert runtime.runs[0].tools == ("submit_advice",)
async def test_advisors_are_scoped_to_their_pillar_skill():
seen: dict[str, tuple[str, ...]] = {}
class RecordingRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
seen[task.context["pillar"]] = role.skills
return await super().run(role, task, tools)
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
async def test_a_subset_of_pillars_can_be_consulted():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["reliability"])
assert "## reliability" in result.message
assert "## security" not in result.message
async def test_unknown_pillars_are_refused():
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["speed"])
assert result.message.startswith("Refused")
assert "security" in result.message
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
def maybe_fail(task):
if task.context["pillar"] == "reliability":
return [Call("submit_advice", {"content": ""})]
return advice(task.context["pillar"])
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
result = await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
)
assert "Use strong defaults for security." in result.message
assert "## reliability\n\nNo guidance:" in result.message
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
async def noop(**_):
return ToolResult("ok")
def tool(name, tier="general"):
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
class Catalog:
def tools_for(self, server_ids):
available = {
"docs": [tool("docs__search")],
"compute": [tool("compute__list_instances", "cloud_read")],
}
return [t for sid in server_ids for t in available.get(sid, [])]
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
factory = make_tool_factory(
runtime,
{PILLAR_ADVISOR: advisor},
{"security": "waf-security"},
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
catalog=Catalog(),
)
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
"mermaid_check"
]
assert [t.name for t in factory(analyst, "requirements")] == [
"compute__list_instances"
]
[consult_tool] = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
)
await consult_tool.handler(brief="b")
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
assert [tool.name for tool in design] == [CONSULT_TOOL]
assert list(other) == []