feat: add local tools and read-only remote MCP servers

Increment 6 of .scratch/solution-architect-agent/spec.md, test-first.

- app/tools/mcp.py: MCP client and catalog exposing only allowlisted,
  readOnlyHint tools as framework-neutral ToolSpecs; unreachable servers
  and failed calls become discovery gaps; ADC auth with quota project.
- app/tools/local.py: Mermaid structure check and Terraform fmt,
  validate and plan in a scratch copy; apply is never constructed.
- Tool factory gives roles their named local tools and bundle MCP
  servers; pillar advisors get general-tier documentation tools.
- providers/gcp.yaml: 42 allowlisted tools reviewed against live
  tools/list of 12 Google Cloud MCP servers (snapshot in
  providers/reviews/), excluding data-reading and secret-capable tools.
- scripts/list-mcp-tools.py, docs/operations/mcp-access.md (IAM deny
  policy, service account roles), cloud-discovery and iac-validation
  Skills, MCP and Terraform settings, pytest asyncio auto mode.
- 31 new tests; full suite 189 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-16 10:14:05 +01:00
parent f040493583
commit ed104e5697
37 changed files with 2302 additions and 65 deletions

View File

@@ -4,7 +4,6 @@ import json
from collections.abc import AsyncGenerator
from pathlib import Path
import pytest
from google.adk.models.base_llm import BaseLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
@@ -20,8 +19,6 @@ from app.execution.recorder import InMemoryRecorder
from app.execution.toolkit import make_tool_factory
from app.providers import load_provider
pytestmark = pytest.mark.asyncio
AGENTS_DIR = Path(__file__).parents[2] / "agents"

View File

@@ -25,8 +25,6 @@ from app.agents.runtime import (
)
from app.providers import load_provider
pytestmark = pytest.mark.asyncio
SKILLS = {name: meta.path for name, meta in load_provider("gcp").skills.items()}

View File

@@ -37,6 +37,14 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
assert roles["orchestrator"].model == bundle.models.default
def test_roles_name_only_the_local_tools_they_need(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["diagrammer"].tools == ("mermaid_check",)
assert roles["validator"].tools == ("terraform_validate", "terraform_plan")
assert roles["requirements-analyst"].tools == ()
def test_preview_mcp_servers_can_be_excluded(bundle):
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
@@ -96,9 +104,9 @@ def write_agents(tmp_path: Path, bundle, overrides: dict[str, str]) -> Path:
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\ntool_tiers: [write]\n---\nB"
"deliverable: technical_decomposition\ntools: [terraform_apply]\n---\nB"
},
"write",
"terraform_apply",
),
(
{

View File

@@ -5,7 +5,6 @@ Driven entirely through ExecutionEngine.start/resume with a scripted agent runti
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
@@ -18,7 +17,6 @@ from app.execution.recorder import InMemoryRecorder
K = DeliverableKind
P = PhaseId
pytestmark = pytest.mark.asyncio
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40),

View File

@@ -1,15 +1,11 @@
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
import pytest
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.roles import PILLAR_ADVISOR
from app.agents.runtime import RoleSpec, ToolSpec
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.execution.toolkit import make_tool_factory
pytestmark = pytest.mark.asyncio
PILLARS = {
"security": "waf-security",
"reliability": "waf-reliability",
@@ -99,6 +95,47 @@ async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
assert "## reliability\n\nNo guidance:" in result.message
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
async def noop(**_):
return ToolResult("ok")
def tool(name, tier="general"):
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
class Catalog:
def tools_for(self, server_ids):
available = {
"docs": [tool("docs__search")],
"compute": [tool("compute__list_instances", "cloud_read")],
}
return [t for sid in server_ids for t in available.get(sid, [])]
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
factory = make_tool_factory(
runtime,
{PILLAR_ADVISOR: advisor},
{"security": "waf-security"},
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
catalog=Catalog(),
)
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
"mermaid_check"
]
assert [t.name for t in factory(analyst, "requirements")] == [
"compute__list_instances"
]
[consult_tool] = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
)
await consult_tool.handler(brief="b")
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)

View File

@@ -12,8 +12,6 @@ from app.execution.checkpointing import open_checkpointer
from app.execution.ledger import LedgerEvent
from app.execution.recorder import DatabaseRecorder
pytestmark = pytest.mark.asyncio
@pytest.fixture
def sqlite_db(tmp_path, monkeypatch):

View File

@@ -177,3 +177,44 @@ def test_invalid_bundles_are_rejected(tmp_path, override, message):
with pytest.raises(ProviderConfigError, match=message):
load_provider("test", root=tmp_path)
REVIEW = REPO_ROOT / "providers" / "reviews" / "gcp-mcp-tools.json"
def test_every_allowlisted_mcp_tool_was_reviewed_as_read_only():
import json
review = json.loads(REVIEW.read_text(encoding="utf-8"))
bundle = load_provider("gcp")
for server in bundle.mcp_servers.values():
assert server.allow, f"{server.id} exposes no tools"
reviewed = {tool["name"]: tool for tool in review[server.id]["tools"]}
for name in server.allow:
assert name in reviewed, (
f"{server.id}.{name} was not in the reviewed tools/list"
)
assert reviewed[name]["read_only"], f"{server.id}.{name} is not read-only"
def test_data_reading_and_secret_capable_tools_stay_excluded():
bundle = load_provider("gcp")
excluded = {
"cloud_sql": {"execute_sql", "execute_sql_readonly"},
"storage": {"read_object", "read_text", "list_objects"},
"gke": {"get_k8s_resource", "describe_k8s_resource"},
}
for server_id, tools in excluded.items():
assert not tools & set(bundle.mcp_servers[server_id].allow), server_id
def test_bundle_places_each_local_tool_in_its_own_tier():
from app.tools.local import LOCAL_TOOLS
bundle = load_provider("gcp")
for tier, names in bundle.tools.items():
for name in names:
assert LOCAL_TOOLS[name][0] == tier, name

0
tests/tools/__init__.py Normal file
View File

142
tests/tools/test_local.py Normal file
View File

@@ -0,0 +1,142 @@
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
import shutil
import pytest
from app.tools.local import (
LOCAL_TOOLS,
TerraformRunner,
local_tools,
mermaid_check,
)
needs_terraform = pytest.mark.skipif(
shutil.which("terraform") is None, reason="terraform CLI not installed"
)
VALID_TF = {
"main.tf": 'variable "region" {\n type = string\n}\n\noutput "region" {\n value = var.region\n}\n',
}
# --- mermaid ---------------------------------------------------------------------------
async def test_mermaid_accepts_a_flowchart():
result = await mermaid_check(
"flowchart TD\n A[Client] -->|HTTPS| B(Cloud Run)\n B --> C[(Database)]\n"
)
assert result.message.startswith("Mermaid check passed")
@pytest.mark.parametrize(
("diagram", "problem"),
[
("", "empty"),
("A --> B", "diagram type"),
("flowchart TD\n A[Client --> B", "Unbalanced"),
("%% comment only\n", "diagram type"),
],
)
async def test_mermaid_reports_problems(diagram, problem):
result = await mermaid_check(diagram)
assert result.message.startswith("Mermaid check failed")
assert problem in result.message
# --- terraform ---------------------------------------------------------------------------
@needs_terraform
async def test_fmt_returns_formatted_files_without_touching_the_input():
files = {"main.tf": 'variable "region" {\ntype=string\n}\n'}
result = await TerraformRunner().fmt(files)
assert "main.tf" in result.message
assert " type = string" in result.message
assert files["main.tf"] == 'variable "region" {\ntype=string\n}\n'
@needs_terraform
async def test_fmt_reports_already_formatted_files():
result = await TerraformRunner().fmt(VALID_TF)
assert result.message == "terraform fmt: all files are already formatted."
@needs_terraform
async def test_validate_passes_a_valid_configuration():
result = await TerraformRunner().validate(VALID_TF)
assert result.message.startswith("terraform validate passed")
@needs_terraform
async def test_validate_reports_errors():
result = await TerraformRunner().validate(
{"main.tf": 'output "x" {\n value = var.missing\n}\n'}
)
assert result.message.startswith("terraform validate failed")
assert "missing" in result.message
async def test_unsafe_file_names_are_refused():
runner = TerraformRunner()
for files in ({"../main.tf": ""}, {"/etc/main.tf": ""}, {"notes.txt": ""}, {}):
result = await runner.validate(files)
assert result.message.startswith("Refused"), files
async def test_missing_terraform_binary_is_reported_clearly():
result = await TerraformRunner(binary="terraform-not-installed").validate(VALID_TF)
assert "terraform-not-installed" in result.message
assert "not installed" in result.message
async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
commands: list[list[str]] = []
async def fake_run(self, args, cwd, env=None):
commands.append(list(args))
return 0, "ok"
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
runner = TerraformRunner()
result = await runner.plan(VALID_TF, {"region": "europe-west2"})
refused = await runner.plan(VALID_TF, {"bad name": "x"})
assert result.message.startswith("terraform plan")
assert refused.message.startswith("Refused")
flat = [arg for command in commands for arg in command]
assert "apply" not in flat and "destroy" not in flat
plan = next(command for command in commands if command[1] == "plan")
assert "-lock=false" in plan and "-input=false" in plan
assert "-var=region=europe-west2" in plan
# --- registry ------------------------------------------------------------------------------
def test_local_tools_are_built_by_name_with_their_tier():
tools = local_tools(["mermaid_check", "terraform_plan"], TerraformRunner())
assert [(t.name, t.tier) for t in tools] == [
("mermaid_check", "general"),
("terraform_plan", "cloud_read"),
]
assert set(LOCAL_TOOLS) == {
"mermaid_check",
"terraform_fmt",
"terraform_validate",
"terraform_plan",
}
with pytest.raises(KeyError, match="terraform_apply"):
local_tools(["terraform_apply"], TerraformRunner())

229
tests/tools/test_mcp.py Normal file
View File

@@ -0,0 +1,229 @@
"""Remote MCP servers as read-only, allowlisted ToolSpecs, against a local fake server."""
import asyncio
import socket
import threading
import time
import pytest
import uvicorn
from mcp.server.fastmcp import FastMCP
from mcp.types import ToolAnnotations
from app.execution.tools import gate_cloud_read
from app.providers import McpServerSpec
from app.tools.mcp import (
GoogleAdcAuth,
McpCatalog,
McpConfigError,
NoAuth,
StaticHeadersAuth,
auth_from_config,
tool_name,
)
READ_ONLY = ToolAnnotations(readOnlyHint=True)
def build_server(seen_headers: list[dict]) -> FastMCP:
server = FastMCP("fake-cloud", stateless_http=True, json_response=True)
@server.tool(annotations=READ_ONLY)
def list_projects(filter: str = "") -> str:
"""List projects."""
return f"projects matching {filter!r}: demo-project"
@server.tool(annotations=READ_ONLY)
def get_instance(name: str) -> str:
"""Get an instance."""
return f"instance {name}: RUNNING"
@server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=True))
def delete_instance(name: str) -> str:
"""Delete an instance."""
return f"deleted {name}"
@server.tool()
def unannotated() -> str:
"""No annotations."""
return "?"
@server.tool(annotations=READ_ONLY)
def failing() -> str:
"""Always fails."""
raise RuntimeError("permission denied on project")
app = server.streamable_http_app()
async def recording_app(scope, receive, send):
if scope["type"] == "http":
seen_headers.append({k.decode(): v.decode() for k, v in scope["headers"]})
await app(scope, receive, send)
server.recording_app = recording_app # type: ignore[attr-defined]
return server
@pytest.fixture
def fake_server():
"""A fake MCP server on its own thread and event loop, like a real remote server."""
seen_headers: list[dict] = []
server = build_server(seen_headers)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
uv = uvicorn.Server(
uvicorn.Config(
server.recording_app,
host="127.0.0.1",
port=port,
log_level="warning",
lifespan="off",
)
)
async def serve():
async with server.session_manager.run():
await uv.serve()
thread = threading.Thread(target=asyncio.run, args=(serve(),), daemon=True)
thread.start()
deadline = time.monotonic() + 10
while not uv.started:
if time.monotonic() > deadline:
raise RuntimeError("fake MCP server did not start")
time.sleep(0.02)
yield f"http://127.0.0.1:{port}/mcp", seen_headers
uv.should_exit = True
thread.join(timeout=10)
def spec(url: str, allow, tier="cloud_read", server_id="compute") -> McpServerSpec:
return McpServerSpec(id=server_id, url=url, tier=tier, allow=tuple(allow))
async def test_only_allowlisted_read_only_tools_are_exposed(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects", "get_instance"])], NoAuth()
)
tools = catalog.tools_for(["compute"])
assert [t.name for t in tools] == [
"compute__list_projects",
"compute__get_instance",
]
assert all(t.tier == "cloud_read" for t in tools)
assert tools[1].parameters["required"] == ["name"]
assert catalog.gaps == {}
async def test_calling_a_tool_returns_its_text(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["get_instance"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler(name="web-1")
assert result.message == "instance web-1: RUNNING"
assert result.outcome is None
@pytest.mark.parametrize("tool", ["delete_instance", "unannotated"])
async def test_allowlisting_a_tool_not_marked_read_only_fails_loudly(fake_server, tool):
url, _ = fake_server
with pytest.raises(McpConfigError, match=tool):
await McpCatalog.load([spec(url, [tool])], NoAuth())
async def test_empty_allowlist_exposes_nothing_and_never_connects():
catalog = await McpCatalog.load([spec("http://127.0.0.1:9/mcp", [])], NoAuth())
assert catalog.tools_for(["compute"]) == []
assert catalog.gaps == {}
async def test_allowlisted_tool_missing_upstream_is_recorded_as_a_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects", "renamed"])], NoAuth())
assert [t.name for t in catalog.tools_for(["compute"])] == [
"compute__list_projects"
]
assert "renamed" in catalog.gaps["compute"]
async def test_unreachable_server_becomes_a_discovery_gap_tool():
catalog = await McpCatalog.load(
[spec("http://127.0.0.1:9/mcp", ["list_projects"])], NoAuth(), timeout=2
)
[tool] = catalog.tools_for(["compute"])
result = await tool.handler()
assert tool.name == "compute__unavailable"
assert tool.tier == "cloud_read"
assert result.message.startswith("Discovery gap: compute MCP server unavailable")
assert "compute" in catalog.gaps
async def test_tool_error_is_returned_as_a_discovery_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["failing"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler()
assert result.message.startswith("Discovery gap: compute__failing failed")
assert "permission denied" in result.message
async def test_cloud_read_mcp_tools_are_gated_like_any_tool(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects"])], NoAuth())
tool = catalog.tools_for(["compute"])[0]
asked = await gate_cloud_read(tool, None).handler()
refused = await gate_cloud_read(tool, False).handler()
assert asked.outcome is not None and asked.outcome.permission == "cloud_read"
assert refused.message.startswith("Refused")
async def test_auth_headers_reach_the_server(fake_server):
url, seen = fake_server
await McpCatalog.load(
[spec(url, ["list_projects"])],
StaticHeadersAuth({"Authorization": "Bearer t", "X-Goog-User-Project": "p"}),
)
assert any(h.get("x-goog-user-project") == "p" for h in seen)
assert any(h.get("authorization") == "Bearer t" for h in seen)
async def test_long_results_are_truncated(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects"])], NoAuth(), max_result_chars=20
)
result = await catalog.tools_for(["compute"])[0].handler(filter="x" * 50)
assert result.message.endswith("[truncated]")
def test_tool_names_are_safe_for_every_model_provider():
assert tool_name("asset_inventory", "search.all-resources") == (
"asset_inventory__search_all-resources"
)
assert len(tool_name("s" * 40, "t" * 40)) <= 64
def test_auth_is_chosen_from_bundle_config():
assert isinstance(auth_from_config({"kind": "none"}), NoAuth)
assert isinstance(
auth_from_config({"kind": "google_adc"}, quota_project="q"), GoogleAdcAuth
)
with pytest.raises(McpConfigError, match="oauth"):
auth_from_config({"kind": "oauth"})