feat: add local tools and read-only remote MCP servers

Increment 6 of .scratch/solution-architect-agent/spec.md, test-first.

- app/tools/mcp.py: MCP client and catalog exposing only allowlisted,
  readOnlyHint tools as framework-neutral ToolSpecs; unreachable servers
  and failed calls become discovery gaps; ADC auth with quota project.
- app/tools/local.py: Mermaid structure check and Terraform fmt,
  validate and plan in a scratch copy; apply is never constructed.
- Tool factory gives roles their named local tools and bundle MCP
  servers; pillar advisors get general-tier documentation tools.
- providers/gcp.yaml: 42 allowlisted tools reviewed against live
  tools/list of 12 Google Cloud MCP servers (snapshot in
  providers/reviews/), excluding data-reading and secret-capable tools.
- scripts/list-mcp-tools.py, docs/operations/mcp-access.md (IAM deny
  policy, service account roles), cloud-discovery and iac-validation
  Skills, MCP and Terraform settings, pytest asyncio auto mode.
- 31 new tests; full suite 189 passed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-16 10:14:05 +01:00
parent f040493583
commit ed104e5697
37 changed files with 2302 additions and 65 deletions

0
tests/tools/__init__.py Normal file
View File

142
tests/tools/test_local.py Normal file
View File

@@ -0,0 +1,142 @@
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
import shutil
import pytest
from app.tools.local import (
LOCAL_TOOLS,
TerraformRunner,
local_tools,
mermaid_check,
)
needs_terraform = pytest.mark.skipif(
shutil.which("terraform") is None, reason="terraform CLI not installed"
)
VALID_TF = {
"main.tf": 'variable "region" {\n type = string\n}\n\noutput "region" {\n value = var.region\n}\n',
}
# --- mermaid ---------------------------------------------------------------------------
async def test_mermaid_accepts_a_flowchart():
result = await mermaid_check(
"flowchart TD\n A[Client] -->|HTTPS| B(Cloud Run)\n B --> C[(Database)]\n"
)
assert result.message.startswith("Mermaid check passed")
@pytest.mark.parametrize(
("diagram", "problem"),
[
("", "empty"),
("A --> B", "diagram type"),
("flowchart TD\n A[Client --> B", "Unbalanced"),
("%% comment only\n", "diagram type"),
],
)
async def test_mermaid_reports_problems(diagram, problem):
result = await mermaid_check(diagram)
assert result.message.startswith("Mermaid check failed")
assert problem in result.message
# --- terraform ---------------------------------------------------------------------------
@needs_terraform
async def test_fmt_returns_formatted_files_without_touching_the_input():
files = {"main.tf": 'variable "region" {\ntype=string\n}\n'}
result = await TerraformRunner().fmt(files)
assert "main.tf" in result.message
assert " type = string" in result.message
assert files["main.tf"] == 'variable "region" {\ntype=string\n}\n'
@needs_terraform
async def test_fmt_reports_already_formatted_files():
result = await TerraformRunner().fmt(VALID_TF)
assert result.message == "terraform fmt: all files are already formatted."
@needs_terraform
async def test_validate_passes_a_valid_configuration():
result = await TerraformRunner().validate(VALID_TF)
assert result.message.startswith("terraform validate passed")
@needs_terraform
async def test_validate_reports_errors():
result = await TerraformRunner().validate(
{"main.tf": 'output "x" {\n value = var.missing\n}\n'}
)
assert result.message.startswith("terraform validate failed")
assert "missing" in result.message
async def test_unsafe_file_names_are_refused():
runner = TerraformRunner()
for files in ({"../main.tf": ""}, {"/etc/main.tf": ""}, {"notes.txt": ""}, {}):
result = await runner.validate(files)
assert result.message.startswith("Refused"), files
async def test_missing_terraform_binary_is_reported_clearly():
result = await TerraformRunner(binary="terraform-not-installed").validate(VALID_TF)
assert "terraform-not-installed" in result.message
assert "not installed" in result.message
async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
commands: list[list[str]] = []
async def fake_run(self, args, cwd, env=None):
commands.append(list(args))
return 0, "ok"
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
runner = TerraformRunner()
result = await runner.plan(VALID_TF, {"region": "europe-west2"})
refused = await runner.plan(VALID_TF, {"bad name": "x"})
assert result.message.startswith("terraform plan")
assert refused.message.startswith("Refused")
flat = [arg for command in commands for arg in command]
assert "apply" not in flat and "destroy" not in flat
plan = next(command for command in commands if command[1] == "plan")
assert "-lock=false" in plan and "-input=false" in plan
assert "-var=region=europe-west2" in plan
# --- registry ------------------------------------------------------------------------------
def test_local_tools_are_built_by_name_with_their_tier():
tools = local_tools(["mermaid_check", "terraform_plan"], TerraformRunner())
assert [(t.name, t.tier) for t in tools] == [
("mermaid_check", "general"),
("terraform_plan", "cloud_read"),
]
assert set(LOCAL_TOOLS) == {
"mermaid_check",
"terraform_fmt",
"terraform_validate",
"terraform_plan",
}
with pytest.raises(KeyError, match="terraform_apply"):
local_tools(["terraform_apply"], TerraformRunner())

229
tests/tools/test_mcp.py Normal file
View File

@@ -0,0 +1,229 @@
"""Remote MCP servers as read-only, allowlisted ToolSpecs, against a local fake server."""
import asyncio
import socket
import threading
import time
import pytest
import uvicorn
from mcp.server.fastmcp import FastMCP
from mcp.types import ToolAnnotations
from app.execution.tools import gate_cloud_read
from app.providers import McpServerSpec
from app.tools.mcp import (
GoogleAdcAuth,
McpCatalog,
McpConfigError,
NoAuth,
StaticHeadersAuth,
auth_from_config,
tool_name,
)
READ_ONLY = ToolAnnotations(readOnlyHint=True)
def build_server(seen_headers: list[dict]) -> FastMCP:
server = FastMCP("fake-cloud", stateless_http=True, json_response=True)
@server.tool(annotations=READ_ONLY)
def list_projects(filter: str = "") -> str:
"""List projects."""
return f"projects matching {filter!r}: demo-project"
@server.tool(annotations=READ_ONLY)
def get_instance(name: str) -> str:
"""Get an instance."""
return f"instance {name}: RUNNING"
@server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=True))
def delete_instance(name: str) -> str:
"""Delete an instance."""
return f"deleted {name}"
@server.tool()
def unannotated() -> str:
"""No annotations."""
return "?"
@server.tool(annotations=READ_ONLY)
def failing() -> str:
"""Always fails."""
raise RuntimeError("permission denied on project")
app = server.streamable_http_app()
async def recording_app(scope, receive, send):
if scope["type"] == "http":
seen_headers.append({k.decode(): v.decode() for k, v in scope["headers"]})
await app(scope, receive, send)
server.recording_app = recording_app # type: ignore[attr-defined]
return server
@pytest.fixture
def fake_server():
"""A fake MCP server on its own thread and event loop, like a real remote server."""
seen_headers: list[dict] = []
server = build_server(seen_headers)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
uv = uvicorn.Server(
uvicorn.Config(
server.recording_app,
host="127.0.0.1",
port=port,
log_level="warning",
lifespan="off",
)
)
async def serve():
async with server.session_manager.run():
await uv.serve()
thread = threading.Thread(target=asyncio.run, args=(serve(),), daemon=True)
thread.start()
deadline = time.monotonic() + 10
while not uv.started:
if time.monotonic() > deadline:
raise RuntimeError("fake MCP server did not start")
time.sleep(0.02)
yield f"http://127.0.0.1:{port}/mcp", seen_headers
uv.should_exit = True
thread.join(timeout=10)
def spec(url: str, allow, tier="cloud_read", server_id="compute") -> McpServerSpec:
return McpServerSpec(id=server_id, url=url, tier=tier, allow=tuple(allow))
async def test_only_allowlisted_read_only_tools_are_exposed(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects", "get_instance"])], NoAuth()
)
tools = catalog.tools_for(["compute"])
assert [t.name for t in tools] == [
"compute__list_projects",
"compute__get_instance",
]
assert all(t.tier == "cloud_read" for t in tools)
assert tools[1].parameters["required"] == ["name"]
assert catalog.gaps == {}
async def test_calling_a_tool_returns_its_text(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["get_instance"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler(name="web-1")
assert result.message == "instance web-1: RUNNING"
assert result.outcome is None
@pytest.mark.parametrize("tool", ["delete_instance", "unannotated"])
async def test_allowlisting_a_tool_not_marked_read_only_fails_loudly(fake_server, tool):
url, _ = fake_server
with pytest.raises(McpConfigError, match=tool):
await McpCatalog.load([spec(url, [tool])], NoAuth())
async def test_empty_allowlist_exposes_nothing_and_never_connects():
catalog = await McpCatalog.load([spec("http://127.0.0.1:9/mcp", [])], NoAuth())
assert catalog.tools_for(["compute"]) == []
assert catalog.gaps == {}
async def test_allowlisted_tool_missing_upstream_is_recorded_as_a_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects", "renamed"])], NoAuth())
assert [t.name for t in catalog.tools_for(["compute"])] == [
"compute__list_projects"
]
assert "renamed" in catalog.gaps["compute"]
async def test_unreachable_server_becomes_a_discovery_gap_tool():
catalog = await McpCatalog.load(
[spec("http://127.0.0.1:9/mcp", ["list_projects"])], NoAuth(), timeout=2
)
[tool] = catalog.tools_for(["compute"])
result = await tool.handler()
assert tool.name == "compute__unavailable"
assert tool.tier == "cloud_read"
assert result.message.startswith("Discovery gap: compute MCP server unavailable")
assert "compute" in catalog.gaps
async def test_tool_error_is_returned_as_a_discovery_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["failing"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler()
assert result.message.startswith("Discovery gap: compute__failing failed")
assert "permission denied" in result.message
async def test_cloud_read_mcp_tools_are_gated_like_any_tool(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects"])], NoAuth())
tool = catalog.tools_for(["compute"])[0]
asked = await gate_cloud_read(tool, None).handler()
refused = await gate_cloud_read(tool, False).handler()
assert asked.outcome is not None and asked.outcome.permission == "cloud_read"
assert refused.message.startswith("Refused")
async def test_auth_headers_reach_the_server(fake_server):
url, seen = fake_server
await McpCatalog.load(
[spec(url, ["list_projects"])],
StaticHeadersAuth({"Authorization": "Bearer t", "X-Goog-User-Project": "p"}),
)
assert any(h.get("x-goog-user-project") == "p" for h in seen)
assert any(h.get("authorization") == "Bearer t" for h in seen)
async def test_long_results_are_truncated(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects"])], NoAuth(), max_result_chars=20
)
result = await catalog.tools_for(["compute"])[0].handler(filter="x" * 50)
assert result.message.endswith("[truncated]")
def test_tool_names_are_safe_for_every_model_provider():
assert tool_name("asset_inventory", "search.all-resources") == (
"asset_inventory__search_all-resources"
)
assert len(tool_name("s" * 40, "t" * 40)) <= 64
def test_auth_is_chosen_from_bundle_config():
assert isinstance(auth_from_config({"kind": "none"}), NoAuth)
assert isinstance(
auth_from_config({"kind": "google_adc"}, quota_project="q"), GoogleAdcAuth
)
with pytest.raises(McpConfigError, match="oauth"):
auth_from_config({"kind": "oauth"})