Files

230 lines
7.2 KiB
Python

"""Remote MCP servers as read-only, allowlisted ToolSpecs, against a local fake server."""
import asyncio
import socket
import threading
import time
import pytest
import uvicorn
from mcp.server.fastmcp import FastMCP
from mcp.types import ToolAnnotations
from execution_engine.providers import McpServerSpec
from execution_engine.tools.execution import gate_cloud_read
from execution_engine.tools.mcp import (
GoogleAdcAuth,
McpCatalog,
McpConfigError,
NoAuth,
StaticHeadersAuth,
auth_from_config,
tool_name,
)
READ_ONLY = ToolAnnotations(readOnlyHint=True)
def build_server(seen_headers: list[dict]) -> FastMCP:
server = FastMCP("fake-cloud", stateless_http=True, json_response=True)
@server.tool(annotations=READ_ONLY)
def list_projects(filter: str = "") -> str:
"""List projects."""
return f"projects matching {filter!r}: demo-project"
@server.tool(annotations=READ_ONLY)
def get_instance(name: str) -> str:
"""Get an instance."""
return f"instance {name}: RUNNING"
@server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=True))
def delete_instance(name: str) -> str:
"""Delete an instance."""
return f"deleted {name}"
@server.tool()
def unannotated() -> str:
"""No annotations."""
return "?"
@server.tool(annotations=READ_ONLY)
def failing() -> str:
"""Always fails."""
raise RuntimeError("permission denied on project")
app = server.streamable_http_app()
async def recording_app(scope, receive, send):
if scope["type"] == "http":
seen_headers.append({k.decode(): v.decode() for k, v in scope["headers"]})
await app(scope, receive, send)
server.recording_app = recording_app # type: ignore[attr-defined]
return server
@pytest.fixture
def fake_server():
"""A fake MCP server on its own thread and event loop, like a real remote server."""
seen_headers: list[dict] = []
server = build_server(seen_headers)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
uv = uvicorn.Server(
uvicorn.Config(
server.recording_app,
host="127.0.0.1",
port=port,
log_level="warning",
lifespan="off",
)
)
async def serve():
async with server.session_manager.run():
await uv.serve()
thread = threading.Thread(target=asyncio.run, args=(serve(),), daemon=True)
thread.start()
deadline = time.monotonic() + 10
while not uv.started:
if time.monotonic() > deadline:
raise RuntimeError("fake MCP server did not start")
time.sleep(0.02)
yield f"http://127.0.0.1:{port}/mcp", seen_headers
uv.should_exit = True
thread.join(timeout=10)
def spec(url: str, allow, tier="cloud_read", server_id="compute") -> McpServerSpec:
return McpServerSpec(id=server_id, url=url, tier=tier, allow=tuple(allow))
async def test_only_allowlisted_read_only_tools_are_exposed(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects", "get_instance"])], NoAuth()
)
tools = catalog.tools_for(["compute"])
assert [t.name for t in tools] == [
"compute__list_projects",
"compute__get_instance",
]
assert all(t.tier == "cloud_read" for t in tools)
assert tools[1].parameters["required"] == ["name"]
assert catalog.gaps == {}
async def test_calling_a_tool_returns_its_text(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["get_instance"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler(name="web-1")
assert result.message == "instance web-1: RUNNING"
assert result.outcome is None
@pytest.mark.parametrize("tool", ["delete_instance", "unannotated"])
async def test_allowlisting_a_tool_not_marked_read_only_fails_loudly(fake_server, tool):
url, _ = fake_server
with pytest.raises(McpConfigError, match=tool):
await McpCatalog.load([spec(url, [tool])], NoAuth())
async def test_empty_allowlist_exposes_nothing_and_never_connects():
catalog = await McpCatalog.load([spec("http://127.0.0.1:9/mcp", [])], NoAuth())
assert catalog.tools_for(["compute"]) == []
assert catalog.gaps == {}
async def test_allowlisted_tool_missing_upstream_is_recorded_as_a_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects", "renamed"])], NoAuth())
assert [t.name for t in catalog.tools_for(["compute"])] == [
"compute__list_projects"
]
assert "renamed" in catalog.gaps["compute"]
async def test_unreachable_server_becomes_a_discovery_gap_tool():
catalog = await McpCatalog.load(
[spec("http://127.0.0.1:9/mcp", ["list_projects"])], NoAuth(), timeout=2
)
[tool] = catalog.tools_for(["compute"])
result = await tool.handler()
assert tool.name == "compute__unavailable"
assert tool.tier == "cloud_read"
assert result.message.startswith("Discovery gap: compute MCP server unavailable")
assert "compute" in catalog.gaps
async def test_tool_error_is_returned_as_a_discovery_gap(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["failing"])], NoAuth())
result = await catalog.tools_for(["compute"])[0].handler()
assert result.message.startswith("Discovery gap: compute__failing failed")
assert "permission denied" in result.message
async def test_auth_headers_reach_the_server(fake_server):
url, seen = fake_server
await McpCatalog.load(
[spec(url, ["list_projects"])],
StaticHeadersAuth({"Authorization": "Bearer t", "X-Goog-User-Project": "p"}),
)
assert any(h.get("x-goog-user-project") == "p" for h in seen)
assert any(h.get("authorization") == "Bearer t" for h in seen)
async def test_long_results_are_truncated(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load(
[spec(url, ["list_projects"])], NoAuth(), max_result_chars=20
)
result = await catalog.tools_for(["compute"])[0].handler(filter="x" * 50)
assert result.message.endswith("[truncated]")
def test_tool_names_are_safe_for_every_model_provider():
assert tool_name("asset_inventory", "search.all-resources") == (
"asset_inventory__search_all-resources"
)
assert len(tool_name("s" * 40, "t" * 40)) <= 64
def test_auth_is_chosen_from_bundle_config():
assert isinstance(auth_from_config({"kind": "none"}), NoAuth)
assert isinstance(
auth_from_config({"kind": "google_adc"}, quota_project="q"), GoogleAdcAuth
)
with pytest.raises(McpConfigError, match="oauth"):
auth_from_config({"kind": "oauth"})
async def test_cloud_read_mcp_tools_are_gated_like_any_tool(fake_server):
url, _ = fake_server
catalog = await McpCatalog.load([spec(url, ["list_projects"])], NoAuth())
tool = catalog.tools_for(["compute"])[0]
asked = await gate_cloud_read(tool, None).handler()
refused = await gate_cloud_read(tool, False).handler()
assert asked.outcome is not None and asked.outcome.permission == "cloud_read"
assert refused.message.startswith("Refused")