Files
cp1-agents/outcome-pattern/solution_architecture/mcp-servers/tests/test_servers.py
Jonathan Boniface bc84ee4df6
Some checks failed
check agents / check (push) Failing after 13s
Solution architecture outcome pattern: agent and its MCP servers
The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate
stream from the individual and sub-agent patterns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

229 lines
9.6 KiB
Python

"""Run once per server: SERVER_DIR=<folder> pytest tests/test_servers.py. Upstreams are simulated; nothing leaves this machine."""
import ast
import asyncio
import json
import os
import re
import sys
from pathlib import Path
import httpx
import pytest
SERVER_DIR = Path(os.environ["SERVER_DIR"]).resolve()
sys.path.insert(0, str(SERVER_DIR))
import mcp_connection # noqa: E402
import server # noqa: E402
NAME = SERVER_DIR.name
HERE = Path(__file__).resolve().parents[1]
CONNECTIONS = {
"sa-gcp-cloud": {"scope_root": "projects/p1", "service_account_json": "{}"},
"sa-azure-cloud": {"tenant_id": "t", "client_id": "c", "client_secret": "s", "scope_root": "subscriptions/sub-1"},
"sa-aws-cloud": {"access_key_id": "a", "secret_access_key": "b", "role_arn": "arn:aws:iam::111111111111:role/r",
"regions": "eu-west-2"},
}
# Every tool the source allows, by the name it has on each server. See README.md for the mapping.
SOURCE_CAPABILITIES = ("search_documents", "get_documents", "analyze", "list_instances", "get_instance_basic_info",
"list_instance_attached_disks", "list_instance_group_managers",
"get_instance_group_manager_basic_info", "list_instance_templates",
"get_instance_template_properties", "list_disks", "get_disk_basic_info", "list_reservations",
"list_commitments", "list_clusters", "get_cluster", "list_node_pools", "get_node_pool",
"get_k8s_version", "list_services", "get_service", "list_buckets", "list_topics", "get_topic",
"list_subscriptions", "list_roles", "get_role", "list_deny_policies", "list_alert_policies",
"get_alert_policy", "list_dashboards", "list_metric_descriptors", "list_timeseries",
"list_connectivity_tests", "get_connectivity_test")
def tool_names():
return [t["name"] for t in server.TOOLS_REGISTRY]
def run(tool, arguments, connection=None):
async def go():
bound = mcp_connection.bind(connection or CONNECTIONS[NAME])
try:
return await getattr(server, tool)(arguments)
finally:
mcp_connection.unbind(bound)
return asyncio.run(go())
def test_every_tool_has_a_handler_and_a_real_check():
names = tool_names()
assert "verify_connection" in names and len(names) == len(set(names))
assert all(callable(getattr(server, n)) for n in names)
assert "does not check it against" not in (SERVER_DIR / "server.py").read_text()
@pytest.mark.parametrize("capability", SOURCE_CAPABILITIES)
def test_each_source_capability_has_a_tool(capability):
assert [n for n in tool_names() if capability in n], capability
def test_a_connection_is_required():
assert server.config.REQUIRE_VAULT_CONNECTION is True
with pytest.raises(ValueError):
server.check_connection(None)
def test_people_and_secrets_are_scrubbed():
text = json.dumps(server._scrub({"tags": {"owner": "Ana Silva"}, "note": "ana@customer.example",
"osProfile": {"adminPassword": "x"}}))
assert "Ana Silva" not in text and "ana@" not in text and '"x"' not in text
def test_failures_do_not_quote_the_upstream():
assert "ana@" not in json.dumps(server._upstream_failed("x", RuntimeError("ana@customer.example")))
def aws_calls():
"""Every (service, operation, result key, token in, token out) the AWS definition names, from the rendered code."""
code = (SERVER_DIR / "server.py").read_text()
entries = re.findall(r'\("([a-z0-9]+)", "([a-z_]+)", "([A-Za-z]+)", "([A-Za-z]+)", "([A-Za-z]+)", \{', code)
gets = re.findall(r'_aws_get\("aws_[a-z0-9_]+", arguments, "([a-z0-9]+)", "([a-z_]+)"', code)
return entries, gets
@pytest.mark.skipif(NAME != "sa-aws-cloud", reason="only AWS names operations")
def test_aws_names_only_read_operations_that_exist():
import boto3
from botocore import xform_name
entries, gets = aws_calls()
assert len(entries) + len(gets) > 35
session = boto3.Session(aws_access_key_id="a", aws_secret_access_key="b", region_name="eu-west-2")
problems = []
for service, operation, *rest in [*entries, *gets]:
if operation.split("_")[0] not in ("list", "describe", "get"):
problems.append(f"{operation} isn't a read")
model = session.client(service).meta.service_model
names = {xform_name(n): n for n in model.operation_names}
if operation not in names:
problems.append(f"{service}.{operation} doesn't exist")
continue
if rest:
shape = model.operation_model(names[operation])
output = shape.output_shape.members if shape.output_shape else {}
inputs = shape.input_shape.members if shape.input_shape else {}
result, token_in, token_out = rest
problems += [f"{operation}: no {n}" for n, there in ((result, output), (token_out, output), (token_in, inputs)) if n not in there]
assert not problems, problems
@pytest.mark.skipif(NAME != "sa-aws-cloud", reason="AWS")
def test_aws_stays_inside_the_connection_regions():
result = run("aws_list_instances", {"region": "us-east-1"})
assert result["error"]["code"] == "outside_scope"
@pytest.mark.skipif(NAME != "sa-aws-cloud", reason="AWS")
def test_aws_get_needs_its_identifier():
assert run("aws_get_role", {})["error"]["code"] == "invalid_argument"
@pytest.mark.skipif(NAME != "sa-azure-cloud", reason="Azure")
def test_azure_refuses_a_quote_in_an_id():
assert run("azure_get_cluster", {"resource_id": "/subscriptions/s/x' or 1==1"})["success"] is False
class Hosted:
"""A hosted Google MCP server."""
def __init__(self, read_only=True, name="list_instances", error=False):
self.read_only, self.name, self.error, self.called, self.urls = read_only, name, error, None, []
def __call__(self, request):
self.urls.append(str(request.url))
body = json.loads(request.content)
method = body.get("method")
if method == "initialize":
return httpx.Response(200, json={"result": {}}, headers={"mcp-session-id": "s1"})
if method == "tools/list":
return httpx.Response(200, json={"result": {"tools": [
{"name": self.name, "annotations": {"readOnlyHint": self.read_only}, "inputSchema": {"type": "object"}}]}})
if method == "tools/call":
self.called = body["params"]
return httpx.Response(200, json={"result": {"isError": self.error, "content": [
{"type": "text", "text": json.dumps({"owner": "ana@customer.example", "name": "vm-1"})}]}})
return httpx.Response(202)
def forward(tool, arguments, hosted):
async def go():
client = httpx.AsyncClient(transport=httpx.MockTransport(hosted))
original, token = server.get_upstream_client, server._gcp_token
server.get_upstream_client = lambda kind, build: client
async def fake():
return "tok"
server._gcp_token = fake
bound = mcp_connection.bind(CONNECTIONS[NAME])
try:
return await getattr(server, tool)(arguments)
finally:
mcp_connection.unbind(bound)
server.get_upstream_client, server._gcp_token = original, token
await client.aclose()
return asyncio.run(go())
gcp = pytest.mark.skipif(NAME != "sa-gcp-cloud", reason="Google Cloud")
@gcp
def test_gcp_exposes_exactly_the_source_allow_list():
source = (HERE / "_support" / "gcp_tools.py").read_text()
catalog = next(ast.literal_eval(n.value) for n in ast.parse(source).body
if isinstance(n, ast.Assign) and n.targets[0].id == "GCP_MCP_SERVERS")
assert sum(len(e["allow"]) for e in catalog.values()) == 42
assert {f"gcp_{s}_{t}" for s, e in catalog.items() for t in e["allow"]} <= set(tool_names())
@gcp
def test_gcp_forwards_arguments_and_scrubs_the_reply():
hosted = Hosted()
result = forward("gcp_compute_list_instances", {"tool_arguments": {"project": "p1"}}, hosted)
assert hosted.called == {"name": "list_instances", "arguments": {"project": "p1"}}
assert hosted.urls[0] == "https://compute.googleapis.com/mcp"
assert result["success"] and "ana@" not in json.dumps(result)
@gcp
def test_gcp_refuses_a_tool_that_is_not_read_only():
hosted = Hosted(read_only=False)
assert forward("gcp_compute_list_instances", {}, hosted)["error"]["code"] == "not_read_only"
assert hosted.called is None
@gcp
def test_gcp_reports_a_refused_call_without_quoting_it():
assert forward("gcp_compute_list_instances", {}, Hosted(error=True))["error"]["code"] == "upstream_refused"
@gcp
def test_gcp_refuses_anything_off_the_allow_list():
assert forward("gcp_describe_tool", {"server": "compute", "tool": "delete_instance"}, Hosted())["error"]["code"] == "not_allowed"
@gcp
def test_gcp_logs_documentation_queries(monkeypatch):
seen = []
monkeypatch.setattr(server, "_docs_query_logged", lambda *a: seen.append(a))
forward("gcp_developer_knowledge_search_documents", {"tool_arguments": {"query": "Cloud SQL limits"}}, Hosted(name="search_documents"))
assert seen and "Cloud SQL limits" in seen[0][2]
@pytest.mark.skipif(NAME != "sa-aws-cloud", reason="AWS")
def test_aws_documentation_reader_gets_the_shape_it_takes(monkeypatch):
sent = {}
async def call(provider, connection, name, arguments):
sent.update(name=name, arguments=arguments)
return [{"text": "page"}]
monkeypatch.setattr(server, "_docs_call", call)
run("aws_get_documents", {"url": "https://docs.aws.amazon.com/x"})
assert sent["name"] == "aws___read_documentation"
assert sent["arguments"] == {"requests": [{"url": "https://docs.aws.amazon.com/x", "max_length": 100000}]}