"""Conformance checks for an MCP server that takes its upstream credentials from the MCP Gateway. Run these against a running server before you push it. They need only `pytest` and `httpx`, import nothing from the server and talk to it over HTTP, so they work on a server from the MCP Server Builder and on one written by hand. pip install pytest httpx MCP_SERVER_URL=http://localhost:8100 MCP_REGISTRATION_API_KEY= pytest tests/ -v What is checked, and why: Protocol the server answers /health, refuses callers without its key, and speaks JSON-RPC correctly (initialize, tools/list, ids echoed, errors as errors). Connection a server that needs an upstream account says so in its initialize result (`_meta["com.kyndryl.mcp/connection-schema"]`) and takes the account from each request (`params._meta["com.kyndryl.mcp/connection"]`), not from its own environment. Such a server refuses a call that carries none (fail closed), reports which required field is missing, takes nothing from `arguments`, and never repeats a credential in a reply (the values of fields marked `writeOnly`; every value if none is marked). A server that needs no account skips these checks. Settings (environment variables): MCP_SERVER_URL default http://localhost:8100 MCP_REGISTRATION_API_KEY the key the server was started with (required unless the server has authentication switched off) CONFORMANCE_EXPECT_FAIL_CLOSED default true. Set false only for a server you run locally with REQUIRE_VAULT_CONNECTION=false, so that checks which need a refusal are skipped rather than failed. """ from __future__ import annotations import os import secrets import uuid from typing import Any, Dict, List, Optional import httpx import pytest VENDOR = "com.kyndryl.mcp" SCHEMA_KEY = f"{VENDOR}/connection-schema" VERIFY_TOOL_KEY = f"{VENDOR}/verify-tool" CONNECTION_KEY = f"{VENDOR}/connection" BASE_URL = os.getenv("MCP_SERVER_URL", "http://localhost:8100").rstrip("/") API_KEY = os.getenv("MCP_REGISTRATION_API_KEY", "") EXPECT_FAIL_CLOSED = os.getenv("CONFORMANCE_EXPECT_FAIL_CLOSED", "true").strip().lower() != "false" # A value no server could contain by accident, so finding it in a reply means the server repeated it. SENTINEL = f"sentinel-{secrets.token_hex(6)}" # ------------------------------------------------------------------------------------ helpers def auth_headers(key: Optional[str] = None) -> Dict[str, str]: key = API_KEY if key is None else key return {"Authorization": f"Bearer {key}"} if key else {} def rpc( client: httpx.Client, method: str, params: Optional[Dict[str, Any]] = None, *, request_id: Any = "1", headers: Optional[Dict[str, str]] = None, ) -> httpx.Response: return client.post( "/messages", json={"jsonrpc": "2.0", "id": request_id, "method": method, "params": params or {}}, headers=auth_headers() if headers is None else headers, ) def result_of(client: httpx.Client, method: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: response = rpc(client, method, params) assert response.status_code == 200, f"{method} answered HTTP {response.status_code}: {response.text[:300]}" body = response.json() assert not body.get("error"), f"{method} returned an error: {body['error']}" return body["result"] def with_connection(params: Dict[str, Any], connection: Optional[Dict[str, Any]]) -> Dict[str, Any]: if connection is None: return params return {**params, "_meta": {CONNECTION_KEY: connection}} def value_for(name: str, spec: Dict[str, Any]) -> Any: """A value of the right shape for a connection field, carrying the sentinel.""" kind = spec.get("type", "string") if kind == "string" and spec.get("format") == "uri": return f"https://{SENTINEL}.invalid" if kind == "integer": return 12345 if kind == "boolean": return True return f"{SENTINEL}-{name}" def full_connection(schema: Dict[str, Any]) -> Dict[str, Any]: return {name: value_for(name, spec) for name, spec in (schema.get("properties") or {}).items()} def secret_values(schema: Dict[str, Any]) -> List[str]: """The sent values that must never come back: those of fields marked secret (`writeOnly`). A tool may repeat a base URL or a region. If the server marks no field secret, every value is treated as one, because then nothing says which are safe to repeat. """ connection = full_connection(schema) properties = schema.get("properties") or {} marked = [n for n, spec in properties.items() if spec.get("writeOnly")] return [str(connection[n]) for n in (marked or list(properties))] def repeated(response: httpx.Response, schema: Dict[str, Any]) -> List[str]: return [v for v in secret_values(schema) if v in response.text] def _string_values(node: Any) -> List[str]: """Every string anywhere in a JSON value.""" if isinstance(node, str): return [node] if isinstance(node, dict): return [v for item in node.values() for v in _string_values(item)] if isinstance(node, list): return [v for item in node for v in _string_values(item)] return [] def call(client: httpx.Client, tool: str, connection: Optional[Dict[str, Any]], arguments: Optional[Dict] = None): return rpc(client, "tools/call", with_connection({"name": tool, "arguments": arguments or {}}, connection)) # ------------------------------------------------------------------------------------ fixtures @pytest.fixture(scope="module") def client(): with httpx.Client(base_url=BASE_URL, timeout=30.0) as session: try: session.get("/health") except httpx.HTTPError as error: pytest.exit(f"Cannot reach the server at {BASE_URL}: {error}. Start it, or set MCP_SERVER_URL.", 2) yield session @pytest.fixture(scope="module") def initialize(client) -> Dict[str, Any]: return result_of(client, "initialize", {"protocolVersion": "1.0.0", "clientInfo": {"name": "conformance"}}) @pytest.fixture(scope="module") def tools(client) -> List[Dict[str, Any]]: return result_of(client, "tools/list")["tools"] @pytest.fixture(scope="module") def schema(initialize) -> Optional[Dict[str, Any]]: return (initialize.get("_meta") or {}).get(SCHEMA_KEY) @pytest.fixture(scope="module") def needs_connection(schema) -> Dict[str, Any]: if not schema or not schema.get("properties"): pytest.skip("the server advertises no connection fields, so it takes no per-request credentials") return schema @pytest.fixture(scope="module") def probe_tool(initialize, tools) -> str: """The tool to call in the connection checks: the verify tool when there is one.""" verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY) if verify: return verify assert tools, "the server lists no tools to call" return tools[0]["name"] # ------------------------------------------------------------------------------------ protocol def test_health_answers(client): assert client.get("/health").status_code == 200 def test_the_public_auth_endpoint_reveals_no_key(client): response = client.get("/.well-known/mcp-auth") if response.status_code == 404: pytest.skip("the server has no /.well-known/mcp-auth endpoint") assert response.status_code == 200 if API_KEY: # A value that equals the key is always a leak. A short key can also appear inside ordinary words # ("nc" in "auth_required"), so searching the text for it is only meaningful for a real-length key. assert API_KEY not in _string_values(response.json()), "/.well-known/mcp-auth returns the server's key" if len(API_KEY) >= 12: assert API_KEY not in response.text, "/.well-known/mcp-auth returns the server's key" def test_a_call_without_the_servers_key_is_refused(client): response = rpc(client, "tools/list", headers={}) assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with no key" def test_a_call_with_the_wrong_key_is_refused(client): response = rpc(client, "tools/list", headers=auth_headers("not-the-key-" + uuid.uuid4().hex)) assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with the wrong key" def test_initialize_describes_the_server(initialize): assert initialize["serverInfo"]["name"] assert isinstance(initialize.get("capabilities"), dict) def test_tools_have_a_name_a_description_and_an_object_schema(tools): assert isinstance(tools, list) names = [t["name"] for t in tools] assert len(names) == len(set(names)), "two tools share a name" for tool in tools: assert tool["name"] and tool.get("description"), f"tool {tool.get('name')!r} lacks a name or description" assert tool["inputSchema"]["type"] == "object", f"{tool['name']} has no object inputSchema" @pytest.mark.parametrize("request_id", ["conformance-1", 7]) def test_the_reply_echoes_the_request_id(client, request_id): response = rpc(client, "tools/list", request_id=request_id) assert response.status_code == 200 assert response.json()["id"] == request_id def test_an_unknown_method_is_an_error_not_a_crash(client): response = rpc(client, "no/such/method") assert response.status_code == 200 assert response.json().get("error"), "an unknown method returned no error" def test_an_unknown_tool_is_an_error_not_a_crash(client): response = rpc(client, "tools/call", {"name": f"no_such_tool_{uuid.uuid4().hex[:8]}", "arguments": {}}) assert response.status_code == 200 assert response.json().get("error"), "an unknown tool returned no error" def test_arguments_that_are_not_an_object_are_the_callers_mistake(client, tools, schema): params = {"name": tools[0]["name"], "arguments": ["not", "an", "object"]} # With a connection, when the server wants one: a refusal for lacking it would answer -32602 too response = rpc(client, "tools/call", with_connection(params, full_connection(schema) if schema else None)) assert response.status_code == 200 error = response.json().get("error") assert error and error.get("code") == -32602, f"expected invalid params (-32602), got: {response.text[:300]}" def test_an_invalid_request_does_not_repeat_what_it_carried(): # No "id": the request is not valid JSON-RPC, and what it carried (a connection) must not come back. # It has a connection of its own: a server that answers a bad request by dropping the connection (an # unhandled error makes uvicorn do that) must not make the NEXT check fail with "connection reset". marker = f"{SENTINEL}-carried" try: response = httpx.post( f"{BASE_URL}/messages", json={"jsonrpc": "2.0", "method": "tools/list", "params": {"_meta": {CONNECTION_KEY: {"field": marker}}}}, headers=auth_headers(), timeout=30.0, ) except httpx.TransportError: return # nothing came back, so nothing was repeated assert marker not in response.text, "the reply to an invalid request repeated the request" # ------------------------------------------------------------------------------------ connection def test_the_connection_schema_is_a_json_schema_object(needs_connection): assert needs_connection["type"] == "object" properties = needs_connection["properties"] assert isinstance(properties, dict) and properties assert set(needs_connection.get("required", [])) <= set(properties), "a required field is not a property" def test_the_verify_tool_is_listed(initialize, tools, needs_connection): verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY) assert verify, f"initialize does not name a verify tool in _meta[{VERIFY_TOOL_KEY!r}]" assert verify in [t["name"] for t in tools], f"the verify tool {verify!r} is not in tools/list" def test_a_call_with_no_connection_is_refused(client, needs_connection, probe_tool): if not EXPECT_FAIL_CLOSED: pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false") response = call(client, probe_tool, None) assert response.json().get("error"), ( "a call with no connection was served. The server is using its own environment for the " "upstream account, so every tenant would share one. Refuse it (REQUIRE_VAULT_CONNECTION)." ) def test_a_connection_in_the_arguments_is_not_accepted(client, needs_connection, probe_tool): """The credential travels in `_meta`, which the model never sees; one in `arguments` is not one.""" if not EXPECT_FAIL_CLOSED: pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false") response = call(client, probe_tool, None, arguments=full_connection(needs_connection)) assert response.json().get("error"), "a connection passed as tool arguments was accepted" assert not repeated(response, needs_connection), "the reply repeats a credential that was sent as an argument" def test_a_call_with_a_connection_is_not_refused_for_lacking_one(client, needs_connection, probe_tool): response = call(client, probe_tool, full_connection(needs_connection)) assert response.status_code == 200 error = response.json().get("error") or {} message = str(error.get("message", "")).lower() assert "no connection" not in message and "missing" not in message, ( f"a complete connection was refused as incomplete: {error}" ) def test_an_incomplete_connection_names_what_is_missing(client, needs_connection, probe_tool): required = list(needs_connection.get("required", [])) if not required: pytest.skip("no field is required") connection = full_connection(needs_connection) dropped = required[-1] connection.pop(dropped) if not connection: # The only field is required, and an empty connection counts as none. Send one that has a value # but not the field, which is what a mistyped field name in the credential looks like. connection = {"not_a_field_of_this_server": "x"} response = call(client, probe_tool, connection) error = response.json().get("error") assert error, f"a connection without the required field {dropped!r} was served" assert dropped in str(error.get("message", "")), f"the refusal does not name the missing field {dropped!r}: {error}" assert not repeated(response, needs_connection), "the refusal repeats a credential value" def test_no_reply_repeats_the_credential(client, needs_connection, probe_tool, tools): connection = full_connection(needs_connection) replies = [call(client, probe_tool, connection)] for tool in tools[:5]: replies.append(call(client, tool["name"], connection, arguments={"conformance": "probe"})) replies.append(call(client, "no_such_tool", connection)) for response in replies: assert not repeated(response, needs_connection), f"a reply repeats the credential: {response.text[:300]}" def test_a_malformed_connection_is_no_better_than_none(client, needs_connection, probe_tool): if not EXPECT_FAIL_CLOSED: pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false") for bad in ("just-a-string", ["a", "list"], 42): params = {"name": probe_tool, "arguments": {}, "_meta": {CONNECTION_KEY: bad}} body = rpc(client, "tools/call", params).json() assert body.get("error"), f"a connection of type {type(bad).__name__} was served" def test_resources_and_prompts_are_listed_when_advertised(client, initialize): capabilities = initialize.get("capabilities") or {} if capabilities.get("resources"): assert isinstance(result_of(client, "resources/list")["resources"], list) if capabilities.get("prompts"): assert isinstance(result_of(client, "prompts/list")["prompts"], list)