All checks were successful
check agents / check (push) Successful in 10s
ALZ-F01 Discovery as a sub-agent orchestrator (sub-agents/) and as eight individual declarative agents (individual-agents/), with their KAB workflow exports. Shared check, render and import scripts, the alz-discovery-sample MCP server, the crucible-kab-dev deploy profile, deployment docs, platform issue reports and the source material. Agent steps time out at 20 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
335 lines
14 KiB
Python
335 lines
14 KiB
Python
"""Conformance checks for an MCP server that takes its upstream credentials from the MCP Gateway.
|
|
|
|
Run these against a running server before you push it. They need only `pytest` and `httpx`, import
|
|
nothing from the server and talk to it over HTTP, so they work on a server from the MCP Server
|
|
Builder and on one written by hand.
|
|
|
|
pip install pytest httpx
|
|
MCP_SERVER_URL=http://localhost:8100 MCP_REGISTRATION_API_KEY=<the server's key> pytest tests/ -v
|
|
|
|
What is checked, and why:
|
|
|
|
Protocol the server answers /health, refuses callers without its key, and speaks JSON-RPC
|
|
correctly (initialize, tools/list, ids echoed, errors as errors).
|
|
Connection a server that needs an upstream account says so in its initialize result
|
|
(`_meta["com.kyndryl.mcp/connection-schema"]`) and takes the account from each
|
|
request (`params._meta["com.kyndryl.mcp/connection"]`), not from its own environment.
|
|
Such a server refuses a call that carries none (fail closed), reports which required
|
|
field is missing, takes nothing from `arguments`, and never repeats a credential in a
|
|
reply (the values of fields marked `writeOnly`; every value if none is marked). A server that needs no account skips these checks.
|
|
|
|
Settings (environment variables):
|
|
|
|
MCP_SERVER_URL default http://localhost:8100
|
|
MCP_REGISTRATION_API_KEY the key the server was started with (required unless the
|
|
server has authentication switched off)
|
|
CONFORMANCE_EXPECT_FAIL_CLOSED default true. Set false only for a server you run locally
|
|
with REQUIRE_VAULT_CONNECTION=false, so that checks which
|
|
need a refusal are skipped rather than failed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import secrets
|
|
import uuid
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
VENDOR = "com.kyndryl.mcp"
|
|
SCHEMA_KEY = f"{VENDOR}/connection-schema"
|
|
VERIFY_TOOL_KEY = f"{VENDOR}/verify-tool"
|
|
CONNECTION_KEY = f"{VENDOR}/connection"
|
|
|
|
BASE_URL = os.getenv("MCP_SERVER_URL", "http://localhost:8100").rstrip("/")
|
|
API_KEY = os.getenv("MCP_REGISTRATION_API_KEY", "")
|
|
EXPECT_FAIL_CLOSED = os.getenv("CONFORMANCE_EXPECT_FAIL_CLOSED", "true").strip().lower() != "false"
|
|
|
|
# A value no server could contain by accident, so finding it in a reply means the server repeated it.
|
|
SENTINEL = f"sentinel-{secrets.token_hex(6)}"
|
|
|
|
|
|
# ------------------------------------------------------------------------------------ helpers
|
|
|
|
|
|
def auth_headers(key: Optional[str] = None) -> Dict[str, str]:
|
|
key = API_KEY if key is None else key
|
|
return {"Authorization": f"Bearer {key}"} if key else {}
|
|
|
|
|
|
def rpc(
|
|
client: httpx.Client,
|
|
method: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
*,
|
|
request_id: Any = "1",
|
|
headers: Optional[Dict[str, str]] = None,
|
|
) -> httpx.Response:
|
|
return client.post(
|
|
"/messages",
|
|
json={"jsonrpc": "2.0", "id": request_id, "method": method, "params": params or {}},
|
|
headers=auth_headers() if headers is None else headers,
|
|
)
|
|
|
|
|
|
def result_of(client: httpx.Client, method: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
response = rpc(client, method, params)
|
|
assert response.status_code == 200, f"{method} answered HTTP {response.status_code}: {response.text[:300]}"
|
|
body = response.json()
|
|
assert not body.get("error"), f"{method} returned an error: {body['error']}"
|
|
return body["result"]
|
|
|
|
|
|
def with_connection(params: Dict[str, Any], connection: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
if connection is None:
|
|
return params
|
|
return {**params, "_meta": {CONNECTION_KEY: connection}}
|
|
|
|
|
|
def value_for(name: str, spec: Dict[str, Any]) -> Any:
|
|
"""A value of the right shape for a connection field, carrying the sentinel."""
|
|
kind = spec.get("type", "string")
|
|
if kind == "string" and spec.get("format") == "uri":
|
|
return f"https://{SENTINEL}.invalid"
|
|
if kind == "integer":
|
|
return 12345
|
|
if kind == "boolean":
|
|
return True
|
|
return f"{SENTINEL}-{name}"
|
|
|
|
|
|
def full_connection(schema: Dict[str, Any]) -> Dict[str, Any]:
|
|
return {name: value_for(name, spec) for name, spec in (schema.get("properties") or {}).items()}
|
|
|
|
|
|
def secret_values(schema: Dict[str, Any]) -> List[str]:
|
|
"""The sent values that must never come back: those of fields marked secret (`writeOnly`).
|
|
|
|
A tool may repeat a base URL or a region. If the server marks no field secret, every value is
|
|
treated as one, because then nothing says which are safe to repeat.
|
|
"""
|
|
connection = full_connection(schema)
|
|
properties = schema.get("properties") or {}
|
|
marked = [n for n, spec in properties.items() if spec.get("writeOnly")]
|
|
return [str(connection[n]) for n in (marked or list(properties))]
|
|
|
|
|
|
def repeated(response: httpx.Response, schema: Dict[str, Any]) -> List[str]:
|
|
return [v for v in secret_values(schema) if v in response.text]
|
|
|
|
|
|
def _string_values(node: Any) -> List[str]:
|
|
"""Every string anywhere in a JSON value."""
|
|
if isinstance(node, str):
|
|
return [node]
|
|
if isinstance(node, dict):
|
|
return [v for item in node.values() for v in _string_values(item)]
|
|
if isinstance(node, list):
|
|
return [v for item in node for v in _string_values(item)]
|
|
return []
|
|
|
|
|
|
def call(client: httpx.Client, tool: str, connection: Optional[Dict[str, Any]], arguments: Optional[Dict] = None):
|
|
return rpc(client, "tools/call", with_connection({"name": tool, "arguments": arguments or {}}, connection))
|
|
|
|
|
|
# ------------------------------------------------------------------------------------ fixtures
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def client():
|
|
with httpx.Client(base_url=BASE_URL, timeout=30.0) as session:
|
|
try:
|
|
session.get("/health")
|
|
except httpx.HTTPError as error:
|
|
pytest.exit(f"Cannot reach the server at {BASE_URL}: {error}. Start it, or set MCP_SERVER_URL.", 2)
|
|
yield session
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def initialize(client) -> Dict[str, Any]:
|
|
return result_of(client, "initialize", {"protocolVersion": "1.0.0", "clientInfo": {"name": "conformance"}})
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def tools(client) -> List[Dict[str, Any]]:
|
|
return result_of(client, "tools/list")["tools"]
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def schema(initialize) -> Optional[Dict[str, Any]]:
|
|
return (initialize.get("_meta") or {}).get(SCHEMA_KEY)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def needs_connection(schema) -> Dict[str, Any]:
|
|
if not schema or not schema.get("properties"):
|
|
pytest.skip("the server advertises no connection fields, so it takes no per-request credentials")
|
|
return schema
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def probe_tool(initialize, tools) -> str:
|
|
"""The tool to call in the connection checks: the verify tool when there is one."""
|
|
verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY)
|
|
if verify:
|
|
return verify
|
|
assert tools, "the server lists no tools to call"
|
|
return tools[0]["name"]
|
|
|
|
|
|
# ------------------------------------------------------------------------------------ protocol
|
|
|
|
|
|
def test_health_answers(client):
|
|
assert client.get("/health").status_code == 200
|
|
|
|
|
|
def test_the_public_auth_endpoint_reveals_no_key(client):
|
|
response = client.get("/.well-known/mcp-auth")
|
|
if response.status_code == 404:
|
|
pytest.skip("the server has no /.well-known/mcp-auth endpoint")
|
|
assert response.status_code == 200
|
|
if API_KEY:
|
|
# A value that equals the key is always a leak. A short key can also appear inside ordinary words
|
|
# ("nc" in "auth_required"), so searching the text for it is only meaningful for a real-length key.
|
|
assert API_KEY not in _string_values(response.json()), "/.well-known/mcp-auth returns the server's key"
|
|
if len(API_KEY) >= 12:
|
|
assert API_KEY not in response.text, "/.well-known/mcp-auth returns the server's key"
|
|
|
|
|
|
def test_a_call_without_the_servers_key_is_refused(client):
|
|
response = rpc(client, "tools/list", headers={})
|
|
assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with no key"
|
|
|
|
|
|
def test_a_call_with_the_wrong_key_is_refused(client):
|
|
response = rpc(client, "tools/list", headers=auth_headers("not-the-key-" + uuid.uuid4().hex))
|
|
assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with the wrong key"
|
|
|
|
|
|
def test_initialize_describes_the_server(initialize):
|
|
assert initialize["serverInfo"]["name"]
|
|
assert isinstance(initialize.get("capabilities"), dict)
|
|
|
|
|
|
def test_tools_have_a_name_a_description_and_an_object_schema(tools):
|
|
assert isinstance(tools, list)
|
|
names = [t["name"] for t in tools]
|
|
assert len(names) == len(set(names)), "two tools share a name"
|
|
for tool in tools:
|
|
assert tool["name"] and tool.get("description"), f"tool {tool.get('name')!r} lacks a name or description"
|
|
assert tool["inputSchema"]["type"] == "object", f"{tool['name']} has no object inputSchema"
|
|
|
|
|
|
@pytest.mark.parametrize("request_id", ["conformance-1", 7])
|
|
def test_the_reply_echoes_the_request_id(client, request_id):
|
|
response = rpc(client, "tools/list", request_id=request_id)
|
|
assert response.status_code == 200
|
|
assert response.json()["id"] == request_id
|
|
|
|
|
|
def test_an_unknown_method_is_an_error_not_a_crash(client):
|
|
response = rpc(client, "no/such/method")
|
|
assert response.status_code == 200
|
|
assert response.json().get("error"), "an unknown method returned no error"
|
|
|
|
|
|
def test_an_unknown_tool_is_an_error_not_a_crash(client):
|
|
response = rpc(client, "tools/call", {"name": f"no_such_tool_{uuid.uuid4().hex[:8]}", "arguments": {}})
|
|
assert response.status_code == 200
|
|
assert response.json().get("error"), "an unknown tool returned no error"
|
|
|
|
|
|
# ------------------------------------------------------------------------------------ connection
|
|
|
|
|
|
def test_the_connection_schema_is_a_json_schema_object(needs_connection):
|
|
assert needs_connection["type"] == "object"
|
|
properties = needs_connection["properties"]
|
|
assert isinstance(properties, dict) and properties
|
|
assert set(needs_connection.get("required", [])) <= set(properties), "a required field is not a property"
|
|
|
|
|
|
def test_the_verify_tool_is_listed(initialize, tools, needs_connection):
|
|
verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY)
|
|
assert verify, f"initialize does not name a verify tool in _meta[{VERIFY_TOOL_KEY!r}]"
|
|
assert verify in [t["name"] for t in tools], f"the verify tool {verify!r} is not in tools/list"
|
|
|
|
|
|
def test_a_call_with_no_connection_is_refused(client, needs_connection, probe_tool):
|
|
if not EXPECT_FAIL_CLOSED:
|
|
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
|
|
response = call(client, probe_tool, None)
|
|
assert response.json().get("error"), (
|
|
"a call with no connection was served. The server is using its own environment for the "
|
|
"upstream account, so every tenant would share one. Refuse it (REQUIRE_VAULT_CONNECTION)."
|
|
)
|
|
|
|
|
|
def test_a_connection_in_the_arguments_is_not_accepted(client, needs_connection, probe_tool):
|
|
"""The credential travels in `_meta`, which the model never sees; one in `arguments` is not one."""
|
|
if not EXPECT_FAIL_CLOSED:
|
|
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
|
|
response = call(client, probe_tool, None, arguments=full_connection(needs_connection))
|
|
assert response.json().get("error"), "a connection passed as tool arguments was accepted"
|
|
assert not repeated(response, needs_connection), "the reply repeats a credential that was sent as an argument"
|
|
|
|
|
|
def test_a_call_with_a_connection_is_not_refused_for_lacking_one(client, needs_connection, probe_tool):
|
|
response = call(client, probe_tool, full_connection(needs_connection))
|
|
assert response.status_code == 200
|
|
error = response.json().get("error") or {}
|
|
message = str(error.get("message", "")).lower()
|
|
assert "no connection" not in message and "missing" not in message, (
|
|
f"a complete connection was refused as incomplete: {error}"
|
|
)
|
|
|
|
|
|
def test_an_incomplete_connection_names_what_is_missing(client, needs_connection, probe_tool):
|
|
required = list(needs_connection.get("required", []))
|
|
if not required:
|
|
pytest.skip("no field is required")
|
|
connection = full_connection(needs_connection)
|
|
dropped = required[-1]
|
|
connection.pop(dropped)
|
|
if not connection:
|
|
# The only field is required, and an empty connection counts as none. Send one that has a value
|
|
# but not the field, which is what a mistyped field name in the credential looks like.
|
|
connection = {"not_a_field_of_this_server": "x"}
|
|
|
|
response = call(client, probe_tool, connection)
|
|
error = response.json().get("error")
|
|
assert error, f"a connection without the required field {dropped!r} was served"
|
|
assert dropped in str(error.get("message", "")), f"the refusal does not name the missing field {dropped!r}: {error}"
|
|
assert not repeated(response, needs_connection), "the refusal repeats a credential value"
|
|
|
|
|
|
def test_no_reply_repeats_the_credential(client, needs_connection, probe_tool, tools):
|
|
connection = full_connection(needs_connection)
|
|
replies = [call(client, probe_tool, connection)]
|
|
for tool in tools[:5]:
|
|
replies.append(call(client, tool["name"], connection, arguments={"conformance": "probe"}))
|
|
replies.append(call(client, "no_such_tool", connection))
|
|
for response in replies:
|
|
assert not repeated(response, needs_connection), f"a reply repeats the credential: {response.text[:300]}"
|
|
|
|
|
|
def test_a_malformed_connection_is_no_better_than_none(client, needs_connection, probe_tool):
|
|
if not EXPECT_FAIL_CLOSED:
|
|
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
|
|
for bad in ("just-a-string", ["a", "list"], 42):
|
|
params = {"name": probe_tool, "arguments": {}, "_meta": {CONNECTION_KEY: bad}}
|
|
body = rpc(client, "tools/call", params).json()
|
|
assert body.get("error"), f"a connection of type {type(bad).__name__} was served"
|
|
|
|
|
|
def test_resources_and_prompts_are_listed_when_advertised(client, initialize):
|
|
capabilities = initialize.get("capabilities") or {}
|
|
if capabilities.get("resources"):
|
|
assert isinstance(result_of(client, "resources/list")["resources"], list)
|
|
if capabilities.get("prompts"):
|
|
assert isinstance(result_of(client, "prompts/list")["prompts"], list)
|