Add hyperscaler discovery MCP servers and Well-Architected guidance
Some checks failed
check agents / check (push) Failing after 12s

azure-discovery, aws-discovery and gcp-discovery collect read-only metadata
on servers, workloads, Kubernetes clusters, networks, data stores, policies
and provider recommendations. Credentials arrive per call from the vault.
Replies are paged, scrubbed of secrets and anonymised, and throttled calls
back off. cloud-knowledge searches each provider's Well-Architected guidance
from fixed lists only, so no customer detail can leave the platform.

Adds shared skills, the render and test tooling, a public Python Dockerfile
until the hardened base image is reachable, the customer access guide, the
implementation plan and the Vault Service platform issue.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 20:11:57 +00:00
parent 416c66ced4
commit c14a94f33b
91 changed files with 13296 additions and 1 deletions

View File

@@ -0,0 +1,223 @@
import asyncio
import json
import sys
from pathlib import Path
import httpx
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import mcp_connection # noqa: E402
import server # noqa: E402
CONNECTION = {"tenant_id": "tenant-1", "client_id": "client", "client_secret": "s3cret",
"scope_root": "managementGroups/mg-root"}
class Azure:
def __init__(self, rows=None, skip=None, status=200):
self.rows, self.skip, self.status, self.queries = rows or [], skip, status, []
def __call__(self, request):
if "login.microsoftonline.com" in str(request.url):
assert b"client_secret=s3cret" in request.content
return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
assert request.headers["Authorization"] == "Bearer tok"
assert "Microsoft.ResourceGraph" in str(request.url)
self.queries.append(json.loads(request.content))
if self.status != 200:
return httpx.Response(self.status, json={"error": "said too much about ana@customer.example"})
body = {"data": self.rows, "count": len(self.rows)}
if self.skip:
body["$skipToken"] = self.skip
return httpx.Response(200, json=body)
def call(tool, arguments, azure, connection=CONNECTION):
server.AZURE_TOKENS.clear()
async def run():
client = httpx.AsyncClient(transport=httpx.MockTransport(azure))
original = server.get_upstream_client
server.get_upstream_client = lambda kind, build: client
bound = mcp_connection.bind(connection)
try:
return await getattr(server, tool)(arguments)
finally:
mcp_connection.unbind(bound)
server.get_upstream_client = original
await client.aclose()
return asyncio.run(run())
def test_every_query_is_scoped_by_the_connection():
azure = Azure([{"id": "/subscriptions/1/x"}])
result = call("azure_list_servers", {}, azure)
assert result["success"]
assert azure.queries[0]["managementGroups"] == ["mg-root"]
assert "microsoft.compute/virtualmachines" in azure.queries[0]["query"]
def test_a_caller_cannot_widen_the_scope():
azure = Azure([])
call("azure_query_resources", {"query": "Resources | limit 1", "managementGroups": ["other"]}, azure)
assert azure.queries[0]["managementGroups"] == ["mg-root"]
def test_query_must_start_with_a_known_table():
azure = Azure([])
result = call("azure_query_resources", {"query": "securityresources | limit 1"}, azure)
assert result["error"]["code"] == "invalid_argument" and not azure.queries
def test_resource_outside_scope_is_refused():
result = call("azure_get_resource_metadata", {"resource_id": "/subscriptions/other/resourceGroups/rg/x/y"}, Azure([]))
assert result["error"]["code"] == "outside_scope"
def test_resource_id_cannot_inject_kql():
azure = Azure([])
result = call("azure_get_resource_metadata", {"resource_id": "/subscriptions/1' or 1==1 //"}, azure)
assert result["error"]["code"] == "invalid_argument" and not azure.queries
def test_resource_type_cannot_inject_kql():
azure = Azure([])
result = call("azure_list_resources", {"resource_type": "x' or 1==1 //"}, azure)
assert result["error"]["code"] == "invalid_argument" and not azure.queries
def test_vm_secrets_and_owner_are_removed():
vm = {"id": "/subscriptions/1/vm", "tags": {"owner": "Ana"},
"properties": {"osProfile": {"customData": "c2VjcmV0", "adminUsername": "azureuser"}}}
text = json.dumps(call("azure_list_servers", {}, Azure([vm])))
assert "c2VjcmV0" not in text and '"Ana"' not in text
def test_key_vault_principals_are_anonymised():
vault = {"id": "/v", "properties": {"accessPolicies": [{"objectId": "person-guid", "permissions": {"secrets": ["get"]}}]}}
text = json.dumps(call("azure_list_resource_policies", {"kind": "key_vault_access"}, Azure([vault])))
assert "person-guid" not in text and '"get"' in text
def test_paging_passes_the_skip_token():
azure = Azure([{"id": "a"}], skip="more")
first = call("azure_list_servers", {}, azure)["data"]
call("azure_list_servers", {"page_token": first["next_page_token"]}, azure)
assert azure.queries[1]["options"]["$skipToken"] == "more"
def test_advisor_categories_map_to_pillars():
row = {"id": "r", "properties": {"category": "HighAvailability"}}
item = call("azure_list_recommendations", {}, Azure([row]))["data"]["items"][0]
assert item["pillar"] == "reliability" and item["provider_category"] == "HighAvailability"
def test_upstream_error_text_is_not_returned():
text = json.dumps(call("azure_list_servers", {}, Azure(status=403)))
assert "403" in text and "ana@" not in text
def test_check_connection_reports_failure():
assert call("azure_check_connection", {}, Azure(status=401))["ok"] is False
assert call("azure_check_connection", {}, Azure([]))["ok"] is True
@pytest.mark.parametrize("scope", ["", "tenants/x", "managementGroups/a/b"])
def test_bad_scope_root_fails_closed(scope):
result = call("azure_list_servers", {}, Azure([]), {**CONNECTION, "scope_root": scope})
assert result["success"] is False
def test_a_stored_access_token_is_used_without_signing_in():
def azure(request):
assert "login.microsoftonline.com" not in str(request.url)
assert request.headers["Authorization"] == "Bearer personal"
return httpx.Response(200, json={"data": [{"id": "/subscriptions/1/vm"}]})
connection = {"access_token": "personal", "scope_root": "subscriptions/sub-1"}
assert call("azure_list_servers", {}, azure, connection)["data"]["count"] == 1
def test_an_incomplete_connection_is_refused():
assert call("azure_list_servers", {}, Azure([]), {"scope_root": "subscriptions/sub-1"})["success"] is False
def throttled_then(ok, times=1, headers=None):
state = {"left": times}
def azure(request):
if "login.microsoftonline.com" in str(request.url):
return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
if state["left"]:
state["left"] -= 1
return httpx.Response(429, headers=headers or {"retry-after": "0"}, json={})
return ok
return azure, state
def test_a_throttled_call_waits_and_succeeds():
azure, state = throttled_then(httpx.Response(200, json={"data": [{"id": "a"}]}), 2)
assert call("azure_list_servers", {}, azure)["data"]["count"] == 1 and state["left"] == 0
def test_resource_graph_quota_header_is_understood():
azure, _ = throttled_then(httpx.Response(200, json={"data": []}), 1, {"x-ms-user-quota-resets-after": "00:00:00"})
assert call("azure_list_servers", {}, azure)["success"] is True
def test_throttling_that_never_ends_fails_cleanly():
azure, _ = throttled_then(None, 99)
result = call("azure_list_servers", {}, azure)
assert result["success"] is False and "429" in result["error"]["message"]
def test_a_long_requested_wait_gives_up_instead_of_hanging():
azure, state = throttled_then(None, 99, {"retry-after": "3600"})
assert call("azure_list_servers", {}, azure)["success"] is False
assert state["left"] >= 90
LOCK_SUB = {**CONNECTION, "scope_root": "subscriptions/sub-1"}
def arm(value, urls):
def azure(request):
if "login.microsoftonline.com" in str(request.url):
return httpx.Response(200, json={"access_token": "tok", "expires_in": 3600})
urls.append(str(request.url))
if "ResourceGraph" in str(request.url):
return httpx.Response(200, json={"data": value})
return httpx.Response(200, json={"value": [{"name": "do-not-delete", "properties": {"level": "CanNotDelete"}}]})
return azure
def test_locks_come_from_arm_not_resource_graph():
urls = []
data = call("azure_list_resource_policies", {"kind": "resource_locks"}, arm([], urls), LOCK_SUB)["data"]
assert data["count"] == 1 and "/subscriptions/sub-1/providers/Microsoft.Authorization/locks" in urls[0]
def test_lock_subscription_outside_the_scope_is_refused():
result = call("azure_list_resource_policies", {"kind": "resource_locks", "subscription_id": "other"}, arm([], []), LOCK_SUB)
assert result["error"]["code"] == "outside_scope"
def test_management_group_needs_a_subscription_in_scope():
urls = []
assert call("azure_list_resource_policies", {"kind": "resource_locks"}, arm([], urls))["error"]["code"] == "outside_scope"
result = call("azure_list_resource_policies", {"kind": "resource_locks", "subscription_id": "sub-9"}, arm([], urls))
assert result["error"]["code"] == "outside_scope"
found = call("azure_list_resource_policies", {"kind": "resource_locks", "subscription_id": "sub-9"}, arm([{"id": "x"}], urls))
assert found["data"]["count"] == 1
def test_a_forged_lock_page_token_cannot_send_the_token_elsewhere():
urls = []
forged = server._write_page("https://attacker.example/steal", 0)
args = {"kind": "resource_locks", "page_token": forged}
assert call("azure_list_resource_policies", args, arm([], urls), LOCK_SUB)["error"]["code"] == "invalid_page_token"
assert not urls

View File

@@ -0,0 +1,360 @@
"""Conformance checks for an MCP server that takes its upstream credentials from the MCP Gateway.
Run these against a running server before you push it. They need only `pytest` and `httpx`, import
nothing from the server and talk to it over HTTP, so they work on a server from the MCP Server
Builder and on one written by hand.
pip install pytest httpx
MCP_SERVER_URL=http://localhost:8100 MCP_REGISTRATION_API_KEY=<the server's key> pytest tests/ -v
What is checked, and why:
Protocol the server answers /health, refuses callers without its key, and speaks JSON-RPC
correctly (initialize, tools/list, ids echoed, errors as errors).
Connection a server that needs an upstream account says so in its initialize result
(`_meta["com.kyndryl.mcp/connection-schema"]`) and takes the account from each
request (`params._meta["com.kyndryl.mcp/connection"]`), not from its own environment.
Such a server refuses a call that carries none (fail closed), reports which required
field is missing, takes nothing from `arguments`, and never repeats a credential in a
reply (the values of fields marked `writeOnly`; every value if none is marked). A server that needs no account skips these checks.
Settings (environment variables):
MCP_SERVER_URL default http://localhost:8100
MCP_REGISTRATION_API_KEY the key the server was started with (required unless the
server has authentication switched off)
CONFORMANCE_EXPECT_FAIL_CLOSED default true. Set false only for a server you run locally
with REQUIRE_VAULT_CONNECTION=false, so that checks which
need a refusal are skipped rather than failed.
"""
from __future__ import annotations
import os
import secrets
import uuid
from typing import Any, Dict, List, Optional
import httpx
import pytest
VENDOR = "com.kyndryl.mcp"
SCHEMA_KEY = f"{VENDOR}/connection-schema"
VERIFY_TOOL_KEY = f"{VENDOR}/verify-tool"
CONNECTION_KEY = f"{VENDOR}/connection"
BASE_URL = os.getenv("MCP_SERVER_URL", "http://localhost:8100").rstrip("/")
API_KEY = os.getenv("MCP_REGISTRATION_API_KEY", "")
EXPECT_FAIL_CLOSED = os.getenv("CONFORMANCE_EXPECT_FAIL_CLOSED", "true").strip().lower() != "false"
# A value no server could contain by accident, so finding it in a reply means the server repeated it.
SENTINEL = f"sentinel-{secrets.token_hex(6)}"
# ------------------------------------------------------------------------------------ helpers
def auth_headers(key: Optional[str] = None) -> Dict[str, str]:
key = API_KEY if key is None else key
return {"Authorization": f"Bearer {key}"} if key else {}
def rpc(
client: httpx.Client,
method: str,
params: Optional[Dict[str, Any]] = None,
*,
request_id: Any = "1",
headers: Optional[Dict[str, str]] = None,
) -> httpx.Response:
return client.post(
"/messages",
json={"jsonrpc": "2.0", "id": request_id, "method": method, "params": params or {}},
headers=auth_headers() if headers is None else headers,
)
def result_of(client: httpx.Client, method: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
response = rpc(client, method, params)
assert response.status_code == 200, f"{method} answered HTTP {response.status_code}: {response.text[:300]}"
body = response.json()
assert not body.get("error"), f"{method} returned an error: {body['error']}"
return body["result"]
def with_connection(params: Dict[str, Any], connection: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if connection is None:
return params
return {**params, "_meta": {CONNECTION_KEY: connection}}
def value_for(name: str, spec: Dict[str, Any]) -> Any:
"""A value of the right shape for a connection field, carrying the sentinel."""
kind = spec.get("type", "string")
if kind == "string" and spec.get("format") == "uri":
return f"https://{SENTINEL}.invalid"
if kind == "integer":
return 12345
if kind == "boolean":
return True
return f"{SENTINEL}-{name}"
def full_connection(schema: Dict[str, Any]) -> Dict[str, Any]:
return {name: value_for(name, spec) for name, spec in (schema.get("properties") or {}).items()}
def secret_values(schema: Dict[str, Any]) -> List[str]:
"""The sent values that must never come back: those of fields marked secret (`writeOnly`).
A tool may repeat a base URL or a region. If the server marks no field secret, every value is
treated as one, because then nothing says which are safe to repeat.
"""
connection = full_connection(schema)
properties = schema.get("properties") or {}
marked = [n for n, spec in properties.items() if spec.get("writeOnly")]
return [str(connection[n]) for n in (marked or list(properties))]
def repeated(response: httpx.Response, schema: Dict[str, Any]) -> List[str]:
return [v for v in secret_values(schema) if v in response.text]
def _string_values(node: Any) -> List[str]:
"""Every string anywhere in a JSON value."""
if isinstance(node, str):
return [node]
if isinstance(node, dict):
return [v for item in node.values() for v in _string_values(item)]
if isinstance(node, list):
return [v for item in node for v in _string_values(item)]
return []
def call(client: httpx.Client, tool: str, connection: Optional[Dict[str, Any]], arguments: Optional[Dict] = None):
return rpc(client, "tools/call", with_connection({"name": tool, "arguments": arguments or {}}, connection))
# ------------------------------------------------------------------------------------ fixtures
@pytest.fixture(scope="module")
def client():
with httpx.Client(base_url=BASE_URL, timeout=30.0) as session:
try:
session.get("/health")
except httpx.HTTPError as error:
pytest.exit(f"Cannot reach the server at {BASE_URL}: {error}. Start it, or set MCP_SERVER_URL.", 2)
yield session
@pytest.fixture(scope="module")
def initialize(client) -> Dict[str, Any]:
return result_of(client, "initialize", {"protocolVersion": "1.0.0", "clientInfo": {"name": "conformance"}})
@pytest.fixture(scope="module")
def tools(client) -> List[Dict[str, Any]]:
return result_of(client, "tools/list")["tools"]
@pytest.fixture(scope="module")
def schema(initialize) -> Optional[Dict[str, Any]]:
return (initialize.get("_meta") or {}).get(SCHEMA_KEY)
@pytest.fixture(scope="module")
def needs_connection(schema) -> Dict[str, Any]:
if not schema or not schema.get("properties"):
pytest.skip("the server advertises no connection fields, so it takes no per-request credentials")
return schema
@pytest.fixture(scope="module")
def probe_tool(initialize, tools) -> str:
"""The tool to call in the connection checks: the verify tool when there is one."""
verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY)
if verify:
return verify
assert tools, "the server lists no tools to call"
return tools[0]["name"]
# ------------------------------------------------------------------------------------ protocol
def test_health_answers(client):
assert client.get("/health").status_code == 200
def test_the_public_auth_endpoint_reveals_no_key(client):
response = client.get("/.well-known/mcp-auth")
if response.status_code == 404:
pytest.skip("the server has no /.well-known/mcp-auth endpoint")
assert response.status_code == 200
if API_KEY:
# A value that equals the key is always a leak. A short key can also appear inside ordinary words
# ("nc" in "auth_required"), so searching the text for it is only meaningful for a real-length key.
assert API_KEY not in _string_values(response.json()), "/.well-known/mcp-auth returns the server's key"
if len(API_KEY) >= 12:
assert API_KEY not in response.text, "/.well-known/mcp-auth returns the server's key"
def test_a_call_without_the_servers_key_is_refused(client):
response = rpc(client, "tools/list", headers={})
assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with no key"
def test_a_call_with_the_wrong_key_is_refused(client):
response = rpc(client, "tools/list", headers=auth_headers("not-the-key-" + uuid.uuid4().hex))
assert response.status_code in (401, 403), f"answered HTTP {response.status_code} to a caller with the wrong key"
def test_initialize_describes_the_server(initialize):
assert initialize["serverInfo"]["name"]
assert isinstance(initialize.get("capabilities"), dict)
def test_tools_have_a_name_a_description_and_an_object_schema(tools):
assert isinstance(tools, list)
names = [t["name"] for t in tools]
assert len(names) == len(set(names)), "two tools share a name"
for tool in tools:
assert tool["name"] and tool.get("description"), f"tool {tool.get('name')!r} lacks a name or description"
assert tool["inputSchema"]["type"] == "object", f"{tool['name']} has no object inputSchema"
@pytest.mark.parametrize("request_id", ["conformance-1", 7])
def test_the_reply_echoes_the_request_id(client, request_id):
response = rpc(client, "tools/list", request_id=request_id)
assert response.status_code == 200
assert response.json()["id"] == request_id
def test_an_unknown_method_is_an_error_not_a_crash(client):
response = rpc(client, "no/such/method")
assert response.status_code == 200
assert response.json().get("error"), "an unknown method returned no error"
def test_an_unknown_tool_is_an_error_not_a_crash(client):
response = rpc(client, "tools/call", {"name": f"no_such_tool_{uuid.uuid4().hex[:8]}", "arguments": {}})
assert response.status_code == 200
assert response.json().get("error"), "an unknown tool returned no error"
def test_arguments_that_are_not_an_object_are_the_callers_mistake(client, tools, schema):
params = {"name": tools[0]["name"], "arguments": ["not", "an", "object"]}
# With a connection, when the server wants one: a refusal for lacking it would answer -32602 too
response = rpc(client, "tools/call", with_connection(params, full_connection(schema) if schema else None))
assert response.status_code == 200
error = response.json().get("error")
assert error and error.get("code") == -32602, f"expected invalid params (-32602), got: {response.text[:300]}"
def test_an_invalid_request_does_not_repeat_what_it_carried():
# No "id": the request is not valid JSON-RPC, and what it carried (a connection) must not come back.
# It has a connection of its own: a server that answers a bad request by dropping the connection (an
# unhandled error makes uvicorn do that) must not make the NEXT check fail with "connection reset".
marker = f"{SENTINEL}-carried"
try:
response = httpx.post(
f"{BASE_URL}/messages",
json={"jsonrpc": "2.0", "method": "tools/list", "params": {"_meta": {CONNECTION_KEY: {"field": marker}}}},
headers=auth_headers(),
timeout=30.0,
)
except httpx.TransportError:
return # nothing came back, so nothing was repeated
assert marker not in response.text, "the reply to an invalid request repeated the request"
# ------------------------------------------------------------------------------------ connection
def test_the_connection_schema_is_a_json_schema_object(needs_connection):
assert needs_connection["type"] == "object"
properties = needs_connection["properties"]
assert isinstance(properties, dict) and properties
assert set(needs_connection.get("required", [])) <= set(properties), "a required field is not a property"
def test_the_verify_tool_is_listed(initialize, tools, needs_connection):
verify = (initialize.get("_meta") or {}).get(VERIFY_TOOL_KEY)
assert verify, f"initialize does not name a verify tool in _meta[{VERIFY_TOOL_KEY!r}]"
assert verify in [t["name"] for t in tools], f"the verify tool {verify!r} is not in tools/list"
def test_a_call_with_no_connection_is_refused(client, needs_connection, probe_tool):
if not EXPECT_FAIL_CLOSED:
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
response = call(client, probe_tool, None)
assert response.json().get("error"), (
"a call with no connection was served. The server is using its own environment for the "
"upstream account, so every tenant would share one. Refuse it (REQUIRE_VAULT_CONNECTION)."
)
def test_a_connection_in_the_arguments_is_not_accepted(client, needs_connection, probe_tool):
"""The credential travels in `_meta`, which the model never sees; one in `arguments` is not one."""
if not EXPECT_FAIL_CLOSED:
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
response = call(client, probe_tool, None, arguments=full_connection(needs_connection))
assert response.json().get("error"), "a connection passed as tool arguments was accepted"
assert not repeated(response, needs_connection), "the reply repeats a credential that was sent as an argument"
def test_a_call_with_a_connection_is_not_refused_for_lacking_one(client, needs_connection, probe_tool):
response = call(client, probe_tool, full_connection(needs_connection))
assert response.status_code == 200
error = response.json().get("error") or {}
message = str(error.get("message", "")).lower()
assert "no connection" not in message and "missing" not in message, (
f"a complete connection was refused as incomplete: {error}"
)
def test_an_incomplete_connection_names_what_is_missing(client, needs_connection, probe_tool):
required = list(needs_connection.get("required", []))
if not required:
pytest.skip("no field is required")
connection = full_connection(needs_connection)
dropped = required[-1]
connection.pop(dropped)
if not connection:
# The only field is required, and an empty connection counts as none. Send one that has a value
# but not the field, which is what a mistyped field name in the credential looks like.
connection = {"not_a_field_of_this_server": "x"}
response = call(client, probe_tool, connection)
error = response.json().get("error")
assert error, f"a connection without the required field {dropped!r} was served"
assert dropped in str(error.get("message", "")), f"the refusal does not name the missing field {dropped!r}: {error}"
assert not repeated(response, needs_connection), "the refusal repeats a credential value"
def test_no_reply_repeats_the_credential(client, needs_connection, probe_tool, tools):
connection = full_connection(needs_connection)
replies = [call(client, probe_tool, connection)]
for tool in tools[:5]:
replies.append(call(client, tool["name"], connection, arguments={"conformance": "probe"}))
replies.append(call(client, "no_such_tool", connection))
for response in replies:
assert not repeated(response, needs_connection), f"a reply repeats the credential: {response.text[:300]}"
def test_a_malformed_connection_is_no_better_than_none(client, needs_connection, probe_tool):
if not EXPECT_FAIL_CLOSED:
pytest.skip("CONFORMANCE_EXPECT_FAIL_CLOSED=false")
for bad in ("just-a-string", ["a", "list"], 42):
params = {"name": probe_tool, "arguments": {}, "_meta": {CONNECTION_KEY: bad}}
body = rpc(client, "tools/call", params).json()
assert body.get("error"), f"a connection of type {type(bad).__name__} was served"
def test_resources_and_prompts_are_listed_when_advertised(client, initialize):
capabilities = initialize.get("capabilities") or {}
if capabilities.get("resources"):
assert isinstance(result_of(client, "resources/list")["resources"], list)
if capabilities.get("prompts"):
assert isinstance(result_of(client, "prompts/list")["prompts"], list)