Add hyperscaler discovery MCP servers and Well-Architected guidance
Some checks failed
check agents / check (push) Failing after 12s
Some checks failed
check agents / check (push) Failing after 12s
azure-discovery, aws-discovery and gcp-discovery collect read-only metadata on servers, workloads, Kubernetes clusters, networks, data stores, policies and provider recommendations. Credentials arrive per call from the vault. Replies are paged, scrubbed of secrets and anonymised, and throttled calls back off. cloud-knowledge searches each provider's Well-Architected guidance from fixed lists only, so no customer detail can leave the platform. Adds shared skills, the render and test tooling, a public Python Dockerfile until the hardened base image is reachable, the customer access guide, the implementation plan and the Vault Service platform issue. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
19
mcp-servers/_common/Dockerfile.python
Normal file
19
mcp-servers/_common/Dockerfile.python
Normal file
@@ -0,0 +1,19 @@
|
||||
# Builds any server in this folder on the public python image, for when the template's base image isn't reachable:
|
||||
#
|
||||
# podman build -f mcp-servers/_common/Dockerfile.python -t <server>:<version> mcp-servers/<server>
|
||||
#
|
||||
# The rendered Dockerfile builds on the platform's hardened base, and this doesn't: it's a larger image, with pip
|
||||
# and a full Debian userland. Replace it with the rendered Dockerfile once the base image is available.
|
||||
FROM docker.io/library/python:3.13.16
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt constraints.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt -c constraints.txt \
|
||||
&& useradd --uid 1000 --no-create-home --shell /usr/sbin/nologin mcp
|
||||
COPY server.py mcp_connection.py ./
|
||||
|
||||
ENV LOG_LEVEL=INFO
|
||||
ENV LOG_FORMAT=json
|
||||
|
||||
USER 1000
|
||||
CMD ["python", "server.py"]
|
||||
159
mcp-servers/_common/aws_support.py
Normal file
159
mcp-servers/_common/aws_support.py
Normal file
@@ -0,0 +1,159 @@
|
||||
# aws-discovery: every read assumes the customer's read-only role, inside the accounts and regions of the connection.
|
||||
import asyncio
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
|
||||
AWS_SESSIONS = {}
|
||||
# kind -> (service, operation, result key, token parameter, token in the reply, fixed arguments)
|
||||
AWS_SERVERS = {
|
||||
"instances": ("ec2", "describe_instances", "Reservations", "NextToken", "NextToken", {}),
|
||||
"auto_scaling_groups": ("autoscaling", "describe_auto_scaling_groups", "AutoScalingGroups", "NextToken", "NextToken", {}),
|
||||
"volumes": ("ec2", "describe_volumes", "Volumes", "NextToken", "NextToken", {}),
|
||||
}
|
||||
AWS_WORKLOADS = {
|
||||
"lambda_functions": ("lambda", "list_functions", "Functions", "Marker", "NextMarker", {}),
|
||||
"ecs_clusters": ("ecs", "list_clusters", "clusterArns", "nextToken", "nextToken", {}),
|
||||
"app_runner_services": ("apprunner", "list_services", "ServiceSummaryList", "NextToken", "NextToken", {}),
|
||||
"elastic_beanstalk_environments": ("elasticbeanstalk", "describe_environments", "Environments", "NextToken", "NextToken", {}),
|
||||
}
|
||||
AWS_NETWORK = {
|
||||
"vpcs": ("ec2", "describe_vpcs", "Vpcs", "NextToken", "NextToken", {}),
|
||||
"subnets": ("ec2", "describe_subnets", "Subnets", "NextToken", "NextToken", {}),
|
||||
"route_tables": ("ec2", "describe_route_tables", "RouteTables", "NextToken", "NextToken", {}),
|
||||
"security_groups": ("ec2", "describe_security_groups", "SecurityGroups", "NextToken", "NextToken", {}),
|
||||
"nat_gateways": ("ec2", "describe_nat_gateways", "NatGateways", "NextToken", "NextToken", {}),
|
||||
"internet_gateways": ("ec2", "describe_internet_gateways", "InternetGateways", "NextToken", "NextToken", {}),
|
||||
"transit_gateways": ("ec2", "describe_transit_gateways", "TransitGateways", "NextToken", "NextToken", {}),
|
||||
"vpc_peering": ("ec2", "describe_vpc_peering_connections", "VpcPeeringConnections", "NextToken", "NextToken", {}),
|
||||
"vpc_endpoints": ("ec2", "describe_vpc_endpoints", "VpcEndpoints", "NextToken", "NextToken", {}),
|
||||
"load_balancers": ("elbv2", "describe_load_balancers", "LoadBalancers", "Marker", "NextMarker", {}),
|
||||
}
|
||||
AWS_DATA = {
|
||||
"rds_instances": ("rds", "describe_db_instances", "DBInstances", "Marker", "Marker", {}),
|
||||
"dynamodb_tables": ("dynamodb", "list_tables", "TableNames", "ExclusiveStartTableName", "LastEvaluatedTableName", {}),
|
||||
"s3_buckets": ("s3", "list_buckets", "Buckets", "ContinuationToken", "ContinuationToken", {}),
|
||||
}
|
||||
AWS_ORG_POLICY_TYPES = {"service_control_policies": "SERVICE_CONTROL_POLICY", "resource_control_policies": "RESOURCE_CONTROL_POLICY",
|
||||
"tag_policies": "TAG_POLICY", "backup_policies": "BACKUP_POLICY"}
|
||||
AWS_PILLARS = {"cost_optimizing": "cost", "security": "security", "fault_tolerance": "reliability",
|
||||
"performance": "performance", "operational_excellence": "operations", "service_limits": "reliability"}
|
||||
|
||||
|
||||
def _aws_csv(connection: Dict[str, Any], field: str) -> List[str]:
|
||||
return [v.strip() for v in str(connection.get(field) or "").split(",") if v.strip()]
|
||||
|
||||
|
||||
def _aws_session(connection: Dict[str, Any], account_id: Optional[str]):
|
||||
"""A session on the customer's read-only role, or on the member account's when one is named."""
|
||||
key = hashlib.sha256(json.dumps([connection, account_id], sort_keys=True).encode()).hexdigest()
|
||||
cached = AWS_SESSIONS.get(key)
|
||||
now = time.time()
|
||||
if cached and cached[1] > now + 60:
|
||||
return cached[0]
|
||||
|
||||
def assume(session, role_arn, external_id=None):
|
||||
region = _aws_csv(connection, "regions")[0]
|
||||
request = {"RoleArn": role_arn, "RoleSessionName": "kab-discovery", "DurationSeconds": 3600}
|
||||
if external_id:
|
||||
request["ExternalId"] = external_id
|
||||
found = session.client("sts", region_name=region).assume_role(**request)["Credentials"]
|
||||
return boto3.Session(aws_access_key_id=found["AccessKeyId"], aws_secret_access_key=found["SecretAccessKey"],
|
||||
aws_session_token=found["SessionToken"])
|
||||
|
||||
base = boto3.Session(aws_access_key_id=connection["access_key_id"], aws_secret_access_key=connection["secret_access_key"])
|
||||
session = assume(base, connection["role_arn"], connection.get("external_id"))
|
||||
if account_id:
|
||||
partition = str(connection["role_arn"]).split(":")[1]
|
||||
session = assume(session, f"arn:{partition}:iam::{account_id}:role/{connection['member_role_name']}")
|
||||
AWS_SESSIONS[key] = (session, now + 3000)
|
||||
return session
|
||||
|
||||
|
||||
def _aws_target(arguments: Dict[str, Any]):
|
||||
"""(account, region) or a refusal. Nothing outside the connection's accounts and regions is ever reached."""
|
||||
connection = get_connection()
|
||||
regions = _aws_csv(connection, "regions")
|
||||
region = arguments.get("region") or regions[0]
|
||||
if region not in regions:
|
||||
return None, None, _refuse("outside_scope", "region isn't in this connection's regions")
|
||||
account = arguments.get("account_id")
|
||||
if account and (account not in _aws_csv(connection, "account_ids") or not connection.get("member_role_name")):
|
||||
return None, None, _refuse("outside_scope", "account_id isn't in this connection's accounts")
|
||||
return account, region, None
|
||||
|
||||
|
||||
async def _aws_call(arguments: Dict[str, Any], service: str, operation: str, region: Optional[str] = None, **request):
|
||||
connection = get_connection()
|
||||
account = arguments.get("account_id")
|
||||
where = region or arguments.get("region") or _aws_csv(connection, "regions")[0]
|
||||
|
||||
def run():
|
||||
# Adaptive mode slows the client down when AWS throttles it, on top of retrying.
|
||||
retries = BotoConfig(retries={"max_attempts": 6, "mode": "adaptive"}, read_timeout=60, connect_timeout=10)
|
||||
client = _aws_session(connection, account).client(service, region_name=where, config=retries)
|
||||
return getattr(client, operation)(**request)
|
||||
|
||||
return await asyncio.to_thread(run)
|
||||
|
||||
|
||||
async def _aws_kind(tool: str, arguments: Dict[str, Any], kinds: Dict[str, tuple], default: str) -> Dict[str, Any]:
|
||||
kind = arguments.get("kind") or default
|
||||
if kind not in kinds:
|
||||
return _refuse("unknown_kind", "kind must be one of: " + ", ".join(kinds))
|
||||
account, region, refused = _aws_target(arguments)
|
||||
if refused:
|
||||
return refused
|
||||
service, operation, result_key, token_in, token_out, fixed = kinds[kind]
|
||||
try:
|
||||
upstream, skip = _read_page(arguments)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
request = dict(fixed)
|
||||
if upstream:
|
||||
request[token_in] = upstream
|
||||
try:
|
||||
body = await _aws_call(arguments, service, operation, region, **request)
|
||||
except Exception as error:
|
||||
return _upstream_failed("aws", error)
|
||||
extra = {"account_id": account, "region": region, "kind": kind}
|
||||
return _page_result("aws", tool, arguments, body.get(result_key, []), upstream, body.get(token_out), skip, extra)
|
||||
|
||||
|
||||
async def _aws_each(tool: str, arguments: Dict[str, Any], list_kind: tuple, read) -> Dict[str, Any]:
|
||||
"""List one page of things, then read a detail for each. A detail that's absent or refused is reported."""
|
||||
account, region, refused = _aws_target(arguments)
|
||||
if refused:
|
||||
return refused
|
||||
service, operation, result_key, token_in, token_out, fixed = list_kind
|
||||
try:
|
||||
upstream, skip = _read_page(arguments)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
request = dict(fixed)
|
||||
if upstream:
|
||||
request[token_in] = upstream
|
||||
try:
|
||||
body = await _aws_call(arguments, service, operation, region, **request)
|
||||
except Exception as error:
|
||||
return _upstream_failed("aws", error)
|
||||
items = []
|
||||
for entry in body.get(result_key, [])[skip:skip + _page_size(arguments)]:
|
||||
try:
|
||||
items.append(await read(entry, region))
|
||||
except Exception as error:
|
||||
items.append({"resource": entry, "unread": type(error).__name__})
|
||||
consumed = skip + len(items)
|
||||
more = consumed < len(body.get(result_key, []))
|
||||
next_upstream = upstream if more else body.get(token_out)
|
||||
result = _page_result("aws", tool, arguments, items, None, None, 0, {"account_id": account, "region": region})
|
||||
if more or next_upstream:
|
||||
result["data"]["next_page_token"] = _write_page(next_upstream, consumed if more else 0)
|
||||
return result
|
||||
|
||||
|
||||
def _aws_document(text: Any) -> Any:
|
||||
"""A policy document, parsed when it arrives as a JSON string, so its principals can be scrubbed."""
|
||||
try:
|
||||
return json.loads(text) if isinstance(text, str) else text
|
||||
except ValueError:
|
||||
return text
|
||||
171
mcp-servers/_common/azure_support.py
Normal file
171
mcp-servers/_common/azure_support.py
Normal file
@@ -0,0 +1,171 @@
|
||||
# azure-discovery: every read goes through Azure Resource Graph, so the scope sent with each query is the boundary.
|
||||
AZURE_CLOUDS = {
|
||||
"public": {"login": "https://login.microsoftonline.com", "arm": "https://management.azure.com"},
|
||||
"usgov": {"login": "https://login.microsoftonline.us", "arm": "https://management.usgovcloudapi.net"},
|
||||
"china": {"login": "https://login.chinacloudapi.cn", "arm": "https://management.chinacloudapi.cn"},
|
||||
}
|
||||
AZURE_GRAPH_VERSION = "2022-10-01"
|
||||
AZURE_LOCKS_VERSION = "2020-05-01"
|
||||
AZURE_TABLES = ("resources", "resourcecontainers", "policyresources", "advisorresources", "networkresources")
|
||||
AZURE_TOKENS = {}
|
||||
AZURE_SERVERS = {
|
||||
"virtual_machines": ["microsoft.compute/virtualmachines"],
|
||||
"scale_sets": ["microsoft.compute/virtualmachinescalesets"],
|
||||
"disks": ["microsoft.compute/disks"],
|
||||
}
|
||||
AZURE_WORKLOADS = {
|
||||
"app_service": ["microsoft.web/sites", "microsoft.web/serverfarms"],
|
||||
"container_apps": ["microsoft.app/containerapps", "microsoft.app/managedenvironments"],
|
||||
"container_instances": ["microsoft.containerinstance/containergroups"],
|
||||
}
|
||||
AZURE_KUBERNETES = {"clusters": ["microsoft.containerservice/managedclusters"]}
|
||||
AZURE_NETWORK = {
|
||||
"virtual_networks": ["microsoft.network/virtualnetworks"],
|
||||
"route_tables": ["microsoft.network/routetables"],
|
||||
"network_security_groups": ["microsoft.network/networksecuritygroups"],
|
||||
"gateways": ["microsoft.network/virtualnetworkgateways", "microsoft.network/expressroutecircuits",
|
||||
"microsoft.network/natgateways", "microsoft.network/azurefirewalls",
|
||||
"microsoft.network/applicationgateways"],
|
||||
"load_balancers": ["microsoft.network/loadbalancers"],
|
||||
"private_endpoints": ["microsoft.network/privateendpoints"],
|
||||
"public_ips": ["microsoft.network/publicipaddresses"],
|
||||
}
|
||||
AZURE_DATA = {
|
||||
"sql": ["microsoft.sql/servers", "microsoft.sql/servers/databases"],
|
||||
"cosmos_db": ["microsoft.documentdb/databaseaccounts"],
|
||||
"storage_accounts": ["microsoft.storage/storageaccounts"],
|
||||
}
|
||||
AZURE_ORG_POLICIES = {
|
||||
"assignments": ["microsoft.authorization/policyassignments"],
|
||||
"definitions": ["microsoft.authorization/policydefinitions"],
|
||||
"initiatives": ["microsoft.authorization/policysetdefinitions"],
|
||||
"exemptions": ["microsoft.authorization/policyexemptions"],
|
||||
"compliance": ["microsoft.policyinsights/policystates"],
|
||||
}
|
||||
AZURE_RESOURCE_POLICIES = {
|
||||
"key_vault_access": ["microsoft.keyvault/vaults"],
|
||||
"storage_network_rules": ["microsoft.storage/storageaccounts"],
|
||||
}
|
||||
AZURE_PILLARS = {"cost": "cost", "security": "security", "highavailability": "reliability",
|
||||
"performance": "performance", "operationalexcellence": "operations"}
|
||||
|
||||
|
||||
def _azure_scope(connection: Dict[str, Any]) -> Dict[str, List[str]]:
|
||||
"""The query scope: the connection's root, and nothing the caller can widen."""
|
||||
kind, _, identifier = str(connection.get("scope_root", "")).strip("/").partition("/")
|
||||
if not identifier or "/" in identifier or kind not in ("managementGroups", "subscriptions"):
|
||||
raise ValueError("scope_root must be managementGroups/<id> or subscriptions/<id>")
|
||||
return {kind: [identifier]}
|
||||
|
||||
|
||||
def _azure_endpoints(connection: Dict[str, Any]) -> Dict[str, str]:
|
||||
return AZURE_CLOUDS.get(str(connection.get("cloud") or "public"), AZURE_CLOUDS["public"])
|
||||
|
||||
|
||||
def _azure_client():
|
||||
return get_upstream_client("azure", lambda c: httpx.AsyncClient(timeout=100))
|
||||
|
||||
|
||||
async def _azure_token(connection: Dict[str, Any]) -> str:
|
||||
"""A token for Resource Graph: the short-lived one stored for a trial, or one signed in with the app registration."""
|
||||
if connection.get("access_token"):
|
||||
return str(connection["access_token"])
|
||||
if not all(connection.get(f) for f in ("tenant_id", "client_id", "client_secret")):
|
||||
raise ValueError("the connection needs access_token, or tenant_id, client_id and client_secret")
|
||||
key = hashlib.sha256(json.dumps(connection, sort_keys=True).encode()).hexdigest()
|
||||
cached = AZURE_TOKENS.get(key)
|
||||
now = time.time()
|
||||
if cached and cached[1] > now + 60:
|
||||
return cached[0]
|
||||
endpoints = _azure_endpoints(connection)
|
||||
response = await _azure_client().post(
|
||||
f"{endpoints['login']}/{connection['tenant_id']}/oauth2/v2.0/token",
|
||||
data={"grant_type": "client_credentials", "client_id": connection["client_id"],
|
||||
"client_secret": connection["client_secret"], "scope": f"{endpoints['arm']}/.default"})
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
AZURE_TOKENS[key] = (body["access_token"], now + int(body.get("expires_in", 3600)))
|
||||
return body["access_token"]
|
||||
|
||||
|
||||
async def _azure_graph(query: str, top: int, skip_token: Optional[str] = None) -> Dict[str, Any]:
|
||||
connection = get_connection()
|
||||
options = {"$top": top, "resultFormat": "objectArray"}
|
||||
if skip_token:
|
||||
options["$skipToken"] = skip_token
|
||||
token = await _azure_token(connection)
|
||||
response = await _with_backoff(lambda: _azure_client().post(
|
||||
f"{_azure_endpoints(connection)['arm']}/providers/Microsoft.ResourceGraph/resources",
|
||||
params={"api-version": AZURE_GRAPH_VERSION}, headers={"Authorization": f"Bearer {token}"},
|
||||
json={"query": query, "options": options, **_azure_scope(connection)}))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
async def _azure_page(tool: str, arguments: Dict[str, Any], query: str, finish=None) -> Dict[str, Any]:
|
||||
try:
|
||||
upstream, skip = _read_page(arguments)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
try:
|
||||
body = await _azure_graph(query, _page_size(arguments), upstream)
|
||||
except Exception as error:
|
||||
return _upstream_failed("azure", error)
|
||||
items = [finish(i) for i in body.get("data", [])] if finish else body.get("data", [])
|
||||
return _page_result("azure", tool, arguments, items, upstream, body.get("$skipToken"), skip)
|
||||
|
||||
|
||||
def _azure_type_query(table: str, types: List[str]) -> str:
|
||||
listed = ", ".join(f"'{t}'" for t in types)
|
||||
return f"{table} | where type in~ ({listed}) | order by id asc"
|
||||
|
||||
|
||||
async def _azure_kind(tool: str, arguments: Dict[str, Any], kinds: Dict[str, List[str]], table: str = "Resources",
|
||||
default: Optional[str] = None, finish=None) -> Dict[str, Any]:
|
||||
kind = arguments.get("kind") or default or next(iter(kinds))
|
||||
if kind not in kinds:
|
||||
return _refuse("unknown_kind", "kind must be one of: " + ", ".join(kinds))
|
||||
return await _azure_page(tool, arguments, _azure_type_query(table, kinds[kind]), finish)
|
||||
|
||||
|
||||
def _azure_recommendation(item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
category = str(item.get("properties", {}).get("category", ""))
|
||||
return {**item, "pillar": AZURE_PILLARS.get(category.lower(), "other"), "provider_category": category}
|
||||
|
||||
|
||||
async def _azure_lock_subscription(arguments: Dict[str, Any]):
|
||||
"""The subscription to read locks from, or None. Resource Graph, under the connection's scope, sets the boundary."""
|
||||
kind, identifiers = next(iter(_azure_scope(get_connection()).items()))
|
||||
wanted = arguments.get("subscription_id")
|
||||
if kind == "subscriptions":
|
||||
return identifiers[0] if wanted in (None, identifiers[0]) else None
|
||||
if not wanted or not re.fullmatch(r"[A-Za-z0-9\-]+", str(wanted)):
|
||||
return None
|
||||
query = f"ResourceContainers | where type =~ 'microsoft.resources/subscriptions' and subscriptionId =~ '{wanted}'"
|
||||
return wanted if (await _azure_graph(query, 1)).get("data") else None
|
||||
|
||||
|
||||
async def _azure_locks(tool: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Resource locks, from ARM: Resource Graph doesn't hold them, and answers 0 whether or not any exist."""
|
||||
try:
|
||||
upstream, skip = _read_page(arguments)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
connection = get_connection()
|
||||
try:
|
||||
subscription = await _azure_lock_subscription(arguments)
|
||||
if not subscription:
|
||||
return _refuse("outside_scope", "subscription_id must name a subscription under this connection's scope")
|
||||
base = f"{_azure_endpoints(connection)['arm']}/subscriptions/{subscription}/providers/Microsoft.Authorization/locks"
|
||||
# A page token carries ARM's next link, and the bearer token goes wherever it points, so it must stay here.
|
||||
url = upstream or f"{base}?api-version={AZURE_LOCKS_VERSION}"
|
||||
if not url.startswith(f"{base}?"):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
token = await _azure_token(connection)
|
||||
response = await _with_backoff(lambda: _azure_client().get(url, headers={"Authorization": f"Bearer {token}"}))
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
except Exception as error:
|
||||
return _upstream_failed("azure", error)
|
||||
return _page_result("azure", tool, arguments, body.get("value", []), upstream, body.get("nextLink"), skip,
|
||||
{"subscription_id": subscription, "kind": "resource_locks"})
|
||||
133
mcp-servers/_common/gcp_support.py
Normal file
133
mcp-servers/_common/gcp_support.py
Normal file
@@ -0,0 +1,133 @@
|
||||
# gcp-discovery: every read goes through Cloud Asset Inventory or another read API, under the connection's scope_root.
|
||||
import asyncio
|
||||
from google.auth.transport.requests import Request as GoogleRequest
|
||||
from google.oauth2 import service_account
|
||||
|
||||
GCP_ASSET_API = "https://cloudasset.googleapis.com/v1"
|
||||
GCP_ORG_POLICY_API = "https://orgpolicy.googleapis.com/v2"
|
||||
GCP_RECOMMENDER_API = "https://recommender.googleapis.com/v1"
|
||||
GCP_READ_ONLY = "https://www.googleapis.com/auth/cloud-platform.read-only"
|
||||
GCP_TOKENS = {}
|
||||
GCP_SERVERS = {
|
||||
"instances": ["compute.googleapis.com/Instance"],
|
||||
"instance_groups": ["compute.googleapis.com/InstanceGroupManager"],
|
||||
"disks": ["compute.googleapis.com/Disk"],
|
||||
}
|
||||
GCP_WORKLOADS = {
|
||||
"cloud_run": ["run.googleapis.com/Service"],
|
||||
"cloud_functions": ["cloudfunctions.googleapis.com/Function"],
|
||||
"app_engine": ["appengine.googleapis.com/Service"],
|
||||
}
|
||||
GCP_KUBERNETES = {"clusters": ["container.googleapis.com/Cluster"], "node_pools": ["container.googleapis.com/NodePool"]}
|
||||
GCP_NETWORK = {
|
||||
"networks": ["compute.googleapis.com/Network"],
|
||||
"subnetworks": ["compute.googleapis.com/Subnetwork"],
|
||||
"routes": ["compute.googleapis.com/Route"],
|
||||
"firewall_rules": ["compute.googleapis.com/Firewall", "compute.googleapis.com/FirewallPolicy"],
|
||||
"load_balancers": ["compute.googleapis.com/ForwardingRule", "compute.googleapis.com/BackendService"],
|
||||
"gateways": ["compute.googleapis.com/Router", "compute.googleapis.com/VpnGateway",
|
||||
"compute.googleapis.com/InterconnectAttachment"],
|
||||
}
|
||||
GCP_DATA = {
|
||||
"cloud_sql": ["sqladmin.googleapis.com/Instance"],
|
||||
"firestore": ["firestore.googleapis.com/Database"],
|
||||
"cloud_storage": ["storage.googleapis.com/Bucket"],
|
||||
}
|
||||
GCP_ACCOUNTS = {
|
||||
"projects": ["cloudresourcemanager.googleapis.com/Project"],
|
||||
"folders": ["cloudresourcemanager.googleapis.com/Folder"],
|
||||
}
|
||||
GCP_PERIMETERS = {"vpc_service_controls": ["accesscontextmanager.googleapis.com/ServicePerimeter"]}
|
||||
GCP_RECOMMENDERS = ("google.compute.instance.MachineTypeRecommender", "google.compute.instance.IdleResourceRecommender",
|
||||
"google.compute.disk.IdleResourceRecommender", "google.iam.policy.Recommender")
|
||||
GCP_PILLARS = {"cost": "cost", "security": "security", "performance": "performance", "reliability": "reliability",
|
||||
"manageability": "operations", "sustainability": "sustainability"}
|
||||
|
||||
|
||||
def _gcp_scope() -> str:
|
||||
scope = str(get_connection().get("scope_root", "")).strip("/")
|
||||
kind, _, identifier = scope.partition("/")
|
||||
if kind not in ("organizations", "folders", "projects") or not identifier or "/" in identifier:
|
||||
raise ValueError("scope_root must be organizations/<id>, folders/<id> or projects/<id>")
|
||||
return scope
|
||||
|
||||
|
||||
def _gcp_client():
|
||||
return get_upstream_client("gcp", lambda c: httpx.AsyncClient(timeout=100))
|
||||
|
||||
|
||||
async def _gcp_token() -> str:
|
||||
connection = get_connection()
|
||||
key = hashlib.sha256(json.dumps(connection, sort_keys=True).encode()).hexdigest()
|
||||
cached = GCP_TOKENS.get(key)
|
||||
now = time.time()
|
||||
if cached and cached[1] > now + 60:
|
||||
return cached[0]
|
||||
|
||||
def refresh():
|
||||
info = json.loads(connection["service_account_json"])
|
||||
credentials = service_account.Credentials.from_service_account_info(info, scopes=[GCP_READ_ONLY])
|
||||
credentials.refresh(GoogleRequest())
|
||||
return credentials.token
|
||||
|
||||
token = await asyncio.to_thread(refresh)
|
||||
GCP_TOKENS[key] = (token, now + 3000)
|
||||
return token
|
||||
|
||||
|
||||
async def _gcp_get(url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
token = await _gcp_token()
|
||||
response = await _with_backoff(
|
||||
lambda: _gcp_client().get(url, params=params, headers={"Authorization": f"Bearer {token}"}))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
async def _gcp_page(tool: str, arguments: Dict[str, Any], url: str, params: Dict[str, Any], result_key: str,
|
||||
finish=None) -> Dict[str, Any]:
|
||||
try:
|
||||
upstream, skip = _read_page(arguments)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
||||
request = {**params, "pageSize": _page_size(arguments)}
|
||||
if upstream:
|
||||
request["pageToken"] = upstream
|
||||
try:
|
||||
body = await _gcp_get(url, request)
|
||||
except Exception as error:
|
||||
return _upstream_failed("gcp", error)
|
||||
items = [finish(i) for i in body.get(result_key, [])] if finish else body.get(result_key, [])
|
||||
return _page_result("gcp", tool, arguments, items, upstream, body.get("nextPageToken"), skip)
|
||||
|
||||
|
||||
async def _gcp_search(tool: str, arguments: Dict[str, Any], asset_types: List[str], query: str = "") -> Dict[str, Any]:
|
||||
try:
|
||||
url = f"{GCP_ASSET_API}/{_gcp_scope()}:searchAllResources"
|
||||
except ValueError as error:
|
||||
return _refuse("invalid_connection", str(error))
|
||||
params = {"assetTypes": asset_types, "readMask": "*"}
|
||||
if query:
|
||||
params["query"] = query
|
||||
return await _gcp_page(tool, arguments, url, params, "results")
|
||||
|
||||
|
||||
async def _gcp_kind(tool: str, arguments: Dict[str, Any], kinds: Dict[str, List[str]], default: str) -> Dict[str, Any]:
|
||||
kind = arguments.get("kind") or default
|
||||
if kind not in kinds:
|
||||
return _refuse("unknown_kind", "kind must be one of: " + ", ".join(kinds))
|
||||
return await _gcp_search(tool, arguments, kinds[kind])
|
||||
|
||||
|
||||
async def _gcp_project_in_scope(project_id: str) -> bool:
|
||||
"""Whether Cloud Asset, queried under the scope, can see this project. The API sets the boundary."""
|
||||
scope = _gcp_scope()
|
||||
if scope == f"projects/{project_id}":
|
||||
return True
|
||||
body = await _gcp_get(f"{GCP_ASSET_API}/{scope}:searchAllResources",
|
||||
{"assetTypes": GCP_ACCOUNTS["projects"], "query": f"name:{project_id}", "pageSize": 5})
|
||||
return any(str(r.get("name", "")).endswith(f"/projects/{project_id}") for r in body.get("results", []))
|
||||
|
||||
|
||||
def _gcp_recommendation(item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
category = str(item.get("primaryImpact", {}).get("category", ""))
|
||||
return {**item, "pillar": GCP_PILLARS.get(category.lower(), "other"), "provider_category": category}
|
||||
108
mcp-servers/_common/knowledge_support.py
Normal file
108
mcp-servers/_common/knowledge_support.py
Normal file
@@ -0,0 +1,108 @@
|
||||
# cloud-knowledge: calls each hyperscaler's hosted knowledge server over MCP. A query is only ever built from the
|
||||
# fixed lists below, so a customer's name or resource can't reach a third party.
|
||||
import asyncio
|
||||
import boto3
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
|
||||
KNOWLEDGE_SERVERS = {
|
||||
"azure": {"url": "https://learn.microsoft.com/api/mcp", "tool": "microsoft_docs_search",
|
||||
"framework": "Azure Well-Architected Framework"},
|
||||
"aws": {"url": "https://aws-mcp.eu-west-1.api.aws/mcp", "tool": "aws___search_documentation",
|
||||
"framework": "AWS Well-Architected Framework"},
|
||||
"gcp": {"url": "https://developerknowledge.googleapis.com/mcp", "tool": "search_documents",
|
||||
"framework": "Google Cloud Well-Architected Framework"},
|
||||
}
|
||||
KNOWLEDGE_PILLARS = ("reliability", "security", "cost", "operations", "performance", "sustainability")
|
||||
KNOWLEDGE_TOPICS = (
|
||||
"landing zone", "resource organization", "identity and access", "network segmentation", "hybrid connectivity",
|
||||
"encryption", "logging and monitoring", "backup and recovery", "disaster recovery", "high availability",
|
||||
"scaling", "kubernetes", "containers", "virtual machines", "databases", "storage", "serverless", "governance",
|
||||
"policy enforcement", "tagging", "cost allocation", "migration", "incident response", "patching",
|
||||
)
|
||||
KNOWLEDGE_MAX_TEXT = 4000
|
||||
|
||||
|
||||
def _knowledge_client():
|
||||
return get_upstream_client("knowledge", lambda c: httpx.AsyncClient(timeout=60, follow_redirects=False))
|
||||
|
||||
|
||||
def _knowledge_message(response) -> Dict[str, Any]:
|
||||
"""The JSON-RPC reply, whether it came back as JSON or as a server-sent event."""
|
||||
if "text/event-stream" in response.headers.get("content-type", ""):
|
||||
lines = [line[5:].strip() for line in response.text.splitlines() if line.startswith("data:")]
|
||||
return json.loads(lines[-1]) if lines else {}
|
||||
return response.json()
|
||||
|
||||
|
||||
async def _knowledge_post(provider: str, connection: Dict[str, Any], body: Dict[str, Any],
|
||||
session: Optional[str] = None):
|
||||
url = KNOWLEDGE_SERVERS[provider]["url"]
|
||||
headers = {"Accept": "application/json, text/event-stream", "Content-Type": "application/json"}
|
||||
if session:
|
||||
headers["Mcp-Session-Id"] = session
|
||||
if provider == "gcp":
|
||||
headers["X-Goog-Api-Key"] = connection.get("google_api_key", "")
|
||||
content = json.dumps(body)
|
||||
if provider == "aws":
|
||||
def sign():
|
||||
credentials = boto3.Session(aws_access_key_id=connection.get("aws_access_key_id"),
|
||||
aws_secret_access_key=connection.get("aws_secret_access_key")).get_credentials()
|
||||
request = AWSRequest(method="POST", url=url, data=content, headers=headers)
|
||||
SigV4Auth(credentials, "aws-mcp", "eu-west-1").add_auth(request)
|
||||
return dict(request.headers)
|
||||
|
||||
headers = await asyncio.to_thread(sign)
|
||||
response = await _with_backoff(lambda: _knowledge_client().post(url, content=content, headers=headers))
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
|
||||
async def _knowledge_session(provider: str, connection: Dict[str, Any]) -> Optional[str]:
|
||||
response = await _knowledge_post(provider, connection, {
|
||||
"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {"protocolVersion": "2025-06-18", "capabilities": {},
|
||||
"clientInfo": {"name": "cloud-knowledge", "version": "0.1.0"}}})
|
||||
return response.headers.get("mcp-session-id")
|
||||
|
||||
|
||||
async def _knowledge_tools(provider: str, connection: Dict[str, Any], session: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
session = session or await _knowledge_session(provider, connection)
|
||||
response = await _knowledge_post(provider, connection, {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, session)
|
||||
return _knowledge_message(response).get("result", {}).get("tools", [])
|
||||
|
||||
|
||||
def _knowledge_argument(tool: Dict[str, Any]) -> str:
|
||||
"""The search tool's query argument. Vendors name it differently and may change it, so it's read from the tool."""
|
||||
schema = tool.get("inputSchema", {})
|
||||
properties = schema.get("properties", {})
|
||||
for name in list(schema.get("required", [])) + list(properties):
|
||||
if properties.get(name, {}).get("type") == "string":
|
||||
return name
|
||||
return "query"
|
||||
|
||||
|
||||
async def _knowledge_search(provider: str, connection: Dict[str, Any], query: str) -> List[Dict[str, Any]]:
|
||||
session = await _knowledge_session(provider, connection)
|
||||
name = KNOWLEDGE_SERVERS[provider]["tool"]
|
||||
tools = {t.get("name"): t for t in await _knowledge_tools(provider, connection, session)}
|
||||
if name not in tools:
|
||||
raise LookupError(f"{provider} no longer offers {name}")
|
||||
response = await _knowledge_post(provider, connection, {
|
||||
"jsonrpc": "2.0", "id": 3, "method": "tools/call",
|
||||
"params": {"name": name, "arguments": {_knowledge_argument(tools[name]): query}}}, session)
|
||||
message = _knowledge_message(response)
|
||||
results = []
|
||||
for part in message.get("result", {}).get("content", []):
|
||||
if part.get("type") != "text":
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(part["text"])
|
||||
except ValueError:
|
||||
parsed = {"content": part["text"]}
|
||||
found = parsed.get("results", parsed) if isinstance(parsed, dict) else parsed
|
||||
for item in found if isinstance(found, list) else [found]:
|
||||
if isinstance(item, dict):
|
||||
results.append({"title": item.get("title"), "url": item.get("contentUrl") or item.get("url"),
|
||||
"text": str(item.get("content") or item.get("text") or "")[:KNOWLEDGE_MAX_TEXT]})
|
||||
return results
|
||||
125
mcp-servers/_common/test_contract.py
Normal file
125
mcp-servers/_common/test_contract.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""The contract every hyperscaler server must meet. Run once per server: SERVER_DIR=<folder> pytest _common/test_contract.py"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SERVER_DIR = Path(os.environ["SERVER_DIR"]).resolve()
|
||||
sys.path.insert(0, str(SERVER_DIR))
|
||||
import server # noqa: E402
|
||||
|
||||
CONNECTIONS = {
|
||||
"azure-discovery": {"tenant_id": "t", "client_id": "c", "client_secret": "s", "scope_root": "subscriptions/sub-1"},
|
||||
"aws-discovery": {"access_key_id": "a", "secret_access_key": "b", "role_arn": "arn:aws:iam::111111111111:role/r",
|
||||
"external_id": "e", "regions": "eu-west-2"},
|
||||
"gcp-discovery": {"scope_root": "projects/p1", "service_account_json": "{}"},
|
||||
"cloud-knowledge": {"google_api_key": "k"},
|
||||
}
|
||||
|
||||
|
||||
def test_every_tool_has_a_handler():
|
||||
names = [t["name"] for t in server.TOOLS_REGISTRY]
|
||||
assert "verify_connection" in names
|
||||
assert len(names) == len(set(names))
|
||||
for name in names:
|
||||
assert callable(getattr(server, name))
|
||||
|
||||
|
||||
def test_verify_connection_is_implemented():
|
||||
assert server.VERIFY_TOOL == "verify_connection"
|
||||
assert "does not check it against" not in Path(SERVER_DIR / "server.py").read_text()
|
||||
|
||||
|
||||
def test_connection_is_required_by_default():
|
||||
assert server.config.REQUIRE_VAULT_CONNECTION is True
|
||||
with pytest.raises(ValueError):
|
||||
server.check_connection(None)
|
||||
|
||||
|
||||
def test_no_credential_is_read_from_the_environment():
|
||||
code = Path(SERVER_DIR / "server.py").read_text()
|
||||
for field in CONNECTIONS[SERVER_DIR.name]:
|
||||
assert not re.search(rf"(getenv|environ)[^\n]*{field}", code)
|
||||
|
||||
|
||||
def test_only_read_operations_are_named():
|
||||
"""No create, update, delete or put anywhere in the upstream calls."""
|
||||
if SERVER_DIR.name != "aws-discovery":
|
||||
pytest.skip("only AWS names operations; Azure and Google Cloud are tested on the URLs they reach")
|
||||
code = Path(SERVER_DIR / "server.py").read_text()
|
||||
operations = re.findall(r'_aws_call\([^,]+,\s*"[a-z0-9]+",\s*"([a-z_]+)"', code)
|
||||
operations += re.findall(r'\("[a-z0-9]+",\s*"([a-z_]+)",\s*"[A-Za-z]+",\s*"[A-Za-z]+",\s*"[A-Za-z]+",', code)
|
||||
assert all(op.split("_")[0] in ("list", "describe", "get", "select") for op in operations), operations
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value,expected", [
|
||||
("user:ana@customer.example", "[user]"),
|
||||
("group:admins@customer.example", "[group]"),
|
||||
("domain:customer.example", "[domain]"),
|
||||
("deleted:user:ana@customer.example?uid=1", "[user]"),
|
||||
("serviceAccount:scan@p1.iam.gserviceaccount.com", "serviceAccount:scan@p1.iam.gserviceaccount.com"),
|
||||
])
|
||||
def test_members_are_anonymised(value, expected):
|
||||
assert server._scrub({"members": [value]})["members"] == [expected]
|
||||
|
||||
|
||||
def test_people_never_survive_scrubbing():
|
||||
record = {
|
||||
"tags": {"owner": "Ana Silva", "environment": "prod"},
|
||||
"Tags": [{"Key": "Owner", "Value": "Ana Silva"}, {"Key": "Name", "Value": "web-01"}],
|
||||
"systemData": {"createdBy": "ana@customer.example"},
|
||||
"description": "Contact ana@customer.example for access",
|
||||
"Principal": {"AWS": "arn:aws:iam::111111111111:user/ana"},
|
||||
"session": "arn:aws:sts::111111111111:assumed-role/ops/ana@customer.example",
|
||||
"objectId": "00000000-0000-0000-0000-000000000000",
|
||||
}
|
||||
text = json.dumps(server._scrub(record))
|
||||
assert "ana" not in text.lower().replace("[", "") or "Ana" not in text
|
||||
assert "ana@" not in text and "Ana Silva" not in text and ":user/ana" not in text
|
||||
assert "00000000-0000" not in text
|
||||
assert "web-01" in text and "prod" in text and "assumed-role/ops/[session]" in text
|
||||
|
||||
|
||||
def test_secrets_never_survive_scrubbing():
|
||||
record = {"osProfile": {"customData": "c2VjcmV0", "adminPassword": "x"}, "Environment": {"Variables": {"DB": "y"}},
|
||||
"properties": {"connectionStrings": ["z"], "envVars": [{"name": "A"}]}}
|
||||
text = json.dumps(server._scrub(record))
|
||||
assert "c2VjcmV0" not in text and '"DB"' not in text and '"z"' not in text and '"A"' not in text
|
||||
|
||||
|
||||
def test_page_is_cut_and_resumes_at_the_cut():
|
||||
items = [{"id": i, "pad": "x" * 60000} for i in range(10)]
|
||||
first = server._page_result("t", "tool", {}, items, "up", "next")["data"]
|
||||
assert 0 < first["count"] < 10 and first["next_page_token"]
|
||||
upstream, skip = server._read_page({"page_token": first["next_page_token"]})
|
||||
assert (upstream, skip) == ("up", first["count"])
|
||||
last = server._page_result("t", "tool", {}, items, upstream, "next", skip)["data"]
|
||||
assert last["items"][0]["id"] == first["count"]
|
||||
|
||||
|
||||
def test_failures_do_not_quote_the_upstream():
|
||||
class Boom(Exception):
|
||||
pass
|
||||
|
||||
result = server._upstream_failed("azure", Boom("token=abc user ana@customer.example"))
|
||||
assert "abc" not in json.dumps(result) and "ana@" not in json.dumps(result)
|
||||
|
||||
|
||||
def test_evidence_changes_with_the_content():
|
||||
assert server._evidence_ref("t", {}, [1]) != server._evidence_ref("t", {}, [2])
|
||||
|
||||
|
||||
def test_tools_refuse_a_forged_page_token():
|
||||
if SERVER_DIR.name == "cloud-knowledge":
|
||||
pytest.skip("this server has no paged tools")
|
||||
prefix = SERVER_DIR.name.split("-")[0]
|
||||
token = server.mcp_connection.bind(CONNECTIONS[SERVER_DIR.name])
|
||||
try:
|
||||
result = asyncio.run(getattr(server, f"{prefix}_list_servers")({"page_token": "not-a-token"}))
|
||||
finally:
|
||||
server.mcp_connection.unbind(token)
|
||||
assert result["success"] is False and result["error"]["code"] == "invalid_page_token"
|
||||
180
mcp-servers/_common/tool_support.py
Normal file
180
mcp-servers/_common/tool_support.py
Normal file
@@ -0,0 +1,180 @@
|
||||
# Shared by every hyperscaler server. scripts/render_mcp_server.py inlines this file into a server through the
|
||||
# definition's shared_code, because a rendered image holds only the template's files. The builder accepts plain
|
||||
# functions and literal constants here, nothing that runs at import.
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
|
||||
DEFAULT_PAGE_SIZE = 50
|
||||
MAX_PAGE_SIZE = 200
|
||||
# A reply goes into a model's context, which is the real limit on a hand-off, so a page is cut at this size.
|
||||
MAX_RESPONSE_BYTES = 200000
|
||||
# A throttled call is retried inside the budget, then fails and says so.
|
||||
RETRY_ATTEMPTS = 5
|
||||
RETRY_MAX_WAIT_SECONDS = 20
|
||||
RETRY_BUDGET_SECONDS = 60
|
||||
|
||||
# Values under these keys are never returned, in any casing.
|
||||
SECRET_KEYS = (
|
||||
"userdata", "customdata", "customscript", "protectedsettings", "connectionstring", "connectionstrings",
|
||||
"appsettings", "env", "envvars", "environmentvariables", "variables", "password", "adminpassword", "secret",
|
||||
"secretvalue", "sastoken", "accesskey", "primarykey", "secondarykey", "privatekey", "kubeconfig", "credentials",
|
||||
)
|
||||
# Values under these keys name a person, or may. A principal of unknown type counts as a person.
|
||||
PERSON_KEYS = (
|
||||
"createdby", "lastmodifiedby", "updatedby", "creator", "owner", "owneremail", "contact", "contactemail", "email",
|
||||
"objectid", "principalid", "principalname", "userprincipalname",
|
||||
)
|
||||
PRINCIPAL_KINDS = {"user": "[user]", "group": "[group]", "domain": "[domain]", "principal": "[user]",
|
||||
"principalSet": "[group]", "deleted": "[user]"}
|
||||
NON_PERSON_EMAIL_DOMAINS = ("gserviceaccount.com",)
|
||||
|
||||
|
||||
def _clean_text(text: str) -> str:
|
||||
"""Anonymise people inside a string: emails, AWS IAM users, and session names, which are often emails."""
|
||||
def email(match):
|
||||
return match.group(0) if match.group(0).lower().endswith(NON_PERSON_EMAIL_DOMAINS) else "[email]"
|
||||
|
||||
text = re.sub(r"arn:aws[a-z-]*:iam::\d{12}:user/[^\s\"',]+", "[user]", text)
|
||||
text = re.sub(r"(arn:aws[a-z-]*:sts::\d{12}:assumed-role/[^/\s\"',]+)/[^\s\"',]+", r"\1/[session]", text)
|
||||
# Bounded, and skipped without an @: an unbounded pattern takes minutes on one long string.
|
||||
if "@" not in text:
|
||||
return text
|
||||
return re.sub(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]{1,255}\.[A-Za-z]{2,24}", email, text)
|
||||
|
||||
|
||||
def _anonymise_member(value: str) -> str:
|
||||
"""One IAM member, such as user:a@b.example. Service accounts stay: Discovery shows what can reach what."""
|
||||
kind = value.split(":", 1)[0]
|
||||
if ":" in value and kind == "serviceAccount":
|
||||
return value
|
||||
if ":" in value and kind in PRINCIPAL_KINDS:
|
||||
return PRINCIPAL_KINDS[kind]
|
||||
return _clean_text(value)
|
||||
|
||||
|
||||
def _scrub(value, key: str = ""):
|
||||
"""Drop secrets and anonymise people, however deep. Every tool's reply passes through here."""
|
||||
lowered = key.lower().replace("_", "").replace("-", "")
|
||||
if lowered in SECRET_KEYS:
|
||||
return "[redacted]"
|
||||
if lowered in PERSON_KEYS:
|
||||
return "[user]"
|
||||
if isinstance(value, dict):
|
||||
tag = str(value.get("Key", value.get("key", ""))).lower().replace("_", "").replace("-", "")
|
||||
cleaned = {k: _scrub(v, str(k)) for k, v in value.items()}
|
||||
for name in ("Value", "value"):
|
||||
if name in cleaned and tag in PERSON_KEYS:
|
||||
cleaned[name] = "[user]"
|
||||
elif name in cleaned and tag in SECRET_KEYS:
|
||||
cleaned[name] = "[redacted]"
|
||||
return cleaned
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_scrub(v, "members" if lowered == "members" else "") for v in value]
|
||||
if isinstance(value, str):
|
||||
return _anonymise_member(value) if lowered == "members" else _clean_text(value)
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
|
||||
def _retry_delay(response, attempt: int) -> float:
|
||||
"""How long the provider asked us to wait: Retry-After, or Resource Graph's hh:mm:ss quota reset."""
|
||||
reset = response.headers.get("x-ms-user-quota-resets-after", "")
|
||||
if reset.count(":") == 2:
|
||||
hours, minutes, seconds = reset.split(":")
|
||||
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
||||
try:
|
||||
return float(response.headers.get("retry-after", ""))
|
||||
except ValueError:
|
||||
return float(2 ** attempt)
|
||||
|
||||
|
||||
async def _with_backoff(send):
|
||||
"""Send, waiting when the provider throttles. The waits stay well inside the 120-second limit on an MCP call."""
|
||||
waited = 0.0
|
||||
response = await send()
|
||||
for attempt in range(RETRY_ATTEMPTS):
|
||||
if response.status_code not in (429, 503):
|
||||
return response
|
||||
delay = min(_retry_delay(response, attempt), RETRY_MAX_WAIT_SECONDS)
|
||||
if waited + delay > RETRY_BUDGET_SECONDS:
|
||||
return response
|
||||
logger.info("upstream_throttled", status=response.status_code, wait_seconds=delay)
|
||||
await asyncio.sleep(delay)
|
||||
waited += delay
|
||||
response = await send()
|
||||
return response
|
||||
|
||||
|
||||
def _page_size(arguments: Dict[str, Any]) -> int:
|
||||
try:
|
||||
size = int(arguments.get("page_size") or DEFAULT_PAGE_SIZE)
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_PAGE_SIZE
|
||||
return max(1, min(size, MAX_PAGE_SIZE))
|
||||
|
||||
|
||||
def _read_page(arguments: Dict[str, Any]):
|
||||
"""(upstream token, items of that upstream page already returned)."""
|
||||
token = arguments.get("page_token")
|
||||
if not token:
|
||||
return None, 0
|
||||
state = json.loads(base64.urlsafe_b64decode(str(token).encode()).decode())
|
||||
return state["u"], int(state["s"])
|
||||
|
||||
|
||||
def _write_page(upstream, skip: int) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps({"u": upstream, "s": skip}).encode()).decode()
|
||||
|
||||
|
||||
def _evidence_ref(tool: str, arguments: Dict[str, Any], payload) -> str:
|
||||
asked = {k: v for k, v in arguments.items() if k != "page_token"}
|
||||
blob = json.dumps([tool, asked, payload], sort_keys=True, default=str)
|
||||
return "sha256:" + hashlib.sha256(blob.encode()).hexdigest()
|
||||
|
||||
|
||||
def _collected_at() -> str:
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
|
||||
def _refuse(code: str, message: str) -> Dict[str, Any]:
|
||||
return {"success": False, "error": {"code": code, "message": message}}
|
||||
|
||||
|
||||
def _upstream_failed(provider: str, error: Exception) -> Dict[str, Any]:
|
||||
"""Name the failure and never its text: an upstream error can quote a request or an identity."""
|
||||
status = getattr(getattr(error, "response", None), "status_code", None)
|
||||
code = getattr(error, "response", None)
|
||||
if isinstance(code, dict):
|
||||
status = code.get("Error", {}).get("Code")
|
||||
logger.warning("upstream_call_failed", provider=provider, error_type=type(error).__name__, status=status)
|
||||
detail = f" ({status})" if status else ""
|
||||
return _refuse("upstream_failed", f"{provider} refused or failed the call: {type(error).__name__}{detail}")
|
||||
|
||||
|
||||
def _page_result(provider: str, tool: str, arguments: Dict[str, Any], items: List[Any], used, upstream_next,
|
||||
skip: int = 0, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""One page of scrubbed items, cut at MAX_RESPONSE_BYTES; the token resumes exactly where the cut fell."""
|
||||
remaining = [_scrub(item) for item in items[skip:]]
|
||||
kept, size = [], 0
|
||||
for item in remaining:
|
||||
size += len(json.dumps(item, default=str))
|
||||
if kept and size > MAX_RESPONSE_BYTES:
|
||||
break
|
||||
kept.append(item)
|
||||
if len(kept) < len(remaining):
|
||||
next_token = _write_page(used, skip + len(kept))
|
||||
else:
|
||||
next_token = _write_page(upstream_next, 0) if upstream_next else None
|
||||
data = {"provider": provider, "items": kept, "count": len(kept), "next_page_token": next_token,
|
||||
"collected_at": _collected_at(), "evidence_ref": _evidence_ref(tool, arguments, kept)}
|
||||
data.update(extra or {})
|
||||
return {"success": True, "data": data}
|
||||
|
||||
|
||||
def _one_result(provider: str, tool: str, arguments: Dict[str, Any], metadata: Any) -> Dict[str, Any]:
|
||||
clean = _scrub(metadata)
|
||||
return {"success": True, "data": {"provider": provider, "metadata": clean, "collected_at": _collected_at(),
|
||||
"evidence_ref": _evidence_ref(tool, arguments, clean)}}
|
||||
Reference in New Issue
Block a user