Platform scripts, MCP servers, deployment and docs for the KAB runs
- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with --input name=@file), store per-project discovery connections, bootstrap agent MCP keys, refresh the KAB token, publish reports and build estate snapshots. - Hyperscaler discovery servers extended in place; cloud-knowledge folded into them. - deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents. - Platform issue write-ups and current-wiring notes under docs/platform-issues/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
Read-only discovery of a customer's Google Cloud estate - servers, workloads, Kubernetes clusters, networks, data stores, policies and Recommender findings - through Cloud Asset Inventory.
|
||||
|
||||
Generated by Agent Builder Platform - MCP Server Builder
|
||||
Version: 0.1.1
|
||||
Version: 0.1.4
|
||||
Template: mcp-server-template 1.1.0
|
||||
|
||||
Features:
|
||||
@@ -62,13 +62,30 @@ CONNECTION_SCHEMA = {'type': 'object',
|
||||
'title': 'Scope root',
|
||||
'description': 'What may be read: organizations/<id>, '
|
||||
'folders/<id> or projects/<id>.'},
|
||||
'quota_project': {'type': 'string',
|
||||
'title': 'Quota project',
|
||||
'description': 'Optional. The project Google bills '
|
||||
'Recommender and guidance calls to, '
|
||||
'for a user token; defaults to a '
|
||||
'projects/<id> scope root. The caller '
|
||||
'needs serviceusage.services.use on '
|
||||
'it.'},
|
||||
'service_account_json': {'type': 'string',
|
||||
'title': 'Service account key',
|
||||
'description': 'The read-only service '
|
||||
"account's JSON key.",
|
||||
'writeOnly': True,
|
||||
'format': 'password'}},
|
||||
'required': ['scope_root', 'service_account_json'],
|
||||
'format': 'password'},
|
||||
'access_token': {'type': 'string',
|
||||
'title': 'Access token',
|
||||
'description': 'For a short trial only, instead of '
|
||||
'the key. A token from gcloud auth '
|
||||
'print-access-token. It expires within '
|
||||
'about an hour and carries that '
|
||||
"person's rights.",
|
||||
'writeOnly': True,
|
||||
'format': 'password'}},
|
||||
'required': ['scope_root'],
|
||||
'additionalProperties': False}
|
||||
VERIFY_TOOL = 'verify_connection'
|
||||
|
||||
@@ -101,7 +118,7 @@ class Config:
|
||||
# Server Identity
|
||||
SERVER_ID = os.getenv("MCP_SERVER_ID", 'gcp-discovery')
|
||||
SERVER_NAME = os.getenv("MCP_SERVER_NAME", 'Google Cloud Discovery')
|
||||
SERVER_VERSION = os.getenv("MCP_SERVER_VERSION", '0.1.1')
|
||||
SERVER_VERSION = os.getenv("MCP_SERVER_VERSION", '0.1.4')
|
||||
# MCP_SERVER_PORT, or SERVER_PORT as the ConfigMaps in the tools repo name it
|
||||
SERVER_PORT = int(os.getenv("MCP_SERVER_PORT") or os.getenv("SERVER_PORT") or "8113")
|
||||
|
||||
@@ -551,6 +568,18 @@ TOOLS_REGISTRY = [
|
||||
'reply.'}},
|
||||
'required': ['project_id']}
|
||||
},
|
||||
{
|
||||
"name": 'gcp_search_well_architected',
|
||||
"description": "Search the Google Cloud Well-Architected Framework's guidance by pillar and topic. Both come from fixed lists, so no customer detail leaves the platform. Uses this connection's own credential.",
|
||||
"inputSchema": {'type': 'object',
|
||||
'properties': {'pillar': {'type': 'string',
|
||||
'description': 'reliability, security, cost, operations, '
|
||||
'performance or sustainability.'},
|
||||
'topic': {'type': 'string',
|
||||
'description': 'One of the fixed topics; an invalid value '
|
||||
'lists them.'}},
|
||||
'required': ['pillar', 'topic']}
|
||||
},
|
||||
{
|
||||
"name": 'verify_connection',
|
||||
"description": (
|
||||
@@ -607,6 +636,8 @@ SECRET_KEYS = (
|
||||
"secretvalue", "sastoken", "accesskey", "primarykey", "secondarykey", "privatekey", "kubeconfig", "credentials",
|
||||
)
|
||||
|
||||
SCALAR_SAFE_KEYS = ("env",)
|
||||
|
||||
PERSON_KEYS = (
|
||||
"createdby", "lastmodifiedby", "updatedby", "creator", "owner", "owneremail", "contact", "contactemail", "email",
|
||||
"objectid", "principalid", "principalname", "userprincipalname",
|
||||
@@ -638,10 +669,13 @@ def _anonymise_member(value: str) -> str:
|
||||
return PRINCIPAL_KINDS[kind]
|
||||
return _clean_text(value)
|
||||
|
||||
def _scalar(value) -> bool:
|
||||
return value is None or isinstance(value, (str, bool, int, float))
|
||||
|
||||
def _scrub(value, key: str = ""):
|
||||
"""Drop secrets and anonymise people, however deep. Every tool's reply passes through here."""
|
||||
lowered = key.lower().replace("_", "").replace("-", "")
|
||||
if lowered in SECRET_KEYS:
|
||||
if lowered in SECRET_KEYS and not (lowered in SCALAR_SAFE_KEYS and _scalar(value)):
|
||||
return "[redacted]"
|
||||
if lowered in PERSON_KEYS:
|
||||
return "[user]"
|
||||
@@ -651,7 +685,7 @@ def _scrub(value, key: str = ""):
|
||||
for name in ("Value", "value"):
|
||||
if name in cleaned and tag in PERSON_KEYS:
|
||||
cleaned[name] = "[user]"
|
||||
elif name in cleaned and tag in SECRET_KEYS:
|
||||
elif name in cleaned and tag in SECRET_KEYS and not (tag in SCALAR_SAFE_KEYS and _scalar(cleaned[name])):
|
||||
cleaned[name] = "[redacted]"
|
||||
return cleaned
|
||||
if isinstance(value, (list, tuple)):
|
||||
@@ -817,6 +851,8 @@ def _gcp_client():
|
||||
|
||||
async def _gcp_token() -> str:
|
||||
connection = get_connection()
|
||||
if connection.get("access_token"):
|
||||
return str(connection["access_token"])
|
||||
key = hashlib.sha256(json.dumps(connection, sort_keys=True).encode()).hexdigest()
|
||||
cached = GCP_TOKENS.get(key)
|
||||
now = time.time()
|
||||
@@ -833,10 +869,27 @@ async def _gcp_token() -> str:
|
||||
GCP_TOKENS[key] = (token, now + 3000)
|
||||
return token
|
||||
|
||||
def _gcp_quota_project() -> str:
|
||||
"""The project Google bills a user credential's calls to: the connection's quota_project, else a projects/<id>
|
||||
scope root. Recommender and Developer Knowledge refuse a user token without one; Cloud Asset doesn't need it."""
|
||||
connection = get_connection()
|
||||
if connection.get("quota_project"):
|
||||
return str(connection["quota_project"])
|
||||
kind, _, identifier = str(connection.get("scope_root", "")).strip("/").partition("/")
|
||||
return identifier if kind == "projects" else ""
|
||||
|
||||
GCP_QUOTA_APIS = ("https://recommender.googleapis.com/",) # a literal: the builder refuses computed module-level values
|
||||
|
||||
async def _gcp_auth(quota: bool) -> Dict[str, str]:
|
||||
headers = {"Authorization": f"Bearer {await _gcp_token()}"}
|
||||
project = _gcp_quota_project() if quota else ""
|
||||
if project:
|
||||
headers["x-goog-user-project"] = project
|
||||
return headers
|
||||
|
||||
async def _gcp_get(url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
token = await _gcp_token()
|
||||
response = await _with_backoff(
|
||||
lambda: _gcp_client().get(url, params=params, headers={"Authorization": f"Bearer {token}"}))
|
||||
headers = await _gcp_auth(url.startswith(GCP_QUOTA_APIS))
|
||||
response = await _with_backoff(lambda: _gcp_client().get(url, params=params, headers=headers))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
@@ -885,6 +938,130 @@ def _gcp_recommendation(item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
category = str(item.get("primaryImpact", {}).get("category", ""))
|
||||
return {**item, "pillar": GCP_PILLARS.get(category.lower(), "other"), "provider_category": category}
|
||||
|
||||
KNOWLEDGE_SERVERS = {
|
||||
"azure": {"url": "https://learn.microsoft.com/api/mcp", "tool": "microsoft_docs_search",
|
||||
"framework": "Azure Well-Architected Framework"},
|
||||
"aws": {"url": "https://aws-mcp.eu-west-1.api.aws/mcp", "tool": "aws___search_documentation",
|
||||
"framework": "AWS Well-Architected Framework"},
|
||||
"gcp": {"url": "https://developerknowledge.googleapis.com/mcp", "tool": "search_documents",
|
||||
"framework": "Google Cloud Well-Architected Framework"},
|
||||
}
|
||||
|
||||
KNOWLEDGE_PILLARS = ("reliability", "security", "cost", "operations", "performance", "sustainability")
|
||||
|
||||
KNOWLEDGE_TOPICS = (
|
||||
"landing zone", "resource organization", "identity and access", "network segmentation", "hybrid connectivity",
|
||||
"encryption", "logging and monitoring", "backup and recovery", "disaster recovery", "high availability",
|
||||
"scaling", "kubernetes", "containers", "virtual machines", "databases", "storage", "serverless", "governance",
|
||||
"policy enforcement", "tagging", "cost allocation", "migration", "incident response", "patching",
|
||||
)
|
||||
|
||||
KNOWLEDGE_MAX_TEXT = 4000
|
||||
|
||||
def _knowledge_client():
|
||||
return get_upstream_client("knowledge", lambda c: httpx.AsyncClient(timeout=60, follow_redirects=False))
|
||||
|
||||
def _knowledge_message(response) -> Dict[str, Any]:
|
||||
"""The JSON-RPC reply, whether it came back as JSON or as a server-sent event."""
|
||||
if "text/event-stream" in response.headers.get("content-type", ""):
|
||||
lines = [line[5:].strip() for line in response.text.splitlines() if line.startswith("data:")]
|
||||
return json.loads(lines[-1]) if lines else {}
|
||||
return response.json()
|
||||
|
||||
async def _knowledge_post(provider: str, connection: Dict[str, Any], body: Dict[str, Any],
|
||||
session: Optional[str] = None, auth: Optional[Dict[str, str]] = None):
|
||||
"""auth: headers a discovery server supplies from its own connection (gcp-discovery's OAuth token), used in
|
||||
place of the stored API key."""
|
||||
url = KNOWLEDGE_SERVERS[provider]["url"]
|
||||
headers = {"Accept": "application/json, text/event-stream", "Content-Type": "application/json"}
|
||||
if session:
|
||||
headers["Mcp-Session-Id"] = session
|
||||
if auth:
|
||||
headers.update(auth)
|
||||
elif provider == "gcp":
|
||||
headers["X-Goog-Api-Key"] = connection.get("google_api_key", "")
|
||||
content = json.dumps(body)
|
||||
if provider == "aws":
|
||||
import boto3
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
|
||||
def sign():
|
||||
credentials = boto3.Session(aws_access_key_id=connection.get("aws_access_key_id"),
|
||||
aws_secret_access_key=connection.get("aws_secret_access_key")).get_credentials()
|
||||
request = AWSRequest(method="POST", url=url, data=content, headers=headers)
|
||||
SigV4Auth(credentials, "aws-mcp", "eu-west-1").add_auth(request)
|
||||
return dict(request.headers)
|
||||
|
||||
headers = await asyncio.to_thread(sign)
|
||||
response = await _with_backoff(lambda: _knowledge_client().post(url, content=content, headers=headers))
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
async def _knowledge_session(provider: str, connection: Dict[str, Any], auth: Optional[Dict[str, str]] = None) -> Optional[str]:
|
||||
response = await _knowledge_post(provider, connection, {
|
||||
"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {"protocolVersion": "2025-06-18", "capabilities": {},
|
||||
"clientInfo": {"name": "kab-cp1-discovery", "version": "0.1.0"}}}, auth=auth)
|
||||
return response.headers.get("mcp-session-id")
|
||||
|
||||
async def _knowledge_tools(provider: str, connection: Dict[str, Any], session: Optional[str] = None,
|
||||
auth: Optional[Dict[str, str]] = None) -> List[Dict[str, Any]]:
|
||||
session = session or await _knowledge_session(provider, connection, auth)
|
||||
response = await _knowledge_post(provider, connection, {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, session, auth)
|
||||
return _knowledge_message(response).get("result", {}).get("tools", [])
|
||||
|
||||
def _knowledge_argument(tool: Dict[str, Any]) -> str:
|
||||
"""The search tool's query argument. Vendors name it differently and may change it, so it's read from the tool."""
|
||||
schema = tool.get("inputSchema", {})
|
||||
properties = schema.get("properties", {})
|
||||
for name in list(schema.get("required", [])) + list(properties):
|
||||
if properties.get(name, {}).get("type") == "string":
|
||||
return name
|
||||
return "query"
|
||||
|
||||
async def _knowledge_search(provider: str, connection: Dict[str, Any], query: str,
|
||||
auth: Optional[Dict[str, str]] = None) -> List[Dict[str, Any]]:
|
||||
session = await _knowledge_session(provider, connection, auth)
|
||||
name = KNOWLEDGE_SERVERS[provider]["tool"]
|
||||
tools = {t.get("name"): t for t in await _knowledge_tools(provider, connection, session, auth)}
|
||||
if name not in tools:
|
||||
raise LookupError(f"{provider} no longer offers {name}")
|
||||
response = await _knowledge_post(provider, connection, {
|
||||
"jsonrpc": "2.0", "id": 3, "method": "tools/call",
|
||||
"params": {"name": name, "arguments": {_knowledge_argument(tools[name]): query}}}, session, auth)
|
||||
message = _knowledge_message(response)
|
||||
results = []
|
||||
for part in message.get("result", {}).get("content", []):
|
||||
if part.get("type") != "text":
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(part["text"])
|
||||
except ValueError:
|
||||
parsed = {"content": part["text"]}
|
||||
found = parsed.get("results", parsed) if isinstance(parsed, dict) else parsed
|
||||
for item in found if isinstance(found, list) else [found]:
|
||||
if isinstance(item, dict):
|
||||
results.append({"title": item.get("title"), "url": item.get("contentUrl") or item.get("url"),
|
||||
"text": str(item.get("content") or item.get("text") or "")[:KNOWLEDGE_MAX_TEXT]})
|
||||
return results
|
||||
|
||||
async def _knowledge_tool(provider: str, tool: str, arguments: Dict[str, Any], connection: Dict[str, Any],
|
||||
auth: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
|
||||
"""One provider's `<provider>_search_well_architected` tool: fixed pillar and topic in, guidance excerpts out."""
|
||||
pillar, topic = arguments.get("pillar"), arguments.get("topic")
|
||||
if pillar not in KNOWLEDGE_PILLARS:
|
||||
return _refuse("invalid_argument", "pillar must be one of: " + ", ".join(KNOWLEDGE_PILLARS))
|
||||
if topic not in KNOWLEDGE_TOPICS:
|
||||
return _refuse("invalid_argument", "topic must be one of: " + ", ".join(KNOWLEDGE_TOPICS))
|
||||
query = f"{KNOWLEDGE_SERVERS[provider]['framework']} {pillar} {topic}"
|
||||
try:
|
||||
results = await _knowledge_search(provider, connection, query, auth)
|
||||
except Exception as error:
|
||||
return _upstream_failed(provider, error)
|
||||
return {"success": True, "data": {"provider": provider, "pillar": pillar, "topic": topic, "results": results,
|
||||
"collected_at": _collected_at(), "evidence_ref": _evidence_ref(tool, arguments, results)}}
|
||||
|
||||
async def gcp_check_connection(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
try:
|
||||
await _gcp_get(f"{GCP_ASSET_API}/{_gcp_scope()}:searchAllResources", {"pageSize": 1})
|
||||
@@ -965,6 +1142,9 @@ async def gcp_list_recommendations(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
url = f"{GCP_RECOMMENDER_API}/projects/{project_id}/locations/{location}/recommenders/{recommender}/recommendations"
|
||||
return await _gcp_page("gcp_list_recommendations", arguments, url, {}, "recommendations", _gcp_recommendation)
|
||||
|
||||
async def gcp_search_well_architected(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return await _knowledge_tool("gcp", "gcp_search_well_architected", arguments, get_connection(), await _gcp_auth(True))
|
||||
|
||||
async def verify_connection(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Check that the connection sent with this request works against the upstream. Makes no change."""
|
||||
connection = get_connection()
|
||||
@@ -992,6 +1172,7 @@ TOOL_FUNCTIONS = {
|
||||
'gcp_list_organization_policies': gcp_list_organization_policies,
|
||||
'gcp_list_resource_policies': gcp_list_resource_policies,
|
||||
'gcp_list_recommendations': gcp_list_recommendations,
|
||||
'gcp_search_well_architected': gcp_search_well_architected,
|
||||
'verify_connection': verify_connection,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user