Platform scripts, MCP servers, deployment and docs for the KAB runs
- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with --input name=@file), store per-project discovery connections, bootstrap agent MCP keys, refresh the KAB token, publish reports and build estate snapshots. - Hyperscaler discovery servers extended in place; cloud-knowledge folded into them. - deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents. - Platform issue write-ups and current-wiring notes under docs/platform-issues/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
239
scripts/kab_store_connections.py
Normal file
239
scripts/kab_store_connections.py
Normal file
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Store the Azure and Google Cloud connections for the discovery MCP servers in KAB's vault, per project.
|
||||
|
||||
For each server asked for, this:
|
||||
|
||||
1. finds the server in the MCP gateway by name and reads the connection fields it declares;
|
||||
2. builds the connection from the credential source you choose, and checks it carries every required field;
|
||||
3. tests it with the gateway's `connections:verify` before anything is stored;
|
||||
4. with --apply, stores it as an `mcp` / `connection` secret for that server and project
|
||||
(create-only, `If-None-Match: *`; --replace swaps an existing one, `If-Match: *`).
|
||||
|
||||
Without --apply it stops after the test. Credential values never appear in the output, an argument or a file: they
|
||||
come from environment variables, a key file, or the `az` / `gcloud` command line, and go only into the request body.
|
||||
Discovery connections are stored per project, never tenant-wide (mcp-servers/README.md), so --project-id is
|
||||
required. Well-Architected guidance comes from the same servers (`<provider>_search_well_architected`), on the
|
||||
same connection, so it needs nothing stored separately.
|
||||
|
||||
Credential sources:
|
||||
Azure --azure-credential service-principal AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET from the environment
|
||||
--azure-credential cli-token `az account get-access-token` (about an hour; your own rights)
|
||||
GCP --gcp-credential key-file:<path> a service account's JSON key
|
||||
--gcp-credential gcloud-token `gcloud auth print-access-token` (about an hour; your own rights)
|
||||
--gcp-credential adc-token `gcloud auth application-default print-access-token` (the same, from
|
||||
`gcloud auth application-default login`)
|
||||
|
||||
The KAB token comes from $KAB_TOKEN, else --token-file (default ~/.kab_token). Tokens last minutes, so refresh it just
|
||||
before running. API keys (X-API-Key) don't work here: the platform accepts them only on /api/external/ routes.
|
||||
|
||||
Usage (from the repository root):
|
||||
python3 scripts/kab_store_connections.py --tenant <id> --project-id <id> \\
|
||||
--azure-scope-root subscriptions/<id> --azure-credential service-principal \\
|
||||
--gcp-scope-root projects/<id> --gcp-credential key-file:/path/outside/the/repo.json # plan + test
|
||||
python3 scripts/kab_store_connections.py ... --apply # store
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
|
||||
SERVERS = {"azure": "azure-discovery", "gcp": "gcp-discovery"}
|
||||
|
||||
|
||||
class Kab:
|
||||
def __init__(self, api: str, token: str, tenant: str) -> None:
|
||||
self.api = api.rstrip("/")
|
||||
self.headers = {"Authorization": f"Bearer {token}", "X-Tenant-ID": tenant, "Content-Type": "application/json"}
|
||||
|
||||
def call(self, method: str, path: str, body: dict | None = None, headers: dict | None = None,
|
||||
gateway: bool = False) -> tuple[int, dict]:
|
||||
base = f"{self.api}/api/v1/mcp-gateway/api/v1/admin" if gateway else f"{self.api}/api/v1"
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
request = urllib.request.Request(f"{base}{path}", data=data, headers={**self.headers, **(headers or {})},
|
||||
method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
text = response.read()
|
||||
return response.status, (json.loads(text) if text else {})
|
||||
except urllib.error.HTTPError as error:
|
||||
text = error.read()[:2000]
|
||||
try:
|
||||
return error.code, json.loads(text)
|
||||
except ValueError:
|
||||
return error.code, {"detail": text.decode(errors="replace")[:300]}
|
||||
|
||||
|
||||
def token_seconds_left(token: str) -> int | None:
|
||||
try:
|
||||
payload = token.split(".")[1]
|
||||
return int(json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))["exp"] - time.time())
|
||||
except (IndexError, KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def short_error(answer: dict) -> str:
|
||||
"""An error's code and message only. Never the whole answer, in case a service echoes a field."""
|
||||
detail = answer.get("detail", answer)
|
||||
if isinstance(detail, dict):
|
||||
return " ".join(str(detail.get(k)) for k in ("code", "message") if detail.get(k))[:300]
|
||||
return str(detail)[:300]
|
||||
|
||||
|
||||
def run_quiet(command: list[str]) -> str:
|
||||
"""A command's stdout, kept in memory; its output is never echoed."""
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"`{command[0]} {command[1]}` failed (exit {result.returncode}); check you are signed in")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def azure_connection(scope_root: str, source: str, cloud: str | None) -> dict:
|
||||
connection = {"scope_root": scope_root}
|
||||
if cloud:
|
||||
connection["cloud"] = cloud
|
||||
if source == "service-principal":
|
||||
names = {"tenant_id": "AZURE_TENANT_ID", "client_id": "AZURE_CLIENT_ID", "client_secret": "AZURE_CLIENT_SECRET"}
|
||||
missing = [env for env in names.values() if not os.environ.get(env)]
|
||||
if missing:
|
||||
raise SystemExit("set " + ", ".join(missing) + " in the environment for --azure-credential service-principal")
|
||||
connection.update({field: os.environ[env] for field, env in names.items()})
|
||||
elif source == "cli-token":
|
||||
connection["access_token"] = run_quiet(["az", "account", "get-access-token", "--resource",
|
||||
"https://management.azure.com/", "--query", "accessToken", "-o", "tsv"])
|
||||
else:
|
||||
raise SystemExit("--azure-credential is service-principal or cli-token")
|
||||
return connection
|
||||
|
||||
|
||||
def gcp_connection(scope_root: str, source: str) -> dict:
|
||||
connection = {"scope_root": scope_root}
|
||||
if source.startswith("key-file:"):
|
||||
path = Path(source.split(":", 1)[1]).expanduser().resolve()
|
||||
repo = Path(__file__).resolve().parents[1]
|
||||
if repo in path.parents:
|
||||
raise SystemExit("keep the key file outside the repository")
|
||||
text = path.read_text()
|
||||
key = json.loads(text)
|
||||
if key.get("type") != "service_account":
|
||||
raise SystemExit("the key file is not a service account key")
|
||||
connection["service_account_json"] = text
|
||||
elif source == "gcloud-token":
|
||||
connection["access_token"] = run_quiet(["gcloud", "auth", "print-access-token"])
|
||||
elif source == "adc-token":
|
||||
connection["access_token"] = run_quiet(["gcloud", "auth", "application-default", "print-access-token"])
|
||||
else:
|
||||
raise SystemExit("--gcp-credential is key-file:<path>, gcloud-token or adc-token")
|
||||
return connection
|
||||
|
||||
|
||||
def find_server(kab: Kab, name: str) -> dict:
|
||||
status, answer = kab.call("GET", "/servers?page=1&page_size=100", gateway=True)
|
||||
if status != 200:
|
||||
raise SystemExit(f"listing MCP servers: HTTP {status} {short_error(answer)}")
|
||||
servers = answer if isinstance(answer, list) else next((v for v in answer.values() if isinstance(v, list)), [])
|
||||
hits = [s for s in servers if name in (s.get("name"), s.get("server_id"), s.get("slug"))]
|
||||
if len(hits) != 1:
|
||||
known = ", ".join(sorted(str(s.get("name")) for s in servers)) or "none"
|
||||
raise SystemExit(f"{name}: {'not registered' if not hits else 'registered more than once'} on this platform "
|
||||
f"(servers: {known})")
|
||||
return hits[0]
|
||||
|
||||
|
||||
def store(kab: Kab, provider: str, connection: dict, project_id: str, apply: bool, replace: bool) -> bool:
|
||||
assert project_id, "discovery connections are never tenant-wide"
|
||||
name = SERVERS[provider]
|
||||
server = find_server(kab, name)
|
||||
server_id = str(server.get("id") or server.get("server_id"))
|
||||
print(f"{name}: registered as {server_id}, status {server.get('status') or server.get('health_status') or 'unknown'}")
|
||||
|
||||
status, schema = kab.call("GET", f"/connections/schema?mcpServerId={urllib.parse.quote(server_id)}", gateway=True)
|
||||
if status == 200:
|
||||
schema = schema.get("schema", schema)
|
||||
required = set(schema.get("required") or [])
|
||||
known = set((schema.get("properties") or {}).keys())
|
||||
missing, unknown = sorted(required - connection.keys()), sorted(connection.keys() - known) if known else []
|
||||
if missing or unknown:
|
||||
print(f" connection doesn't match the server's fields: missing {missing or 'none'}, unknown {unknown or 'none'}")
|
||||
return False
|
||||
print(f" fields sent: {', '.join(sorted(connection))} (values not shown)")
|
||||
else:
|
||||
print(f" couldn't read the server's connection schema (HTTP {status}); continuing to the test")
|
||||
|
||||
status, verdict = kab.call("POST", "/connections:verify", {"mcpServerId": server_id, "connection": connection,
|
||||
"projectId": project_id}, gateway=True)
|
||||
ok = status == 200 and verdict.get("ok") is True
|
||||
message = verdict.get("message") or short_error(verdict)
|
||||
print(f" test: {'passed' if ok else 'FAILED'} (HTTP {status}{', ' + str(message)[:200] if message else ''})")
|
||||
if not ok:
|
||||
return False
|
||||
if not apply:
|
||||
print(" not stored: run again with --apply")
|
||||
return True
|
||||
|
||||
expires = {"noExpiry": True}
|
||||
if "access_token" in connection:
|
||||
print(" note: an access token expires within about an hour; store a service principal or key for anything longer")
|
||||
body = {"secretType": "mcp", "format": "connection", "displayName": f"{name} ({project_id})",
|
||||
"value": connection, "mcpServerId": server_id, "projectId": project_id, **expires}
|
||||
status, answer = kab.call("PUT", "/secrets", body, headers={"If-Match": "*"} if replace else {"If-None-Match": "*"})
|
||||
if 200 <= status < 300:
|
||||
print(f" stored: {answer.get('providerName') or 'ok'}, version {answer.get('version', '?')}")
|
||||
return True
|
||||
print(f" NOT stored: HTTP {status} {short_error(answer)}"
|
||||
+ (" (it exists already: use --replace)" if "ALREADY_EXISTS" in str(answer) else ""))
|
||||
return False
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--api", default=os.getenv("KAB_API", DEFAULT_API))
|
||||
parser.add_argument("--tenant", required=True, help="the KAB tenant the servers are registered in")
|
||||
parser.add_argument("--project-id", help="the KAB project; required for discovery connections, which are never tenant-wide")
|
||||
parser.add_argument("--azure-scope-root", help="subscriptions/<id>, managementGroups/<id>, or one resource group: subscriptions/<id>/resourceGroups/<name>")
|
||||
parser.add_argument("--azure-credential", help="service-principal or cli-token")
|
||||
parser.add_argument("--azure-cloud", help="optional: the Azure cloud, if not public")
|
||||
parser.add_argument("--gcp-scope-root", help="projects/<id>, folders/<id> or organizations/<id>")
|
||||
parser.add_argument("--gcp-credential", help="key-file:<path>, gcloud-token or adc-token")
|
||||
parser.add_argument("--token-file", type=Path, default=Path.home() / ".kab_token")
|
||||
parser.add_argument("--apply", action="store_true", help="store the connections; without it, only test them")
|
||||
parser.add_argument("--replace", action="store_true", help="replace an existing connection instead of creating one")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
token = os.getenv("KAB_TOKEN") or (args.token_file.read_text().strip() if args.token_file.exists() else "")
|
||||
if not token:
|
||||
raise SystemExit(f"no token: set KAB_TOKEN or write one to {args.token_file}")
|
||||
left = token_seconds_left(token)
|
||||
if left is not None and left < 60:
|
||||
raise SystemExit(f"the KAB token {'expired' if left <= 0 else 'expires in under a minute'}: refresh it")
|
||||
kab = Kab(args.api, token, args.tenant)
|
||||
|
||||
jobs = []
|
||||
if args.azure_scope_root:
|
||||
jobs.append(("azure", azure_connection(args.azure_scope_root, args.azure_credential or "", args.azure_cloud)))
|
||||
if args.gcp_scope_root:
|
||||
jobs.append(("gcp", gcp_connection(args.gcp_scope_root, args.gcp_credential or "")))
|
||||
if jobs and not args.project_id:
|
||||
parser.error("--project-id is required for discovery connections: they are never stored tenant-wide")
|
||||
if not jobs:
|
||||
parser.error("give --azure-scope-root and/or --gcp-scope-root")
|
||||
|
||||
results = [store(kab, provider, connection, args.project_id, args.apply, args.replace)
|
||||
for provider, connection in jobs]
|
||||
for _, connection in jobs: # drop the values as soon as they're sent
|
||||
connection.clear()
|
||||
return 0 if all(results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user