- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with --input name=@file), store per-project discovery connections, bootstrap agent MCP keys, refresh the KAB token, publish reports and build estate snapshots. - Hyperscaler discovery servers extended in place; cloud-knowledge folded into them. - deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents. - Platform issue write-ups and current-wiring notes under docs/platform-issues/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
240 lines
13 KiB
Python
240 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Store the Azure and Google Cloud connections for the discovery MCP servers in KAB's vault, per project.
|
|
|
|
For each server asked for, this:
|
|
|
|
1. finds the server in the MCP gateway by name and reads the connection fields it declares;
|
|
2. builds the connection from the credential source you choose, and checks it carries every required field;
|
|
3. tests it with the gateway's `connections:verify` before anything is stored;
|
|
4. with --apply, stores it as an `mcp` / `connection` secret for that server and project
|
|
(create-only, `If-None-Match: *`; --replace swaps an existing one, `If-Match: *`).
|
|
|
|
Without --apply it stops after the test. Credential values never appear in the output, an argument or a file: they
|
|
come from environment variables, a key file, or the `az` / `gcloud` command line, and go only into the request body.
|
|
Discovery connections are stored per project, never tenant-wide (mcp-servers/README.md), so --project-id is
|
|
required. Well-Architected guidance comes from the same servers (`<provider>_search_well_architected`), on the
|
|
same connection, so it needs nothing stored separately.
|
|
|
|
Credential sources:
|
|
Azure --azure-credential service-principal AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET from the environment
|
|
--azure-credential cli-token `az account get-access-token` (about an hour; your own rights)
|
|
GCP --gcp-credential key-file:<path> a service account's JSON key
|
|
--gcp-credential gcloud-token `gcloud auth print-access-token` (about an hour; your own rights)
|
|
--gcp-credential adc-token `gcloud auth application-default print-access-token` (the same, from
|
|
`gcloud auth application-default login`)
|
|
|
|
The KAB token comes from $KAB_TOKEN, else --token-file (default ~/.kab_token). Tokens last minutes, so refresh it just
|
|
before running. API keys (X-API-Key) don't work here: the platform accepts them only on /api/external/ routes.
|
|
|
|
Usage (from the repository root):
|
|
python3 scripts/kab_store_connections.py --tenant <id> --project-id <id> \\
|
|
--azure-scope-root subscriptions/<id> --azure-credential service-principal \\
|
|
--gcp-scope-root projects/<id> --gcp-credential key-file:/path/outside/the/repo.json # plan + test
|
|
python3 scripts/kab_store_connections.py ... --apply # store
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
|
|
SERVERS = {"azure": "azure-discovery", "gcp": "gcp-discovery"}
|
|
|
|
|
|
class Kab:
|
|
def __init__(self, api: str, token: str, tenant: str) -> None:
|
|
self.api = api.rstrip("/")
|
|
self.headers = {"Authorization": f"Bearer {token}", "X-Tenant-ID": tenant, "Content-Type": "application/json"}
|
|
|
|
def call(self, method: str, path: str, body: dict | None = None, headers: dict | None = None,
|
|
gateway: bool = False) -> tuple[int, dict]:
|
|
base = f"{self.api}/api/v1/mcp-gateway/api/v1/admin" if gateway else f"{self.api}/api/v1"
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
request = urllib.request.Request(f"{base}{path}", data=data, headers={**self.headers, **(headers or {})},
|
|
method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
text = response.read()
|
|
return response.status, (json.loads(text) if text else {})
|
|
except urllib.error.HTTPError as error:
|
|
text = error.read()[:2000]
|
|
try:
|
|
return error.code, json.loads(text)
|
|
except ValueError:
|
|
return error.code, {"detail": text.decode(errors="replace")[:300]}
|
|
|
|
|
|
def token_seconds_left(token: str) -> int | None:
|
|
try:
|
|
payload = token.split(".")[1]
|
|
return int(json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))["exp"] - time.time())
|
|
except (IndexError, KeyError, TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def short_error(answer: dict) -> str:
|
|
"""An error's code and message only. Never the whole answer, in case a service echoes a field."""
|
|
detail = answer.get("detail", answer)
|
|
if isinstance(detail, dict):
|
|
return " ".join(str(detail.get(k)) for k in ("code", "message") if detail.get(k))[:300]
|
|
return str(detail)[:300]
|
|
|
|
|
|
def run_quiet(command: list[str]) -> str:
|
|
"""A command's stdout, kept in memory; its output is never echoed."""
|
|
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
|
if result.returncode != 0:
|
|
raise SystemExit(f"`{command[0]} {command[1]}` failed (exit {result.returncode}); check you are signed in")
|
|
return result.stdout.strip()
|
|
|
|
|
|
def azure_connection(scope_root: str, source: str, cloud: str | None) -> dict:
|
|
connection = {"scope_root": scope_root}
|
|
if cloud:
|
|
connection["cloud"] = cloud
|
|
if source == "service-principal":
|
|
names = {"tenant_id": "AZURE_TENANT_ID", "client_id": "AZURE_CLIENT_ID", "client_secret": "AZURE_CLIENT_SECRET"}
|
|
missing = [env for env in names.values() if not os.environ.get(env)]
|
|
if missing:
|
|
raise SystemExit("set " + ", ".join(missing) + " in the environment for --azure-credential service-principal")
|
|
connection.update({field: os.environ[env] for field, env in names.items()})
|
|
elif source == "cli-token":
|
|
connection["access_token"] = run_quiet(["az", "account", "get-access-token", "--resource",
|
|
"https://management.azure.com/", "--query", "accessToken", "-o", "tsv"])
|
|
else:
|
|
raise SystemExit("--azure-credential is service-principal or cli-token")
|
|
return connection
|
|
|
|
|
|
def gcp_connection(scope_root: str, source: str) -> dict:
|
|
connection = {"scope_root": scope_root}
|
|
if source.startswith("key-file:"):
|
|
path = Path(source.split(":", 1)[1]).expanduser().resolve()
|
|
repo = Path(__file__).resolve().parents[1]
|
|
if repo in path.parents:
|
|
raise SystemExit("keep the key file outside the repository")
|
|
text = path.read_text()
|
|
key = json.loads(text)
|
|
if key.get("type") != "service_account":
|
|
raise SystemExit("the key file is not a service account key")
|
|
connection["service_account_json"] = text
|
|
elif source == "gcloud-token":
|
|
connection["access_token"] = run_quiet(["gcloud", "auth", "print-access-token"])
|
|
elif source == "adc-token":
|
|
connection["access_token"] = run_quiet(["gcloud", "auth", "application-default", "print-access-token"])
|
|
else:
|
|
raise SystemExit("--gcp-credential is key-file:<path>, gcloud-token or adc-token")
|
|
return connection
|
|
|
|
|
|
def find_server(kab: Kab, name: str) -> dict:
|
|
status, answer = kab.call("GET", "/servers?page=1&page_size=100", gateway=True)
|
|
if status != 200:
|
|
raise SystemExit(f"listing MCP servers: HTTP {status} {short_error(answer)}")
|
|
servers = answer if isinstance(answer, list) else next((v for v in answer.values() if isinstance(v, list)), [])
|
|
hits = [s for s in servers if name in (s.get("name"), s.get("server_id"), s.get("slug"))]
|
|
if len(hits) != 1:
|
|
known = ", ".join(sorted(str(s.get("name")) for s in servers)) or "none"
|
|
raise SystemExit(f"{name}: {'not registered' if not hits else 'registered more than once'} on this platform "
|
|
f"(servers: {known})")
|
|
return hits[0]
|
|
|
|
|
|
def store(kab: Kab, provider: str, connection: dict, project_id: str, apply: bool, replace: bool) -> bool:
|
|
assert project_id, "discovery connections are never tenant-wide"
|
|
name = SERVERS[provider]
|
|
server = find_server(kab, name)
|
|
server_id = str(server.get("id") or server.get("server_id"))
|
|
print(f"{name}: registered as {server_id}, status {server.get('status') or server.get('health_status') or 'unknown'}")
|
|
|
|
status, schema = kab.call("GET", f"/connections/schema?mcpServerId={urllib.parse.quote(server_id)}", gateway=True)
|
|
if status == 200:
|
|
schema = schema.get("schema", schema)
|
|
required = set(schema.get("required") or [])
|
|
known = set((schema.get("properties") or {}).keys())
|
|
missing, unknown = sorted(required - connection.keys()), sorted(connection.keys() - known) if known else []
|
|
if missing or unknown:
|
|
print(f" connection doesn't match the server's fields: missing {missing or 'none'}, unknown {unknown or 'none'}")
|
|
return False
|
|
print(f" fields sent: {', '.join(sorted(connection))} (values not shown)")
|
|
else:
|
|
print(f" couldn't read the server's connection schema (HTTP {status}); continuing to the test")
|
|
|
|
status, verdict = kab.call("POST", "/connections:verify", {"mcpServerId": server_id, "connection": connection,
|
|
"projectId": project_id}, gateway=True)
|
|
ok = status == 200 and verdict.get("ok") is True
|
|
message = verdict.get("message") or short_error(verdict)
|
|
print(f" test: {'passed' if ok else 'FAILED'} (HTTP {status}{', ' + str(message)[:200] if message else ''})")
|
|
if not ok:
|
|
return False
|
|
if not apply:
|
|
print(" not stored: run again with --apply")
|
|
return True
|
|
|
|
expires = {"noExpiry": True}
|
|
if "access_token" in connection:
|
|
print(" note: an access token expires within about an hour; store a service principal or key for anything longer")
|
|
body = {"secretType": "mcp", "format": "connection", "displayName": f"{name} ({project_id})",
|
|
"value": connection, "mcpServerId": server_id, "projectId": project_id, **expires}
|
|
status, answer = kab.call("PUT", "/secrets", body, headers={"If-Match": "*"} if replace else {"If-None-Match": "*"})
|
|
if 200 <= status < 300:
|
|
print(f" stored: {answer.get('providerName') or 'ok'}, version {answer.get('version', '?')}")
|
|
return True
|
|
print(f" NOT stored: HTTP {status} {short_error(answer)}"
|
|
+ (" (it exists already: use --replace)" if "ALREADY_EXISTS" in str(answer) else ""))
|
|
return False
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--api", default=os.getenv("KAB_API", DEFAULT_API))
|
|
parser.add_argument("--tenant", required=True, help="the KAB tenant the servers are registered in")
|
|
parser.add_argument("--project-id", help="the KAB project; required for discovery connections, which are never tenant-wide")
|
|
parser.add_argument("--azure-scope-root", help="subscriptions/<id>, managementGroups/<id>, or one resource group: subscriptions/<id>/resourceGroups/<name>")
|
|
parser.add_argument("--azure-credential", help="service-principal or cli-token")
|
|
parser.add_argument("--azure-cloud", help="optional: the Azure cloud, if not public")
|
|
parser.add_argument("--gcp-scope-root", help="projects/<id>, folders/<id> or organizations/<id>")
|
|
parser.add_argument("--gcp-credential", help="key-file:<path>, gcloud-token or adc-token")
|
|
parser.add_argument("--token-file", type=Path, default=Path.home() / ".kab_token")
|
|
parser.add_argument("--apply", action="store_true", help="store the connections; without it, only test them")
|
|
parser.add_argument("--replace", action="store_true", help="replace an existing connection instead of creating one")
|
|
args = parser.parse_args(argv)
|
|
|
|
token = os.getenv("KAB_TOKEN") or (args.token_file.read_text().strip() if args.token_file.exists() else "")
|
|
if not token:
|
|
raise SystemExit(f"no token: set KAB_TOKEN or write one to {args.token_file}")
|
|
left = token_seconds_left(token)
|
|
if left is not None and left < 60:
|
|
raise SystemExit(f"the KAB token {'expired' if left <= 0 else 'expires in under a minute'}: refresh it")
|
|
kab = Kab(args.api, token, args.tenant)
|
|
|
|
jobs = []
|
|
if args.azure_scope_root:
|
|
jobs.append(("azure", azure_connection(args.azure_scope_root, args.azure_credential or "", args.azure_cloud)))
|
|
if args.gcp_scope_root:
|
|
jobs.append(("gcp", gcp_connection(args.gcp_scope_root, args.gcp_credential or "")))
|
|
if jobs and not args.project_id:
|
|
parser.error("--project-id is required for discovery connections: they are never stored tenant-wide")
|
|
if not jobs:
|
|
parser.error("give --azure-scope-root and/or --gcp-scope-root")
|
|
|
|
results = [store(kab, provider, connection, args.project_id, args.apply, args.replace)
|
|
for provider, connection in jobs]
|
|
for _, connection in jobs: # drop the values as soon as they're sent
|
|
connection.clear()
|
|
return 0 if all(results) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|