Platform scripts, MCP servers, deployment and docs for the KAB runs

- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with
  --input name=@file), store per-project discovery connections, bootstrap agent MCP keys,
  refresh the KAB token, publish reports and build estate snapshots.
- Hyperscaler discovery servers extended in place; cloud-knowledge folded into them.
- deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents.
- Platform issue write-ups and current-wiring notes under docs/platform-issues/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent c14a94f33b
commit f659564208
71 changed files with 2818 additions and 2821 deletions

View File

@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Turn collected Google Cloud discovery results into a snapshot the sample server can serve.
For a trial without the Vault Service: the collection runs where the credential is, and only resource metadata reaches
the cluster. The input is one JSON object of tool results, keyed "<tool>/<kind>", with `items` lists as gcp-discovery
returns them. Keep the output out of this repository: it describes a real estate.
Usage: build_estate_snapshot.py <results.json> <snapshot.json> --engagement <id> prints counts only
"""
import argparse
import json
import os
from pathlib import Path
def short(name: str) -> str:
return str(name or "").rstrip("/").rsplit("/", 1)[-1]
def region(location: str) -> str:
"""A zone's region: `site` is a region, the unit network data is collected for."""
parts = str(location or "unknown").split("-")
return "-".join(parts[:2]) if len(parts) > 2 else str(location or "unknown")
def resource(item: dict) -> dict:
versions = item.get("versionedResources") or [{}]
return versions[0].get("resource") or {}
def entry(item: dict, role: str, asset_class: str, depends_on: list, notes: str = "") -> dict:
extra = item.get("additionalAttributes") or {}
ips = extra.get("internalIPs") or [None]
return {"hostname": item.get("displayName") or short(item.get("name")), "role": role, "site": region(item.get("location")),
"asset_class": asset_class, "os": None, "support_status": "unknown", "services": [], "depends_on": depends_on,
"compliance_scope": [], "serial": str(extra.get("id") or item.get("name")), "fqdn": None, "ip": ips[0], "notes": notes}
def build(results: dict, engagement: str) -> dict:
assets = results.get("list_resources/None") or []
def of(kind: str) -> list:
return [a for a in assets if str(a.get("assetType")).lower() == kind]
estate, ids = {}, {}
def add(item: dict, role: str, asset_class: str, depends_on: list) -> None:
key = f"SE-G{len(estate) + 1:02d}"
ids[item.get("name")] = key
estate[key] = entry(item, role, asset_class, depends_on, str(resource(item).get("description") or ""))
def named(*links) -> list:
return sorted({ids[n] for n in ids for link in links if link and short(n) == short(link)})
for network in of("compute.googleapis.com/network"):
add(network, "network", "network_device", [])
for subnet in of("compute.googleapis.com/subnetwork"):
add(subnet, "subnetwork", "network_device", named(resource(subnet).get("network")))
for route in of("compute.googleapis.com/route"):
add(route, "route", "network_device", named(resource(route).get("network")))
for disk in of("compute.googleapis.com/disk"):
add(disk, "disk", "storage_array", [])
for vm in of("compute.googleapis.com/instance"):
extra = vm.get("additionalAttributes") or {}
disks = [d for d in of("compute.googleapis.com/disk") if short(vm.get("name")) in [short(u) for u in (d.get("additionalAttributes") or {}).get("users") or []]]
add(vm, "server", "server", named(*(extra.get("networkInterfaceNetworks") or []), *[d.get("name") for d in disks]))
images = [short(resource(d).get("sourceImage")) for d in disks if resource(d).get("sourceImage")]
estate[ids[vm.get("name")]]["os"] = images[0] if images else None
for bucket in of("storage.googleapis.com/bucket"):
add(bucket, "storage", "storage_array", [])
for account in of("iam.googleapis.com/serviceaccount"):
add(account, "identity", "cloud_resource", [])
sites = {}
def site(name: str) -> dict:
return sites.setdefault(name, {"devices": [], "links": [], "ranges": [], "firewall_rules": []})
for subnet in of("compute.googleapis.com/subnetwork"):
cidr = resource(subnet).get("ipCidrRange")
site(region(subnet.get("location")))["ranges"].append(
{"cidr": cidr, "site": region(subnet.get("location")), "segment": subnet.get("displayName")})
for key, value in estate.items():
site(value["site"])
if value["role"] == "network":
site(value["site"])["devices"].append({"name": value["hostname"], "scope_entry_id": key, "type": "vpc", "os": None,
"support_status": "unknown"})
for rule in results.get("get_network_topology/firewall_rules") or []:
next(iter(sites.values()))["firewall_rules"].append({"rule_id": rule.get("displayName"), "owner": None, "description": "firewall rule"})
project = (of("cloudresourcemanager.googleapis.com/project") or [{}])[0].get("displayName") or engagement
return {"engagement_id": engagement, "estate": estate, "sites": sites,
"apps": [{"app_id": "APP-PROJECT", "name": project, "assets": list(estate)}],
"sources": [{"connector_id": "CONN-GCP-SNAPSHOT", "type": "estate_scan", "mcp_tool": "alz-estate.scan_target", "read_only": True}]}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("results", type=Path)
parser.add_argument("snapshot", type=Path)
parser.add_argument("--engagement", required=True)
args = parser.parse_args()
snapshot = build(json.loads(args.results.read_text()), args.engagement)
descriptor = os.open(args.snapshot, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "w") as handle:
json.dump(snapshot, handle)
print({"entries": len(snapshot["estate"]), "sites": len(snapshot["sites"]), "bytes": args.snapshot.stat().st_size,
"roles": sorted({v["role"] for v in snapshot["estate"].values()}),
"ranges": sum(len(s["ranges"]) for s in snapshot["sites"].values())})
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Prepare the individual-agents Discovery run: the nine logical MCP servers, and each agent's gateway key.
Shows what it would do, and changes nothing without --apply. With --apply it:
1. registers each missing logical name in the MCP gateway, against the sample server, and discovers its tools;
2. for each agent with no <agent>-secrets, creates a gateway key and stores it in that Secret;
3. restarts the agents it gave a key.
A key is shown only once, so it goes from the API's reply straight to kubectl on stdin. It is never written to a file,
an argument or the output. Agent keys belong to the tenant they're created in, so this must run in the servers' tenant.
KAB tokens last five minutes, so refresh the token just before running. The token comes from $KAB_TOKEN or
~/.kab_token. --apply also needs $MCP_REG_KEY, the sample server's MCP_REGISTRATION_API_KEY, when a name is missing.
Usage (from the repository root):
python3 scripts/kab_bootstrap_discovery.py # plan only
python3 scripts/kab_bootstrap_discovery.py --apply
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
CP1_TENANT = "6ae37d13999438461a529409"
SAMPLE_URL = "http://alz-discovery-sample.mcp-servers.svc.cluster.local:8110"
LOGICAL_SERVERS = ("alz-scope", "alz-estate", "alz-cloud", "alz-cmdb", "alz-inventory", "alz-schema",
"alz-dependency", "alz-network", "alz-evidence")
AGENTS = ("cp1-discovery-planner-agent", "cp1-estate-scan-executor-agent", "cp1-asset-inventory-normalisation-agent",
"cp1-cmdb-itsm-reconciliation-agent", "cp1-data-classification-agent",
"cp1-network-discovery-assessment-agent", "cp1-dependency-mapping-agent", "cp1-discovery-critic-agent")
class Gateway:
def __init__(self, api: str, token: str, tenant: str) -> None:
self.base = f"{api.rstrip('/')}/api/v1/mcp-gateway/api/v1/admin"
self.headers = {"Authorization": f"Bearer {token}", "X-Tenant-ID": tenant, "Content-Type": "application/json"}
def call(self, method: str, path: str, body: dict | None = None):
data = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(f"{self.base}{path}", data=data, headers=self.headers, method=method)
try:
with urllib.request.urlopen(request, timeout=60) as response:
text = response.read()
except urllib.error.HTTPError as error:
# The reply to a key request could only name an error here, but keep it short regardless.
raise SystemExit(f"{method} {path}: HTTP {error.code}: {error.read()[:300].decode(errors='replace')}")
return json.loads(text) if text else {}
def items(self, path: str) -> list:
data = self.call("GET", path)
return data if isinstance(data, list) else next((v for v in data.values() if isinstance(v, list)), [])
def token_seconds_left(token: str) -> int | None:
"""How long a KAB token lasts, read from its own expiry; None when that can't be read."""
try:
payload = token.split(".")[1]
return int(json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))["exp"] - time.time())
except (IndexError, KeyError, TypeError, ValueError):
return None
def kubectl(context: str, *arguments: str, stdin: bytes | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["kubectl", "--context", context, *arguments], input=stdin, capture_output=True, timeout=60)
def secret_exists(context: str, name: str) -> bool:
return kubectl(context, "-n", "agents", "get", "secret", name, "-o", "name").returncode == 0
def store_key(context: str, agent: str, key: str) -> bool:
manifest = {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": f"{agent}-secrets", "namespace": "agents"},
"type": "Opaque", "data": {"MCP_AGENT_API_KEY": base64.b64encode(key.encode()).decode()}}
return kubectl(context, "apply", "-f", "-", stdin=json.dumps(manifest).encode()).returncode == 0
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--apply", action="store_true", help="make the changes; without it, only show them")
parser.add_argument("--api", default=os.getenv("KAB_API", DEFAULT_API))
parser.add_argument("--tenant", default=CP1_TENANT, help="the KAB tenant the MCP servers are registered in")
parser.add_argument("--context", default="crucible-kab-dev-aks")
parser.add_argument("--expires-days", type=int, default=30, help="how long each agent key lasts (default 30)")
parser.add_argument("--agent", action="append", help="only these agents (default: the eight individual agents)")
args = parser.parse_args(argv)
path = Path.home() / ".kab_token"
token = os.getenv("KAB_TOKEN") or (path.read_text().strip() if path.exists() else "")
if not token:
raise SystemExit("No token: set KAB_TOKEN or write it to ~/.kab_token")
left = token_seconds_left(token)
if left is not None:
if left <= 0:
raise SystemExit(f"The token expired {-left} s ago. Reload KAB, copy a fresh access_token, and run this again.")
print(f"Token: {left} s left.")
gateway = Gateway(args.api, token, args.tenant)
registered = {server.get("name") for server in gateway.items("/servers?page=1&page_size=100")}
keys = {key.get("agent_name"): key for key in gateway.items("/agent-keys?page=1&page_size=100")}
missing_names = [name for name in LOGICAL_SERVERS if name not in registered]
need_key = [agent for agent in (args.agent or AGENTS) if not secret_exists(args.context, f"{agent}-secrets")]
orphans = [agent for agent in need_key if agent in keys]
print(f"Servers in the tenant: {len(registered)}. Logical names to register: {missing_names or 'none'}")
print(f"Agents needing a key: {need_key or 'none'}")
if orphans:
print(f"These have a gateway key but no Secret, and a key can't be read back: {orphans}. "
"Revoke them in KAB, then run this again.")
if not args.apply:
print("Plan only. Nothing changed. Add --apply to make these changes.")
return 1 if orphans else 0
if orphans:
return 1
if missing_names:
registration_key = os.getenv("MCP_REG_KEY")
if not registration_key:
raise SystemExit("Set MCP_REG_KEY to the sample server's key.")
for name in missing_names:
server_id = str(uuid.uuid4())
gateway.call("POST", "/servers", {"server_id": server_id, "name": name, "url": SAMPLE_URL,
"api_key": registration_key, "enabled": True, "max_connections": 10,
"timeout_seconds": 30, "description": "ALZ sample, logical name"})
found = gateway.call("POST", f"/tools/servers/{server_id}/discover", {})
print(f"Registered {name}: {found.get('discovered', found.get('count', 'tools discovered'))}")
given = []
for agent in need_key:
created = gateway.call("POST", "/agent-keys", {"agent_name": agent, "expires_in_days": args.expires_days,
"description": "CP1 Discovery, individual agents"})
if not store_key(args.context, agent, created.pop("api_key")):
raise SystemExit(f"{agent}: the key was created but not stored. Revoke key id {created.get('id')} in KAB.")
given.append(agent)
print(f"{agent}: key {created.get('key_prefix')}... stored, expires {created.get('expires_at')}")
for agent in given:
restarted = kubectl(args.context, "-n", "agents", "rollout", "restart", f"deploy/{agent}")
print(f"{agent}: {'restarted' if restarted.returncode == 0 else 'restart failed'}")
print("Done.")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -18,11 +18,14 @@ Usage (from the repository root; quote the Feature folder, its name has spaces):
python3 scripts/kab_import_workflow.py "sub-agents/ALZ-F01: Discovery Workflow Agent" --project-id <id> # validate only
python3 scripts/kab_import_workflow.py "<Feature folder>" --project-id <id> --apply # import as a draft
python3 scripts/kab_import_workflow.py "<Feature folder>" --project-id <id> --artifact-id <id> --apply # with an Artifacts step
python3 scripts/kab_import_workflow.py "<Feature folder>" --project-id <id> --apply --update <workflow id> \\
--rollback-file <file> # replace an imported workflow in place
"""
from __future__ import annotations
import argparse
import inspect
import importlib.util
import json
import os
@@ -32,6 +35,7 @@ import urllib.request
from pathlib import Path
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
CP1_TENANT = "6ae37d13999438461a529409"
def _token(path: Path) -> str:
@@ -57,10 +61,10 @@ def _builder(feature: Path):
return module.build
def _post(api: str, path: str, token: str, body: dict) -> dict:
def _post(api: str, path: str, token: str, body, method: str = "POST") -> dict:
request = urllib.request.Request(
f"{api.rstrip('/')}{path}", data=json.dumps(body).encode("utf-8"), method="POST",
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"})
f"{api.rstrip('/')}{path}", data=None if body is None else json.dumps(body).encode("utf-8"), method=method,
headers={"Authorization": f"Bearer {token}", "X-Tenant-ID": CP1_TENANT, "Content-Type": "application/json"})
try:
with urllib.request.urlopen(request, timeout=60) as response:
return json.loads(response.read() or b"{}")
@@ -71,6 +75,32 @@ def _post(api: str, path: str, token: str, body: dict) -> dict:
raise SystemExit(f"{path}: HTTP {error.code} {hint}\n{detail}")
def _step_key(step: dict) -> str:
"""A step's agent, with the two Format for UI steps told apart by what they format."""
sources = list(((step.get("inputs") or {}).get("from_steps") or {}).values())
return step.get("agent", "") + (":" + sources[0].split(".", 1)[-1] if step.get("agent") == "format_for_ui" and sources else "")
def _keep_ids(new: dict, current: dict) -> dict:
"""The new template with the step ids KAB gave the current one on import, matched by agent, so the stored
workflow keeps the ids its runs and history use. A step with no counterpart keeps the export's id."""
ids = {_step_key(s): s["id"] for s in current.get("steps") or []}
rename = {s["id"]: ids.get(_step_key(s), s["id"]) for s in new["steps"]}
if len(set(rename.values())) != len(rename):
raise SystemExit("--update: the current workflow's steps can't be matched one to one; import it instead")
ref = lambda value: rename.get(value.split(".")[0], value.split(".")[0]) + "." + value.split(".", 1)[1]
out = json.loads(json.dumps(new))
for step in out["steps"]:
step["id"] = rename[step["id"]]
step["depends_on"] = [rename.get(d, d) for d in step.get("depends_on") or []]
inputs = step.get("inputs") or {}
if inputs.get("from_steps"):
inputs["from_steps"] = {k: ref(v) for k, v in inputs["from_steps"].items()}
out["connections"] = [{"source": rename.get(c["source"], c["source"]), "target": rename.get(c["target"], c["target"])}
for c in out.get("connections") or []]
return out
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("feature", type=Path, help="the Feature folder whose workflow is imported")
@@ -83,9 +113,18 @@ def main(argv=None) -> int:
parser.add_argument("--published", action="store_true", help="import as published (default: draft)")
parser.add_argument("--apply", action="store_true", help="import; without it the export is only validated")
parser.add_argument("--save", type=Path, help="also write the export to this file")
parser.add_argument("--update", metavar="WORKFLOW_ID",
help="with --apply: replace this existing workflow's template in place (its id is kept) "
"instead of importing a new workflow")
parser.add_argument("--rollback-file", type=Path,
help="with --update: where the workflow's current template is saved first (required)")
args = parser.parse_args(argv)
export = _builder(args.feature)(not args.no_upload, args.artifact_id)
build = _builder(args.feature)
# Discovery's build takes (with_upload, artifact_ids); Assessment's, whose upload is always the rate card, takes
# only artifact_ids.
params = inspect.signature(build).parameters
export = build(not args.no_upload, args.artifact_id) if len(params) >= 2 else build(args.artifact_id)
if args.save:
args.save.write_text(json.dumps(export, indent=2) + "\n")
steps = export["template"]["steps"]
@@ -107,6 +146,20 @@ def main(argv=None) -> int:
print("validated only; add --apply to import")
return 0
if args.update:
if not args.rollback_file:
raise SystemExit("--update needs --rollback-file: the current template is saved there before it is replaced")
path = f"/api/v1/workflow-catalog/workflows/{args.update}"
current = _post(args.api, path, token, None, "GET")
current = current.get("workflow", current)
args.rollback_file.write_text(json.dumps({"workflow_id": args.update, "template": current.get("template")},
indent=2) + "\n")
print(f"saved the current template to {args.rollback_file}")
template = _keep_ids(export["template"], current.get("template") or {})
result = _post(args.api, path, token, {"template": template}, "PUT")
print(f"updated: workflow_id={args.update} success={result.get('success')}")
return 0 if result.get("success") else 1
mappings = [{"source_name": a["name"], "target_url": a["source_url"]} for a in export["dependencies"]["agents"]]
result = _post(args.api, "/api/workflows/import", token, {
"export_data": export, "agent_mappings": mappings, "import_as_draft": not args.published,

View File

@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Upload an approved Discovery report to a KAB project, as a file artifact.
A workflow step's output is always registered as output.json, so the customer's page reaches KAB this way, after the
Engagement Lead has approved the run. The `file` type is used because `report` artifacts need Azure Blob, S3 or MongoDB
storage through a route that has no blob store on crucible-kab-dev; `file` artifacts stay on the shared volume. Render the clean copy first with the Feature's scripts/render_discovery_report.py.
Authentication: your KAB session's access token, from --token-file (default ~/.kab_token). It is sent only as the
Authorization header and never printed. Uploading needs the Editor role in the project. Standard library only.
Usage (from the repository root):
python3 scripts/kab_publish_report.py <report.html> --project <id> --name <name> # shows what it would do
python3 scripts/kab_publish_report.py <report.html> --project <id> --name <name> --apply
"""
from __future__ import annotations
import argparse
import json
import sys
import urllib.error
import urllib.request
import uuid
from pathlib import Path
from kab_run_workflow import CP1_TENANT, DEFAULT_API
TYPES = {".html": "text/html", ".md": "text/markdown"}
def multipart(fields: dict, path: Path, boundary: str) -> bytes:
parts = [f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode() for k, v in fields.items()]
head = (f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{path.name}"\r\n'
f"Content-Type: {TYPES[path.suffix]}\r\n\r\n").encode()
return b"".join(parts) + head + path.read_bytes() + f"\r\n--{boundary}--\r\n".encode()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("report", type=Path)
parser.add_argument("--project", required=True)
parser.add_argument("--name", required=True)
parser.add_argument("--description", default="Discovery report")
parser.add_argument("--api", default=DEFAULT_API)
parser.add_argument("--tenant", default=CP1_TENANT)
parser.add_argument("--token-file", type=Path, default=Path.home() / ".kab_token")
parser.add_argument("--apply", action="store_true", help="upload; without it nothing is sent")
args = parser.parse_args()
if args.report.suffix not in TYPES:
raise SystemExit("the report must be a .html or .md file")
text = args.report.read_text()
if "Draft. Not for release" in text:
raise SystemExit("this copy is marked as a draft: render a clean one from the approved run")
print(f"{args.report.name}: {len(text) // 1024} KB, to project {args.project} as {args.name!r}")
if not args.apply:
print("Nothing sent. Add --apply to upload.")
return 0
boundary = uuid.uuid4().hex
body = multipart({"name": args.name, "type_id": "file", "description": args.description}, args.report, boundary)
request = urllib.request.Request(f"{args.api}/api/v1/projects/{args.project}/artifacts/upload", data=body, method="POST", headers={
"Authorization": f"Bearer {args.token_file.read_text().strip()}", "X-Tenant-ID": args.tenant, "X-Project-ID": args.project,
"Content-Type": f"multipart/form-data; boundary={boundary}"})
try:
with urllib.request.urlopen(request, timeout=60) as response:
print("Uploaded:", json.load(response).get("id"))
except urllib.error.HTTPError as error:
print(f"HTTP {error.code}: {error.read().decode(errors='replace')[:300]}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

216
scripts/kab_run_workflow.py Normal file
View File

@@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""Run a saved KAB workflow from the command line: the same API the KAB app uses.
kab_run_workflow.py start --workflow <id> --project <id> --context run-context.yaml [--dry-run]
kab_run_workflow.py start --workflow <id> --project <id> --input engagement_id=ENG-1 --input run_id=RUN-1 \\
--file brief.txt [--dry-run]
A run is normally started from a YAML context file (--context): its `workflow_inputs:` become the run's inputs, and
the whole file is uploaded to the workflow's upload step, so the agents also read its `context:` section. Files its
`uploads:` list names (beside it, e.g. a CMDB export) are uploaded after it, in order.
kab_run_workflow.py status <execution id>
kab_run_workflow.py review <execution id> continue|stop|rerun [--note "..."]
kab_run_workflow.py outputs <execution id> --out <folder>
`review` answers the pause the run is waiting at. It's a person's decision, so it is never made automatically.
The token comes from $KAB_TOKEN or ~/.kab_token, and lasts five minutes: scripts/kab_token.py refreshes it. A run
doesn't need the token once started.
"""
from __future__ import annotations
import argparse
import base64
import secrets
import json
from datetime import datetime, timezone
import os
import sys
import urllib.error
import urllib.request
from pathlib import Path
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
CP1_TENANT = "6ae37d13999438461a529409"
class Kab:
def __init__(self, api: str, tenant: str, project: str | None = None) -> None:
path = Path.home() / ".kab_token"
token = os.getenv("KAB_TOKEN") or (path.read_text().strip() if path.exists() else "")
if not token:
raise SystemExit("No token: run scripts/kab_token.py")
self.api = api.rstrip("/")
self.headers = {"Authorization": f"Bearer {token}", "X-Tenant-ID": tenant, "Content-Type": "application/json"}
if project:
self.headers["X-Project-ID"] = project
def call(self, method: str, path: str, body: dict | None = None):
data = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(f"{self.api}{path}", data=data, headers=self.headers, method=method)
try:
with urllib.request.urlopen(request, timeout=120) as response:
return json.load(response)
except urllib.error.HTTPError as error:
raise SystemExit(f"{method} {path}: HTTP {error.code}: {error.read()[:500].decode(errors='replace')}")
def start(kab: Kab, args) -> int:
found = kab.call("GET", f"/api/v1/workflow-catalog/workflows/{args.workflow}")
workflow = found.get("workflow", found)
template = workflow.get("template") or workflow
sample = kab.call("GET", f"/api/workflows/{args.workflow}/api-sample")
# Flat: the API applies step-keyed inputs to the first step only, and every step's from_workflow reads these.
# A value written as a JSON list is a list input, e.g. --input 'artifact_ids=["6aec..."]'; `@path` reads the value
# from a file, e.g. --input rate_card_document=@rate-card.json.
def value(v: str):
if v.startswith("@"):
return Path(v[1:]).read_text()
return json.loads(v) if v.lstrip().startswith("[") else v
inputs: dict = {k: value(v) for k, v in (item.split("=", 1) for item in args.input)}
if args.context:
if args.file:
raise SystemExit("Use --context or --file, not both: the upload field takes one file.")
given = load_context(Path(args.context))
clash = sorted(set(given) & set(inputs))
if clash:
raise SystemExit(f"{', '.join(clash)} given both in {args.context} and with --input: keep one.")
if "run_id" in given or "run_id" in inputs:
raise SystemExit("run_id is generated for every run: remove it from the context file and --input.")
inputs.update(given)
generated = new_ids()
inputs["run_id"] = generated["run_id"]
args.upload = ("# Added by kab_run_workflow.py when the run started. Never edit these by hand.\n"
f"generated:\n" + "".join(f" {k}: {v}\n" for k, v in generated.items()) + "\n"
+ Path(args.context).read_text()).encode()
args.file = args.context
args.uploads = context_uploads(Path(args.context))
print(f"Generated: {json.dumps(generated)}")
missing = sorted(f"{i['step_id']}.{i['name']}" for i in sample.get("required", [])
if i["name"] != "file" and i["name"] not in inputs)
if missing:
raise SystemExit(f"Missing inputs: {', '.join(missing)}. Pass them with --input name=value.")
if args.file:
path = Path(args.file)
content = base64.b64encode(getattr(args, "upload", None) or path.read_bytes()).decode()
kind = upload_type(path)
inputs["file"] = [{"name": path.name, "content": f"data:{kind};base64,{content}"}]
for extra in getattr(args, "uploads", []):
data = base64.b64encode(extra.read_bytes()).decode()
inputs["file"].append({"name": extra.name, "content": f"data:{upload_type(extra)};base64,{data}"})
if len(inputs["file"]) > 1:
print(f"Uploads: {', '.join(f['name'] for f in inputs['file'])}")
shown = {k: "<file>" if k == "file" else (f"<{len(v)} characters>" if isinstance(v, str) and len(v) > 200 else v)
for k, v in inputs.items()}
print(f"Workflow: {template.get('name')}\nInputs: {json.dumps(shown)}")
if args.dry_run:
print("Dry run. Nothing started.")
return 0
result = kab.call("POST", "/api/workflows/execute", {"template": template, "inputs": inputs})
print(f"Started: execution_id={result.get('execution_id')} status={result.get('status')}")
return 0
def upload_type(path: Path) -> str:
return {".csv": "text/csv", ".json": "application/json", ".yaml": "application/x-yaml", ".yml": "application/x-yaml",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}.get(path.suffix, "text/plain")
def context_uploads(path: Path) -> list:
"""The files a context file's `uploads:` lists, beside it, uploaded after it in order (e.g. a CMDB export)."""
import yaml
listed = (yaml.safe_load(path.read_text()) or {}).get("uploads") or []
files = []
for entry in listed:
name = entry.get("name") if isinstance(entry, dict) else entry
extra = (path.parent / str(name)).resolve()
if extra.parent != path.parent.resolve() or not extra.is_file():
raise SystemExit(f"{path}: uploads entry {name!r} must be a file in the same folder as the context file")
files.append(extra)
return files
def new_ids() -> dict:
"""A run's unique identifiers: UTC time to the second, then 6 random hex digits, so two starts never collide."""
now = datetime.now(timezone.utc)
run_id = f"RUN-{now:%Y%m%dT%H%M%SZ}-{secrets.token_hex(3)}"
return {"run_id": run_id, "plan_id": f"PLAN-{run_id}", "started_at": f"{now:%Y-%m-%dT%H:%M:%SZ}"}
def load_context(path: Path) -> dict:
"""The run inputs from a YAML context file's `workflow_inputs:`, as the strings the workflow expects."""
try:
import yaml
except ImportError:
raise SystemExit("--context needs PyYAML (pip install pyyaml)")
doc = yaml.safe_load(path.read_text())
if not isinstance(doc, dict) or not isinstance(doc.get("workflow_inputs"), dict):
raise SystemExit(f"{path}: expected a `workflow_inputs:` mapping (and, optionally, `context:`)")
out = {}
for key, value in doc["workflow_inputs"].items():
if isinstance(value, bool):
value = "yes" if value else "no"
if not isinstance(value, (str, int, float)):
raise SystemExit(f"{path}: workflow_inputs.{key} must be a single value")
out[str(key)] = str(value)
return out
def status(kab: Kab, args) -> int:
run = kab.call("GET", f"/api/workflows/status/{args.execution}")
print(json.dumps({k: v for k, v in run.items() if k not in ("result", "inputs", "outputs", "step_results")}, default=str)[:1200])
reviews = kab.call("GET", f"/api/workflows/executions/{args.execution}/reviews")
print("Reviews:", json.dumps(reviews, default=str)[:2500])
return 0
def review(kab: Kab, args) -> int:
reviews = kab.call("GET", f"/api/workflows/executions/{args.execution}/reviews")
pending = reviews.get("pending") or []
if len(pending) != 1:
raise SystemExit(f"Expected one pause, found {len(pending)}: {json.dumps(pending)[:400]}")
checkpoint = pending[0]["checkpoint_id"]
result = kab.call("POST", f"/api/workflows/executions/{args.execution}/reviews/{checkpoint}",
{"decision": args.decision, "note": args.note})
print(json.dumps(result, default=str)[:600])
return 0
def outputs(kab: Kab, args) -> int:
folder = Path(args.out)
folder.mkdir(parents=True, exist_ok=True)
found = kab.call("GET", f"/api/workflows/executions/{args.execution}/steps")
(folder / "steps.json").write_text(json.dumps(found, indent=1, default=str))
print(f"Wrote {folder / 'steps.json'}")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--api", default=DEFAULT_API)
parser.add_argument("--tenant", default=CP1_TENANT)
commands = parser.add_subparsers(dest="command", required=True)
run = commands.add_parser("start")
run.add_argument("--workflow", required=True)
run.add_argument("--project", required=True)
run.add_argument("--input", action="append", default=[], metavar="name=value")
run.add_argument("--file")
run.add_argument("--context", help="a YAML run context file: workflow_inputs become the run inputs; the file is uploaded")
run.add_argument("--dry-run", action="store_true")
run.set_defaults(handler=start, execution=None)
for name, handler in (("status", status), ("outputs", outputs), ("review", review)):
command = commands.add_parser(name)
command.add_argument("execution")
command.set_defaults(handler=handler)
if name == "review":
command.add_argument("decision", choices=["continue", "stop", "rerun"])
command.add_argument("--note", default="")
if name == "outputs":
command.add_argument("--out", required=True)
args = parser.parse_args()
kab = Kab(args.api, args.tenant, getattr(args, "project", None))
return args.handler(kab, args)
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,239 @@
#!/usr/bin/env python3
"""Store the Azure and Google Cloud connections for the discovery MCP servers in KAB's vault, per project.
For each server asked for, this:
1. finds the server in the MCP gateway by name and reads the connection fields it declares;
2. builds the connection from the credential source you choose, and checks it carries every required field;
3. tests it with the gateway's `connections:verify` before anything is stored;
4. with --apply, stores it as an `mcp` / `connection` secret for that server and project
(create-only, `If-None-Match: *`; --replace swaps an existing one, `If-Match: *`).
Without --apply it stops after the test. Credential values never appear in the output, an argument or a file: they
come from environment variables, a key file, or the `az` / `gcloud` command line, and go only into the request body.
Discovery connections are stored per project, never tenant-wide (mcp-servers/README.md), so --project-id is
required. Well-Architected guidance comes from the same servers (`<provider>_search_well_architected`), on the
same connection, so it needs nothing stored separately.
Credential sources:
Azure --azure-credential service-principal AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET from the environment
--azure-credential cli-token `az account get-access-token` (about an hour; your own rights)
GCP --gcp-credential key-file:<path> a service account's JSON key
--gcp-credential gcloud-token `gcloud auth print-access-token` (about an hour; your own rights)
--gcp-credential adc-token `gcloud auth application-default print-access-token` (the same, from
`gcloud auth application-default login`)
The KAB token comes from $KAB_TOKEN, else --token-file (default ~/.kab_token). Tokens last minutes, so refresh it just
before running. API keys (X-API-Key) don't work here: the platform accepts them only on /api/external/ routes.
Usage (from the repository root):
python3 scripts/kab_store_connections.py --tenant <id> --project-id <id> \\
--azure-scope-root subscriptions/<id> --azure-credential service-principal \\
--gcp-scope-root projects/<id> --gcp-credential key-file:/path/outside/the/repo.json # plan + test
python3 scripts/kab_store_connections.py ... --apply # store
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
DEFAULT_API = "https://api.kab-dev.kyndemo.live"
SERVERS = {"azure": "azure-discovery", "gcp": "gcp-discovery"}
class Kab:
def __init__(self, api: str, token: str, tenant: str) -> None:
self.api = api.rstrip("/")
self.headers = {"Authorization": f"Bearer {token}", "X-Tenant-ID": tenant, "Content-Type": "application/json"}
def call(self, method: str, path: str, body: dict | None = None, headers: dict | None = None,
gateway: bool = False) -> tuple[int, dict]:
base = f"{self.api}/api/v1/mcp-gateway/api/v1/admin" if gateway else f"{self.api}/api/v1"
data = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(f"{base}{path}", data=data, headers={**self.headers, **(headers or {})},
method=method)
try:
with urllib.request.urlopen(request, timeout=60) as response:
text = response.read()
return response.status, (json.loads(text) if text else {})
except urllib.error.HTTPError as error:
text = error.read()[:2000]
try:
return error.code, json.loads(text)
except ValueError:
return error.code, {"detail": text.decode(errors="replace")[:300]}
def token_seconds_left(token: str) -> int | None:
try:
payload = token.split(".")[1]
return int(json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))["exp"] - time.time())
except (IndexError, KeyError, TypeError, ValueError):
return None
def short_error(answer: dict) -> str:
"""An error's code and message only. Never the whole answer, in case a service echoes a field."""
detail = answer.get("detail", answer)
if isinstance(detail, dict):
return " ".join(str(detail.get(k)) for k in ("code", "message") if detail.get(k))[:300]
return str(detail)[:300]
def run_quiet(command: list[str]) -> str:
"""A command's stdout, kept in memory; its output is never echoed."""
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
if result.returncode != 0:
raise SystemExit(f"`{command[0]} {command[1]}` failed (exit {result.returncode}); check you are signed in")
return result.stdout.strip()
def azure_connection(scope_root: str, source: str, cloud: str | None) -> dict:
connection = {"scope_root": scope_root}
if cloud:
connection["cloud"] = cloud
if source == "service-principal":
names = {"tenant_id": "AZURE_TENANT_ID", "client_id": "AZURE_CLIENT_ID", "client_secret": "AZURE_CLIENT_SECRET"}
missing = [env for env in names.values() if not os.environ.get(env)]
if missing:
raise SystemExit("set " + ", ".join(missing) + " in the environment for --azure-credential service-principal")
connection.update({field: os.environ[env] for field, env in names.items()})
elif source == "cli-token":
connection["access_token"] = run_quiet(["az", "account", "get-access-token", "--resource",
"https://management.azure.com/", "--query", "accessToken", "-o", "tsv"])
else:
raise SystemExit("--azure-credential is service-principal or cli-token")
return connection
def gcp_connection(scope_root: str, source: str) -> dict:
connection = {"scope_root": scope_root}
if source.startswith("key-file:"):
path = Path(source.split(":", 1)[1]).expanduser().resolve()
repo = Path(__file__).resolve().parents[1]
if repo in path.parents:
raise SystemExit("keep the key file outside the repository")
text = path.read_text()
key = json.loads(text)
if key.get("type") != "service_account":
raise SystemExit("the key file is not a service account key")
connection["service_account_json"] = text
elif source == "gcloud-token":
connection["access_token"] = run_quiet(["gcloud", "auth", "print-access-token"])
elif source == "adc-token":
connection["access_token"] = run_quiet(["gcloud", "auth", "application-default", "print-access-token"])
else:
raise SystemExit("--gcp-credential is key-file:<path>, gcloud-token or adc-token")
return connection
def find_server(kab: Kab, name: str) -> dict:
status, answer = kab.call("GET", "/servers?page=1&page_size=100", gateway=True)
if status != 200:
raise SystemExit(f"listing MCP servers: HTTP {status} {short_error(answer)}")
servers = answer if isinstance(answer, list) else next((v for v in answer.values() if isinstance(v, list)), [])
hits = [s for s in servers if name in (s.get("name"), s.get("server_id"), s.get("slug"))]
if len(hits) != 1:
known = ", ".join(sorted(str(s.get("name")) for s in servers)) or "none"
raise SystemExit(f"{name}: {'not registered' if not hits else 'registered more than once'} on this platform "
f"(servers: {known})")
return hits[0]
def store(kab: Kab, provider: str, connection: dict, project_id: str, apply: bool, replace: bool) -> bool:
assert project_id, "discovery connections are never tenant-wide"
name = SERVERS[provider]
server = find_server(kab, name)
server_id = str(server.get("id") or server.get("server_id"))
print(f"{name}: registered as {server_id}, status {server.get('status') or server.get('health_status') or 'unknown'}")
status, schema = kab.call("GET", f"/connections/schema?mcpServerId={urllib.parse.quote(server_id)}", gateway=True)
if status == 200:
schema = schema.get("schema", schema)
required = set(schema.get("required") or [])
known = set((schema.get("properties") or {}).keys())
missing, unknown = sorted(required - connection.keys()), sorted(connection.keys() - known) if known else []
if missing or unknown:
print(f" connection doesn't match the server's fields: missing {missing or 'none'}, unknown {unknown or 'none'}")
return False
print(f" fields sent: {', '.join(sorted(connection))} (values not shown)")
else:
print(f" couldn't read the server's connection schema (HTTP {status}); continuing to the test")
status, verdict = kab.call("POST", "/connections:verify", {"mcpServerId": server_id, "connection": connection,
"projectId": project_id}, gateway=True)
ok = status == 200 and verdict.get("ok") is True
message = verdict.get("message") or short_error(verdict)
print(f" test: {'passed' if ok else 'FAILED'} (HTTP {status}{', ' + str(message)[:200] if message else ''})")
if not ok:
return False
if not apply:
print(" not stored: run again with --apply")
return True
expires = {"noExpiry": True}
if "access_token" in connection:
print(" note: an access token expires within about an hour; store a service principal or key for anything longer")
body = {"secretType": "mcp", "format": "connection", "displayName": f"{name} ({project_id})",
"value": connection, "mcpServerId": server_id, "projectId": project_id, **expires}
status, answer = kab.call("PUT", "/secrets", body, headers={"If-Match": "*"} if replace else {"If-None-Match": "*"})
if 200 <= status < 300:
print(f" stored: {answer.get('providerName') or 'ok'}, version {answer.get('version', '?')}")
return True
print(f" NOT stored: HTTP {status} {short_error(answer)}"
+ (" (it exists already: use --replace)" if "ALREADY_EXISTS" in str(answer) else ""))
return False
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--api", default=os.getenv("KAB_API", DEFAULT_API))
parser.add_argument("--tenant", required=True, help="the KAB tenant the servers are registered in")
parser.add_argument("--project-id", help="the KAB project; required for discovery connections, which are never tenant-wide")
parser.add_argument("--azure-scope-root", help="subscriptions/<id>, managementGroups/<id>, or one resource group: subscriptions/<id>/resourceGroups/<name>")
parser.add_argument("--azure-credential", help="service-principal or cli-token")
parser.add_argument("--azure-cloud", help="optional: the Azure cloud, if not public")
parser.add_argument("--gcp-scope-root", help="projects/<id>, folders/<id> or organizations/<id>")
parser.add_argument("--gcp-credential", help="key-file:<path>, gcloud-token or adc-token")
parser.add_argument("--token-file", type=Path, default=Path.home() / ".kab_token")
parser.add_argument("--apply", action="store_true", help="store the connections; without it, only test them")
parser.add_argument("--replace", action="store_true", help="replace an existing connection instead of creating one")
args = parser.parse_args(argv)
token = os.getenv("KAB_TOKEN") or (args.token_file.read_text().strip() if args.token_file.exists() else "")
if not token:
raise SystemExit(f"no token: set KAB_TOKEN or write one to {args.token_file}")
left = token_seconds_left(token)
if left is not None and left < 60:
raise SystemExit(f"the KAB token {'expired' if left <= 0 else 'expires in under a minute'}: refresh it")
kab = Kab(args.api, token, args.tenant)
jobs = []
if args.azure_scope_root:
jobs.append(("azure", azure_connection(args.azure_scope_root, args.azure_credential or "", args.azure_cloud)))
if args.gcp_scope_root:
jobs.append(("gcp", gcp_connection(args.gcp_scope_root, args.gcp_credential or "")))
if jobs and not args.project_id:
parser.error("--project-id is required for discovery connections: they are never stored tenant-wide")
if not jobs:
parser.error("give --azure-scope-root and/or --gcp-scope-root")
results = [store(kab, provider, connection, args.project_id, args.apply, args.replace)
for provider, connection in jobs]
for _, connection in jobs: # drop the values as soon as they're sent
connection.clear()
return 0 if all(results) else 1
if __name__ == "__main__":
sys.exit(main())

42
scripts/kab_token.py Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Turn a KAB refresh token into a fresh access token, in ~/.kab_token.
Paste the browser's refresh_token at the prompt. It isn't echoed, and it isn't stored: it's a stronger credential than
the access token, so it stays out of files and shell history. Run this just before a script that needs a token.
Usage: python3 scripts/kab_token.py
"""
import getpass
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
TOKEN_URL = "https://id.kab-dev.kyndemo.live/realms/kab/protocol/openid-connect/token"
CLIENT = "kab-ui"
def main() -> int:
refresh = getpass.getpass("KAB refresh_token: ").strip()
body = urllib.parse.urlencode({"grant_type": "refresh_token", "client_id": CLIENT, "refresh_token": refresh}).encode()
try:
with urllib.request.urlopen(urllib.request.Request(TOKEN_URL, data=body), timeout=30) as response:
reply = json.load(response)
except urllib.error.HTTPError as error:
print(f"Refused: HTTP {error.code}: {json.loads(error.read() or b'{}').get('error_description', 'no reason given')}")
return 1
path = Path.home() / ".kab_token"
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "w") as out:
out.write(reply["access_token"])
print(f"Wrote {path}. It lasts {reply.get('expires_in')} s, until {time.strftime('%H:%M:%S', time.localtime(time.time() + reply.get('expires_in', 0)))}.")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -20,7 +20,7 @@ from pathlib import Path
import httpx
ROOT = Path(__file__).resolve().parents[1] / "mcp-servers"
SERVERS = {"azure-discovery": 8111, "aws-discovery": 8112, "gcp-discovery": 8113, "cloud-knowledge": 8114}
SERVERS = {"azure-discovery": 8111, "aws-discovery": 8112, "gcp-discovery": 8113}
def pytest(env: dict, *paths: str) -> int:
@@ -54,7 +54,7 @@ def main(argv=None) -> int:
env = {**os.environ, "SERVER_DIR": name}
own = sorted(str(p.relative_to(ROOT)) for p in (ROOT / name / "tests").glob("test_*.py")
if p.name != "test_conformance.py")
if pytest(env, "_common/test_contract.py", *own) or conformance(name, SERVERS[name]):
if pytest(env, "_common/test_contract.py", "_common/test_knowledge.py", *own) or conformance(name, SERVERS[name]):
failed.append(name)
print("failed: " + ", ".join(failed) if failed else "all servers passed")
return 1 if failed else 0