- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with --input name=@file), store per-project discovery connections, bootstrap agent MCP keys, refresh the KAB token, publish reports and build estate snapshots. - Hyperscaler discovery servers extended in place; cloud-knowledge folded into them. - deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents. - Platform issue write-ups and current-wiring notes under docs/platform-issues/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
188 lines
8.4 KiB
Python
188 lines
8.4 KiB
Python
# Shared by every hyperscaler server. scripts/render_mcp_server.py inlines this file into a server through the
|
|
# definition's shared_code, because a rendered image holds only the template's files. The builder accepts plain
|
|
# functions and literal constants here, nothing that runs at import.
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
|
|
DEFAULT_PAGE_SIZE = 50
|
|
MAX_PAGE_SIZE = 200
|
|
# A reply goes into a model's context, which is the real limit on a hand-off, so a page is cut at this size.
|
|
MAX_RESPONSE_BYTES = 200000
|
|
# A throttled call is retried inside the budget, then fails and says so.
|
|
RETRY_ATTEMPTS = 5
|
|
RETRY_MAX_WAIT_SECONDS = 20
|
|
RETRY_BUDGET_SECONDS = 60
|
|
|
|
# Values under these keys are never returned, in any casing.
|
|
SECRET_KEYS = (
|
|
"userdata", "customdata", "customscript", "protectedsettings", "connectionstring", "connectionstrings",
|
|
"appsettings", "env", "envvars", "environmentvariables", "variables", "password", "adminpassword", "secret",
|
|
"secretvalue", "sastoken", "accesskey", "primarykey", "secondarykey", "privatekey", "kubeconfig", "credentials",
|
|
)
|
|
# A single value under one of these keys is a tag or label (env=dev), not a secret: only a list or map under it (a
|
|
# container's environment variables) is redacted.
|
|
SCALAR_SAFE_KEYS = ("env",)
|
|
# Values under these keys name a person, or may. A principal of unknown type counts as a person.
|
|
PERSON_KEYS = (
|
|
"createdby", "lastmodifiedby", "updatedby", "creator", "owner", "owneremail", "contact", "contactemail", "email",
|
|
"objectid", "principalid", "principalname", "userprincipalname",
|
|
)
|
|
PRINCIPAL_KINDS = {"user": "[user]", "group": "[group]", "domain": "[domain]", "principal": "[user]",
|
|
"principalSet": "[group]", "deleted": "[user]"}
|
|
NON_PERSON_EMAIL_DOMAINS = ("gserviceaccount.com",)
|
|
|
|
|
|
def _clean_text(text: str) -> str:
|
|
"""Anonymise people inside a string: emails, AWS IAM users, and session names, which are often emails."""
|
|
def email(match):
|
|
return match.group(0) if match.group(0).lower().endswith(NON_PERSON_EMAIL_DOMAINS) else "[email]"
|
|
|
|
text = re.sub(r"arn:aws[a-z-]*:iam::\d{12}:user/[^\s\"',]+", "[user]", text)
|
|
text = re.sub(r"(arn:aws[a-z-]*:sts::\d{12}:assumed-role/[^/\s\"',]+)/[^\s\"',]+", r"\1/[session]", text)
|
|
# Bounded, and skipped without an @: an unbounded pattern takes minutes on one long string.
|
|
if "@" not in text:
|
|
return text
|
|
return re.sub(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]{1,255}\.[A-Za-z]{2,24}", email, text)
|
|
|
|
|
|
def _anonymise_member(value: str) -> str:
|
|
"""One IAM member, such as user:a@b.example. Service accounts stay: Discovery shows what can reach what."""
|
|
kind = value.split(":", 1)[0]
|
|
if ":" in value and kind == "serviceAccount":
|
|
return value
|
|
if ":" in value and kind in PRINCIPAL_KINDS:
|
|
return PRINCIPAL_KINDS[kind]
|
|
return _clean_text(value)
|
|
|
|
|
|
def _scalar(value) -> bool:
|
|
return value is None or isinstance(value, (str, bool, int, float))
|
|
|
|
|
|
def _scrub(value, key: str = ""):
|
|
"""Drop secrets and anonymise people, however deep. Every tool's reply passes through here."""
|
|
lowered = key.lower().replace("_", "").replace("-", "")
|
|
if lowered in SECRET_KEYS and not (lowered in SCALAR_SAFE_KEYS and _scalar(value)):
|
|
return "[redacted]"
|
|
if lowered in PERSON_KEYS:
|
|
return "[user]"
|
|
if isinstance(value, dict):
|
|
tag = str(value.get("Key", value.get("key", ""))).lower().replace("_", "").replace("-", "")
|
|
cleaned = {k: _scrub(v, str(k)) for k, v in value.items()}
|
|
for name in ("Value", "value"):
|
|
if name in cleaned and tag in PERSON_KEYS:
|
|
cleaned[name] = "[user]"
|
|
elif name in cleaned and tag in SECRET_KEYS and not (tag in SCALAR_SAFE_KEYS and _scalar(cleaned[name])):
|
|
cleaned[name] = "[redacted]"
|
|
return cleaned
|
|
if isinstance(value, (list, tuple)):
|
|
return [_scrub(v, "members" if lowered == "members" else "") for v in value]
|
|
if isinstance(value, str):
|
|
return _anonymise_member(value) if lowered == "members" else _clean_text(value)
|
|
if value is None or isinstance(value, (bool, int, float)):
|
|
return value
|
|
return str(value)
|
|
|
|
|
|
def _retry_delay(response, attempt: int) -> float:
|
|
"""How long the provider asked us to wait: Retry-After, or Resource Graph's hh:mm:ss quota reset."""
|
|
reset = response.headers.get("x-ms-user-quota-resets-after", "")
|
|
if reset.count(":") == 2:
|
|
hours, minutes, seconds = reset.split(":")
|
|
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
|
try:
|
|
return float(response.headers.get("retry-after", ""))
|
|
except ValueError:
|
|
return float(2 ** attempt)
|
|
|
|
|
|
async def _with_backoff(send):
|
|
"""Send, waiting when the provider throttles. The waits stay well inside the 120-second limit on an MCP call."""
|
|
waited = 0.0
|
|
response = await send()
|
|
for attempt in range(RETRY_ATTEMPTS):
|
|
if response.status_code not in (429, 503):
|
|
return response
|
|
delay = min(_retry_delay(response, attempt), RETRY_MAX_WAIT_SECONDS)
|
|
if waited + delay > RETRY_BUDGET_SECONDS:
|
|
return response
|
|
logger.info("upstream_throttled", status=response.status_code, wait_seconds=delay)
|
|
await asyncio.sleep(delay)
|
|
waited += delay
|
|
response = await send()
|
|
return response
|
|
|
|
|
|
def _page_size(arguments: Dict[str, Any]) -> int:
|
|
try:
|
|
size = int(arguments.get("page_size") or DEFAULT_PAGE_SIZE)
|
|
except (TypeError, ValueError):
|
|
return DEFAULT_PAGE_SIZE
|
|
return max(1, min(size, MAX_PAGE_SIZE))
|
|
|
|
|
|
def _read_page(arguments: Dict[str, Any]):
|
|
"""(upstream token, items of that upstream page already returned)."""
|
|
token = arguments.get("page_token")
|
|
if not token:
|
|
return None, 0
|
|
state = json.loads(base64.urlsafe_b64decode(str(token).encode()).decode())
|
|
return state["u"], int(state["s"])
|
|
|
|
|
|
def _write_page(upstream, skip: int) -> str:
|
|
return base64.urlsafe_b64encode(json.dumps({"u": upstream, "s": skip}).encode()).decode()
|
|
|
|
|
|
def _evidence_ref(tool: str, arguments: Dict[str, Any], payload) -> str:
|
|
asked = {k: v for k, v in arguments.items() if k != "page_token"}
|
|
blob = json.dumps([tool, asked, payload], sort_keys=True, default=str)
|
|
return "sha256:" + hashlib.sha256(blob.encode()).hexdigest()
|
|
|
|
|
|
def _collected_at() -> str:
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
def _refuse(code: str, message: str) -> Dict[str, Any]:
|
|
return {"success": False, "error": {"code": code, "message": message}}
|
|
|
|
|
|
def _upstream_failed(provider: str, error: Exception) -> Dict[str, Any]:
|
|
"""Name the failure and never its text: an upstream error can quote a request or an identity."""
|
|
status = getattr(getattr(error, "response", None), "status_code", None)
|
|
code = getattr(error, "response", None)
|
|
if isinstance(code, dict):
|
|
status = code.get("Error", {}).get("Code")
|
|
logger.warning("upstream_call_failed", provider=provider, error_type=type(error).__name__, status=status)
|
|
detail = f" ({status})" if status else ""
|
|
return _refuse("upstream_failed", f"{provider} refused or failed the call: {type(error).__name__}{detail}")
|
|
|
|
|
|
def _page_result(provider: str, tool: str, arguments: Dict[str, Any], items: List[Any], used, upstream_next,
|
|
skip: int = 0, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
"""One page of scrubbed items, cut at MAX_RESPONSE_BYTES; the token resumes exactly where the cut fell."""
|
|
remaining = [_scrub(item) for item in items[skip:]]
|
|
kept, size = [], 0
|
|
for item in remaining:
|
|
size += len(json.dumps(item, default=str))
|
|
if kept and size > MAX_RESPONSE_BYTES:
|
|
break
|
|
kept.append(item)
|
|
if len(kept) < len(remaining):
|
|
next_token = _write_page(used, skip + len(kept))
|
|
else:
|
|
next_token = _write_page(upstream_next, 0) if upstream_next else None
|
|
data = {"provider": provider, "items": kept, "count": len(kept), "next_page_token": next_token,
|
|
"collected_at": _collected_at(), "evidence_ref": _evidence_ref(tool, arguments, kept)}
|
|
data.update(extra or {})
|
|
return {"success": True, "data": data}
|
|
|
|
|
|
def _one_result(provider: str, tool: str, arguments: Dict[str, Any], metadata: Any) -> Dict[str, Any]:
|
|
clean = _scrub(metadata)
|
|
return {"success": True, "data": {"provider": provider, "metadata": clean, "collected_at": _collected_at(),
|
|
"evidence_ref": _evidence_ref(tool, arguments, clean)}}
|