Files
cp1-agents/mcp-servers/_common/tool_support.py
Jonathan Boniface f659564208 Platform scripts, MCP servers, deployment and docs for the KAB runs
- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with
  --input name=@file), store per-project discovery connections, bootstrap agent MCP keys,
  refresh the KAB token, publish reports and build estate snapshots.
- Hyperscaler discovery servers extended in place; cloud-knowledge folded into them.
- deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents.
- Platform issue write-ups and current-wiring notes under docs/platform-issues/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

188 lines
8.4 KiB
Python

# Shared by every hyperscaler server. scripts/render_mcp_server.py inlines this file into a server through the
# definition's shared_code, because a rendered image holds only the template's files. The builder accepts plain
# functions and literal constants here, nothing that runs at import.
import base64
import hashlib
import json
import re
DEFAULT_PAGE_SIZE = 50
MAX_PAGE_SIZE = 200
# A reply goes into a model's context, which is the real limit on a hand-off, so a page is cut at this size.
MAX_RESPONSE_BYTES = 200000
# A throttled call is retried inside the budget, then fails and says so.
RETRY_ATTEMPTS = 5
RETRY_MAX_WAIT_SECONDS = 20
RETRY_BUDGET_SECONDS = 60
# Values under these keys are never returned, in any casing.
SECRET_KEYS = (
"userdata", "customdata", "customscript", "protectedsettings", "connectionstring", "connectionstrings",
"appsettings", "env", "envvars", "environmentvariables", "variables", "password", "adminpassword", "secret",
"secretvalue", "sastoken", "accesskey", "primarykey", "secondarykey", "privatekey", "kubeconfig", "credentials",
)
# A single value under one of these keys is a tag or label (env=dev), not a secret: only a list or map under it (a
# container's environment variables) is redacted.
SCALAR_SAFE_KEYS = ("env",)
# Values under these keys name a person, or may. A principal of unknown type counts as a person.
PERSON_KEYS = (
"createdby", "lastmodifiedby", "updatedby", "creator", "owner", "owneremail", "contact", "contactemail", "email",
"objectid", "principalid", "principalname", "userprincipalname",
)
PRINCIPAL_KINDS = {"user": "[user]", "group": "[group]", "domain": "[domain]", "principal": "[user]",
"principalSet": "[group]", "deleted": "[user]"}
NON_PERSON_EMAIL_DOMAINS = ("gserviceaccount.com",)
def _clean_text(text: str) -> str:
"""Anonymise people inside a string: emails, AWS IAM users, and session names, which are often emails."""
def email(match):
return match.group(0) if match.group(0).lower().endswith(NON_PERSON_EMAIL_DOMAINS) else "[email]"
text = re.sub(r"arn:aws[a-z-]*:iam::\d{12}:user/[^\s\"',]+", "[user]", text)
text = re.sub(r"(arn:aws[a-z-]*:sts::\d{12}:assumed-role/[^/\s\"',]+)/[^\s\"',]+", r"\1/[session]", text)
# Bounded, and skipped without an @: an unbounded pattern takes minutes on one long string.
if "@" not in text:
return text
return re.sub(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]{1,255}\.[A-Za-z]{2,24}", email, text)
def _anonymise_member(value: str) -> str:
"""One IAM member, such as user:a@b.example. Service accounts stay: Discovery shows what can reach what."""
kind = value.split(":", 1)[0]
if ":" in value and kind == "serviceAccount":
return value
if ":" in value and kind in PRINCIPAL_KINDS:
return PRINCIPAL_KINDS[kind]
return _clean_text(value)
def _scalar(value) -> bool:
return value is None or isinstance(value, (str, bool, int, float))
def _scrub(value, key: str = ""):
"""Drop secrets and anonymise people, however deep. Every tool's reply passes through here."""
lowered = key.lower().replace("_", "").replace("-", "")
if lowered in SECRET_KEYS and not (lowered in SCALAR_SAFE_KEYS and _scalar(value)):
return "[redacted]"
if lowered in PERSON_KEYS:
return "[user]"
if isinstance(value, dict):
tag = str(value.get("Key", value.get("key", ""))).lower().replace("_", "").replace("-", "")
cleaned = {k: _scrub(v, str(k)) for k, v in value.items()}
for name in ("Value", "value"):
if name in cleaned and tag in PERSON_KEYS:
cleaned[name] = "[user]"
elif name in cleaned and tag in SECRET_KEYS and not (tag in SCALAR_SAFE_KEYS and _scalar(cleaned[name])):
cleaned[name] = "[redacted]"
return cleaned
if isinstance(value, (list, tuple)):
return [_scrub(v, "members" if lowered == "members" else "") for v in value]
if isinstance(value, str):
return _anonymise_member(value) if lowered == "members" else _clean_text(value)
if value is None or isinstance(value, (bool, int, float)):
return value
return str(value)
def _retry_delay(response, attempt: int) -> float:
"""How long the provider asked us to wait: Retry-After, or Resource Graph's hh:mm:ss quota reset."""
reset = response.headers.get("x-ms-user-quota-resets-after", "")
if reset.count(":") == 2:
hours, minutes, seconds = reset.split(":")
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
try:
return float(response.headers.get("retry-after", ""))
except ValueError:
return float(2 ** attempt)
async def _with_backoff(send):
"""Send, waiting when the provider throttles. The waits stay well inside the 120-second limit on an MCP call."""
waited = 0.0
response = await send()
for attempt in range(RETRY_ATTEMPTS):
if response.status_code not in (429, 503):
return response
delay = min(_retry_delay(response, attempt), RETRY_MAX_WAIT_SECONDS)
if waited + delay > RETRY_BUDGET_SECONDS:
return response
logger.info("upstream_throttled", status=response.status_code, wait_seconds=delay)
await asyncio.sleep(delay)
waited += delay
response = await send()
return response
def _page_size(arguments: Dict[str, Any]) -> int:
try:
size = int(arguments.get("page_size") or DEFAULT_PAGE_SIZE)
except (TypeError, ValueError):
return DEFAULT_PAGE_SIZE
return max(1, min(size, MAX_PAGE_SIZE))
def _read_page(arguments: Dict[str, Any]):
"""(upstream token, items of that upstream page already returned)."""
token = arguments.get("page_token")
if not token:
return None, 0
state = json.loads(base64.urlsafe_b64decode(str(token).encode()).decode())
return state["u"], int(state["s"])
def _write_page(upstream, skip: int) -> str:
return base64.urlsafe_b64encode(json.dumps({"u": upstream, "s": skip}).encode()).decode()
def _evidence_ref(tool: str, arguments: Dict[str, Any], payload) -> str:
asked = {k: v for k, v in arguments.items() if k != "page_token"}
blob = json.dumps([tool, asked, payload], sort_keys=True, default=str)
return "sha256:" + hashlib.sha256(blob.encode()).hexdigest()
def _collected_at() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def _refuse(code: str, message: str) -> Dict[str, Any]:
return {"success": False, "error": {"code": code, "message": message}}
def _upstream_failed(provider: str, error: Exception) -> Dict[str, Any]:
"""Name the failure and never its text: an upstream error can quote a request or an identity."""
status = getattr(getattr(error, "response", None), "status_code", None)
code = getattr(error, "response", None)
if isinstance(code, dict):
status = code.get("Error", {}).get("Code")
logger.warning("upstream_call_failed", provider=provider, error_type=type(error).__name__, status=status)
detail = f" ({status})" if status else ""
return _refuse("upstream_failed", f"{provider} refused or failed the call: {type(error).__name__}{detail}")
def _page_result(provider: str, tool: str, arguments: Dict[str, Any], items: List[Any], used, upstream_next,
skip: int = 0, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""One page of scrubbed items, cut at MAX_RESPONSE_BYTES; the token resumes exactly where the cut fell."""
remaining = [_scrub(item) for item in items[skip:]]
kept, size = [], 0
for item in remaining:
size += len(json.dumps(item, default=str))
if kept and size > MAX_RESPONSE_BYTES:
break
kept.append(item)
if len(kept) < len(remaining):
next_token = _write_page(used, skip + len(kept))
else:
next_token = _write_page(upstream_next, 0) if upstream_next else None
data = {"provider": provider, "items": kept, "count": len(kept), "next_page_token": next_token,
"collected_at": _collected_at(), "evidence_ref": _evidence_ref(tool, arguments, kept)}
data.update(extra or {})
return {"success": True, "data": data}
def _one_result(provider: str, tool: str, arguments: Dict[str, Any], metadata: Any) -> Dict[str, Any]:
clean = _scrub(metadata)
return {"success": True, "data": {"provider": provider, "metadata": clean, "collected_at": _collected_at(),
"evidence_ref": _evidence_ref(tool, arguments, clean)}}