- Scripts to run workflows from a YAML context or run inputs (kab_run_workflow.py, with --input name=@file), store per-project discovery connections, bootstrap agent MCP keys, refresh the KAB token, publish reports and build estate snapshots. - Hyperscaler discovery servers extended in place; cloud-knowledge folded into them. - deploy/crucible-kab-dev.yaml: the Discovery package agent and the Assessment agents. - Platform issue write-ups and current-wiring notes under docs/platform-issues/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
158 lines
7.2 KiB
Python
158 lines
7.2 KiB
Python
# gcp-discovery: every read goes through Cloud Asset Inventory or another read API, under the connection's scope_root.
|
|
import asyncio
|
|
from google.auth.transport.requests import Request as GoogleRequest
|
|
from google.oauth2 import service_account
|
|
|
|
GCP_ASSET_API = "https://cloudasset.googleapis.com/v1"
|
|
GCP_ORG_POLICY_API = "https://orgpolicy.googleapis.com/v2"
|
|
GCP_RECOMMENDER_API = "https://recommender.googleapis.com/v1"
|
|
GCP_READ_ONLY = "https://www.googleapis.com/auth/cloud-platform.read-only"
|
|
GCP_TOKENS = {}
|
|
GCP_SERVERS = {
|
|
"instances": ["compute.googleapis.com/Instance"],
|
|
"instance_groups": ["compute.googleapis.com/InstanceGroupManager"],
|
|
"disks": ["compute.googleapis.com/Disk"],
|
|
}
|
|
GCP_WORKLOADS = {
|
|
"cloud_run": ["run.googleapis.com/Service"],
|
|
"cloud_functions": ["cloudfunctions.googleapis.com/Function"],
|
|
"app_engine": ["appengine.googleapis.com/Service"],
|
|
}
|
|
GCP_KUBERNETES = {"clusters": ["container.googleapis.com/Cluster"], "node_pools": ["container.googleapis.com/NodePool"]}
|
|
GCP_NETWORK = {
|
|
"networks": ["compute.googleapis.com/Network"],
|
|
"subnetworks": ["compute.googleapis.com/Subnetwork"],
|
|
"routes": ["compute.googleapis.com/Route"],
|
|
"firewall_rules": ["compute.googleapis.com/Firewall", "compute.googleapis.com/FirewallPolicy"],
|
|
"load_balancers": ["compute.googleapis.com/ForwardingRule", "compute.googleapis.com/BackendService"],
|
|
"gateways": ["compute.googleapis.com/Router", "compute.googleapis.com/VpnGateway",
|
|
"compute.googleapis.com/InterconnectAttachment"],
|
|
}
|
|
GCP_DATA = {
|
|
"cloud_sql": ["sqladmin.googleapis.com/Instance"],
|
|
"firestore": ["firestore.googleapis.com/Database"],
|
|
"cloud_storage": ["storage.googleapis.com/Bucket"],
|
|
}
|
|
GCP_ACCOUNTS = {
|
|
"projects": ["cloudresourcemanager.googleapis.com/Project"],
|
|
"folders": ["cloudresourcemanager.googleapis.com/Folder"],
|
|
}
|
|
GCP_PERIMETERS = {"vpc_service_controls": ["accesscontextmanager.googleapis.com/ServicePerimeter"]}
|
|
GCP_RECOMMENDERS = ("google.compute.instance.MachineTypeRecommender", "google.compute.instance.IdleResourceRecommender",
|
|
"google.compute.disk.IdleResourceRecommender", "google.iam.policy.Recommender")
|
|
GCP_PILLARS = {"cost": "cost", "security": "security", "performance": "performance", "reliability": "reliability",
|
|
"manageability": "operations", "sustainability": "sustainability"}
|
|
|
|
|
|
def _gcp_scope() -> str:
|
|
scope = str(get_connection().get("scope_root", "")).strip("/")
|
|
kind, _, identifier = scope.partition("/")
|
|
if kind not in ("organizations", "folders", "projects") or not identifier or "/" in identifier:
|
|
raise ValueError("scope_root must be organizations/<id>, folders/<id> or projects/<id>")
|
|
return scope
|
|
|
|
|
|
def _gcp_client():
|
|
return get_upstream_client("gcp", lambda c: httpx.AsyncClient(timeout=100))
|
|
|
|
|
|
async def _gcp_token() -> str:
|
|
connection = get_connection()
|
|
if connection.get("access_token"):
|
|
return str(connection["access_token"])
|
|
key = hashlib.sha256(json.dumps(connection, sort_keys=True).encode()).hexdigest()
|
|
cached = GCP_TOKENS.get(key)
|
|
now = time.time()
|
|
if cached and cached[1] > now + 60:
|
|
return cached[0]
|
|
|
|
def refresh():
|
|
info = json.loads(connection["service_account_json"])
|
|
credentials = service_account.Credentials.from_service_account_info(info, scopes=[GCP_READ_ONLY])
|
|
credentials.refresh(GoogleRequest())
|
|
return credentials.token
|
|
|
|
token = await asyncio.to_thread(refresh)
|
|
GCP_TOKENS[key] = (token, now + 3000)
|
|
return token
|
|
|
|
|
|
def _gcp_quota_project() -> str:
|
|
"""The project Google bills a user credential's calls to: the connection's quota_project, else a projects/<id>
|
|
scope root. Recommender and Developer Knowledge refuse a user token without one; Cloud Asset doesn't need it."""
|
|
connection = get_connection()
|
|
if connection.get("quota_project"):
|
|
return str(connection["quota_project"])
|
|
kind, _, identifier = str(connection.get("scope_root", "")).strip("/").partition("/")
|
|
return identifier if kind == "projects" else ""
|
|
|
|
|
|
# The APIs that need a quota project for a user credential. Sent only to these, so Cloud Asset keeps working for a
|
|
# caller who lacks serviceusage.services.use on the quota project.
|
|
GCP_QUOTA_APIS = ("https://recommender.googleapis.com/",) # a literal: the builder refuses computed module-level values
|
|
|
|
|
|
async def _gcp_auth(quota: bool) -> Dict[str, str]:
|
|
headers = {"Authorization": f"Bearer {await _gcp_token()}"}
|
|
project = _gcp_quota_project() if quota else ""
|
|
if project:
|
|
headers["x-goog-user-project"] = project
|
|
return headers
|
|
|
|
|
|
async def _gcp_get(url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
headers = await _gcp_auth(url.startswith(GCP_QUOTA_APIS))
|
|
response = await _with_backoff(lambda: _gcp_client().get(url, params=params, headers=headers))
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
|
|
async def _gcp_page(tool: str, arguments: Dict[str, Any], url: str, params: Dict[str, Any], result_key: str,
|
|
finish=None) -> Dict[str, Any]:
|
|
try:
|
|
upstream, skip = _read_page(arguments)
|
|
except (ValueError, KeyError, TypeError):
|
|
return _refuse("invalid_page_token", "page_token isn't one this server issued")
|
|
request = {**params, "pageSize": _page_size(arguments)}
|
|
if upstream:
|
|
request["pageToken"] = upstream
|
|
try:
|
|
body = await _gcp_get(url, request)
|
|
except Exception as error:
|
|
return _upstream_failed("gcp", error)
|
|
items = [finish(i) for i in body.get(result_key, [])] if finish else body.get(result_key, [])
|
|
return _page_result("gcp", tool, arguments, items, upstream, body.get("nextPageToken"), skip)
|
|
|
|
|
|
async def _gcp_search(tool: str, arguments: Dict[str, Any], asset_types: List[str], query: str = "") -> Dict[str, Any]:
|
|
try:
|
|
url = f"{GCP_ASSET_API}/{_gcp_scope()}:searchAllResources"
|
|
except ValueError as error:
|
|
return _refuse("invalid_connection", str(error))
|
|
params = {"assetTypes": asset_types, "readMask": "*"}
|
|
if query:
|
|
params["query"] = query
|
|
return await _gcp_page(tool, arguments, url, params, "results")
|
|
|
|
|
|
async def _gcp_kind(tool: str, arguments: Dict[str, Any], kinds: Dict[str, List[str]], default: str) -> Dict[str, Any]:
|
|
kind = arguments.get("kind") or default
|
|
if kind not in kinds:
|
|
return _refuse("unknown_kind", "kind must be one of: " + ", ".join(kinds))
|
|
return await _gcp_search(tool, arguments, kinds[kind])
|
|
|
|
|
|
async def _gcp_project_in_scope(project_id: str) -> bool:
|
|
"""Whether Cloud Asset, queried under the scope, can see this project. The API sets the boundary."""
|
|
scope = _gcp_scope()
|
|
if scope == f"projects/{project_id}":
|
|
return True
|
|
body = await _gcp_get(f"{GCP_ASSET_API}/{scope}:searchAllResources",
|
|
{"assetTypes": GCP_ACCOUNTS["projects"], "query": f"name:{project_id}", "pageSize": 5})
|
|
return any(str(r.get("name", "")).endswith(f"/projects/{project_id}") for r in body.get("results", []))
|
|
|
|
|
|
def _gcp_recommendation(item: Dict[str, Any]) -> Dict[str, Any]:
|
|
category = str(item.get("primaryImpact", {}).get("category", ""))
|
|
return {**item, "pillar": GCP_PILLARS.get(category.lower(), "other"), "provider_category": category}
|