"""A live run's dependency graph, built in code (ADR 0019): one graph, three kinds of node. - `resource`: a scope entry the hyperscaler showed (its inventory record). - `external`: a resource a scope entry depends on that lies outside the scope root, or wasn't inventoried. - `ci`: a CMDB CI the reconciliation judged (ADR 0018). Every edge runs parent (`source`) to child (`target`), whatever the kinds: - resource -> resource or external: a dependency the scan recorded in a finding's `dependencies`; - ci -> ci: a relationship the CMDB records; - ci -> resource (`represents`): the reconciliation's match of a CI to its resource. The comparison walks this one graph: a CMDB relationship whose two CIs represent two resources the cloud also links, in the same direction, is `confirmed`; one the cloud doesn't show is `cmdb_only`; a cloud dependency between two represented resources the CMDB doesn't record is `discovered_only`. No model is called. """ from __future__ import annotations import re from typing import Any, Dict, List, Optional, Tuple from app import scope as sc AZURE_CONTAINER = re.compile(r"^(/subscriptions/[^/]+(?:/resourcegroups/[^/]+)?)", re.I) GCP_PROJECT = re.compile(r"/projects/([^/]+)(?:/|$)") NETWORK = ("microsoft.network/virtualnetworks", "microsoft.network/virtualnetworks/subnets", "microsoft.network/loadbalancers", "microsoft.network/publicipaddresses", "microsoft.network/applicationgateways", "microsoft.network/networksecuritygroups", "microsoft.network/privatednszones", "microsoft.network/dnszones", "compute.googleapis.com/network", "compute.googleapis.com/subnetwork", "compute.googleapis.com/address", "compute.googleapis.com/forwardingrule") HOSTING = ("microsoft.compute/disks", "microsoft.compute/virtualmachines", "microsoft.compute/virtualmachinescalesets", "microsoft.containerservice/managedclusters", "microsoft.web/serverfarms", "compute.googleapis.com/disk", "compute.googleapis.com/instance", "container.googleapis.com/cluster") SHARED = ("microsoft.keyvault/vaults", "microsoft.managedidentity/userassignedidentities", "secretmanager.googleapis.com/secret", "cloudkms.googleapis.com/cryptokey") MESSAGING = ("microsoft.servicebus/namespaces", "microsoft.eventhub/namespaces", "microsoft.eventgrid/topics", "microsoft.eventgrid/systemtopics", "pubsub.googleapis.com/topic") CLOUD = {"confidence": 0.8, "validation": "auto_approved"} # the provider's own record of the link (D3) CMDB = {"confidence": 0.7, "validation": "sme_review_recommended"} # what the CMDB says, not seen in the cloud CONFIRMED = 0.95 # a cloud edge the CMDB also records MATCH = {"cloud_resource_id": {"confidence": 0.95, "validation": "auto_approved"}, "name": {"confidence": 0.7, "validation": "sme_review_recommended"}} def key(provider: str, rid: Any) -> str: rid = str(rid or "").strip().rstrip("/") return rid.lower() if provider == "azure" else rid def type_of(provider: str, rid: str) -> str: """A resource's type from its id: microsoft.network/virtualnetworks/subnets, compute.googleapis.com/disk.""" if provider == "azure": tail = rid.split("/providers/")[-1].split("/") if "/providers/" in rid else [] if len(tail) < 3: return "" return "/".join([tail[0]] + tail[1::2][:max(1, (len(tail) - 1) // 2)]).lower() parts = rid.lstrip("/").split("/") if len(parts) < 3: return "" collection = parts[-2] singular = collection[:-1] if collection.endswith("s") else collection return f"{parts[0]}/{singular}".lower() def container(provider: str, rid: str) -> Optional[str]: if provider == "azure": found = AZURE_CONTAINER.match(rid.strip()) return found.group(1).lower() if found else None found = GCP_PROJECT.search(rid) return f"projects/{found.group(1)}" if found else None def edge_type(provider: str, rtype: str) -> str: if rtype in NETWORK or rtype.endswith(("/subnets", "/subnetwork")): return "network_attachment" if rtype in sc.DATA_STORES.get(provider, ()): return "app_to_db" if rtype in HOSTING: return "hosted_on" if rtype in SHARED: return "app_to_shared_service" if rtype in MESSAGING: return "middleware" return "unclassified" def _dependency_id(dep: Any) -> str: if isinstance(dep, dict): return str(dep.get("id") or dep.get("target") or dep.get("resource_id") or dep.get("scope_entry_id") or "") return str(dep or "") def build(payload: Dict[str, Any]) -> Dict[str, Any]: provider = payload["provider"] records = [r for r in payload.get("records") or [] if r.get("scope_entry_id")] by_resource = {key(provider, r["scope_entry_id"]): r for r in records} record_containers = {container(provider, r["scope_entry_id"]) for r in records} nodes: Dict[str, Dict[str, Any]] = {} edges: List[Dict[str, Any]] = [] folded: List[Dict[str, Any]] = [] seen: Dict[Tuple[str, str, str], Dict[str, Any]] = {} for r in records: nodes[r["canonical_id"]] = {"id": r["canonical_id"], "canonical_id": r["canonical_id"], "kind": "resource", "name": r.get("hostname"), "resource_id": r["scope_entry_id"]} def edge(source: str, target: str, kind: str, evidence: List[Dict[str, str]], weight: Dict[str, Any] ) -> Dict[str, Any]: found = seen.get((source, target, kind)) if found: found["evidence"] += [e for e in evidence if e not in found["evidence"]] return found made = {"edge_id": f"E-{len(edges) + 1:03d}", "source": source, "target": target, "type": kind, "protocol_port": None, "evidence": list(evidence), **weight} edges.append(made) seen[(source, target, kind)] = made return made # resource -> resource / external: what the scan recorded for finding in payload.get("findings") or []: source = by_resource.get(key(provider, finding.get("scope_entry_id"))) if not source: continue evidence = [{"kind": "config", "ref": finding.get("evidence_ref") or finding["scope_entry_id"]}] for dep in finding.get("dependencies") or []: rid = _dependency_id(dep) if not rid or key(provider, rid) == key(provider, source["scope_entry_id"]): continue rtype = type_of(provider, rid) target = by_resource.get(key(provider, rid)) if target: edge(source["canonical_id"], target["canonical_id"], edge_type(provider, rtype), evidence, CLOUD) continue if rtype in sc.FOLDED.get(provider, {}): folded.append({"source": source["canonical_id"], "target": rid, "type": rtype, "rule": sc.FOLDED[provider][rtype], "evidence_ref": evidence[0]["ref"]}) continue node = nodes.setdefault(rid, {"id": rid, "canonical_id": rid, "kind": "external", "name": rid.rstrip("/").split("/")[-1], "type": rtype, "container": container(provider, rid), "reason": ("not_inventoried" if container(provider, rid) in record_containers else "outside_scope_root"), "referenced_by": []}) if source["canonical_id"] not in node["referenced_by"]: node["referenced_by"].append(source["canonical_id"]) edge(source["canonical_id"], rid, edge_type(provider, rtype), evidence, CLOUD) # ci nodes, ci -> resource (represents), ci -> ci (relationships) represents: Dict[str, str] = {} for ci in payload.get("cmdb_cis") or []: node_id = f"cmdb:{ci['ci_ref']}" nodes[node_id] = {"id": node_id, "canonical_id": node_id, "kind": "ci", "ci_ref": ci["ci_ref"], "name": ci.get("name"), "ci_class": ci.get("ci_class"), "lifecycle": ci.get("lifecycle"), "result": ci.get("result"), "evidence_ref": ci.get("evidence_ref")} target = ci.get("matched_canonical_id") if target and target in nodes: represents[node_id] = target edge(node_id, target, "represents", [{"kind": "reconciliation", "ref": ci.get("evidence_ref") or node_id}], MATCH.get(ci.get("match_basis") or "name", MATCH["name"])) cloud_pairs: Dict[Tuple[str, str], List[Dict[str, Any]]] = {} for e in edges: if nodes.get(e["source"], {}).get("kind") == "resource" and nodes.get(e["target"], {}).get("kind") == "resource": cloud_pairs.setdefault((e["source"], e["target"]), []).append(e) comparison: List[Dict[str, Any]] = [] matched_pairs = set() for rel in payload.get("cmdb_relationships") or []: parent, child = f"cmdb:{rel['parent_ref']}", f"cmdb:{rel['child_ref']}" if parent not in nodes or child not in nodes: continue cmdb_edge = edge(parent, child, rel["type"], [{"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}], CMDB) pair = (represents.get(parent), represents.get(child)) item = {"cmdb_edge": cmdb_edge["edge_id"], "parent_ci": rel["parent_ref"], "child_ci": rel["child_ref"], "source": pair[0], "target": pair[1], "source_resource": nodes.get(pair[0] or "", {}).get("resource_id"), "target_resource": nodes.get(pair[1] or "", {}).get("resource_id")} if not all(pair): comparison.append({**item, "result": "not_comparable", "reason": "an end has no matched resource, so the cloud can't confirm or deny it"}) elif pair in cloud_pairs: matched_pairs.add(pair) for cloud in cloud_pairs[pair]: cloud["evidence"].append({"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}) cloud["confidence"] = max(cloud["confidence"], CONFIRMED) comparison.append({**item, "result": "confirmed", "cloud_edges": [c["edge_id"] for c in cloud_pairs[pair]]}) else: comparison.append({**item, "result": "cmdb_only"}) represented = set(represents.values()) if payload.get("cmdb_cis"): for pair, cloud in cloud_pairs.items(): if pair[0] in represented and pair[1] in represented and pair not in matched_pairs: comparison.append({"result": "discovered_only", "cloud_edges": [c["edge_id"] for c in cloud], "source": pair[0], "target": pair[1], "source_resource": nodes[pair[0]].get("resource_id"), "target_resource": nodes[pair[1]].get("resource_id"), "parent_ci": next((n[5:] for n, r in represents.items() if r == pair[0]), None), "child_ci": next((n[5:] for n, r in represents.items() if r == pair[1]), None)}) # fan-in and fan-out over dependency edges (represents is a link, not a dependency) dependency_edges = [e for e in edges if e["type"] != "represents"] for node in nodes.values(): node["fan_out"] = sum(1 for e in dependency_edges if e["source"] == node["id"]) node["fan_in"] = sum(1 for e in dependency_edges if e["target"] == node["id"]) node["depended_on"] = node["fan_in"] > 0 node["retire_constraint"] = "cannot_retire_without_mitigation" if node["depended_on"] else "none" orphans = [n["id"] for n in nodes.values() if n["kind"] == "resource" and not n["fan_in"] and not n["fan_out"]] counts = {kind: sum(1 for n in nodes.values() if n["kind"] == kind) for kind in ("resource", "external", "ci")} recorded = sum(len(f.get("dependencies") or []) for f in payload.get("findings") or []) by_result = {r: sum(1 for c in comparison if c["result"] == r) for r in ("confirmed", "cmdb_only", "discovered_only", "not_comparable")} note = (f"Configured links only; traffic isn't known. {recorded} dependencies were recorded by the scan: " f"{sum(1 for e in dependency_edges if nodes[e['source']]['kind'] == 'resource')} edges, " f"{counts['external']} outside the scope as external nodes, {len(folded)} to a resource's own components. " f"{counts['ci']} CMDB CIs, {len(payload.get('cmdb_relationships') or [])} CMDB relationships: " + ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in by_result.items()) + ".") return {"graph_id": f"GRAPH-{payload.get('run_id') or payload['engagement_id']}", "engagement_id": payload["engagement_id"], "edges": edges, "nodes": list(nodes.values()), "orphans": orphans, "validation_queue": [e["edge_id"] for e in edges if e["validation"] != "auto_approved"], "folded_dependencies": folded, "dangling": list(payload.get("cmdb_dangling") or []), "graph_comparison": comparison, "note": note}