- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
221 lines
13 KiB
Python
221 lines
13 KiB
Python
"""A live run's dependency graph, built in code (ADR 0019): one graph, three kinds of node.
|
|
|
|
- `resource`: a scope entry the hyperscaler showed (its inventory record).
|
|
- `external`: a resource a scope entry depends on that lies outside the scope root, or wasn't inventoried.
|
|
- `ci`: a CMDB CI the reconciliation judged (ADR 0018).
|
|
|
|
Every edge runs parent (`source`) to child (`target`), whatever the kinds:
|
|
|
|
- resource -> resource or external: a dependency the scan recorded in a finding's `dependencies`;
|
|
- ci -> ci: a relationship the CMDB records;
|
|
- ci -> resource (`represents`): the reconciliation's match of a CI to its resource.
|
|
|
|
The comparison walks this one graph: a CMDB relationship whose two CIs represent two resources the cloud also links,
|
|
in the same direction, is `confirmed`; one the cloud doesn't show is `cmdb_only`; a cloud dependency between two
|
|
represented resources the CMDB doesn't record is `discovered_only`. No model is called.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from app import scope as sc
|
|
|
|
AZURE_CONTAINER = re.compile(r"^(/subscriptions/[^/]+(?:/resourcegroups/[^/]+)?)", re.I)
|
|
GCP_PROJECT = re.compile(r"/projects/([^/]+)(?:/|$)")
|
|
NETWORK = ("microsoft.network/virtualnetworks", "microsoft.network/virtualnetworks/subnets",
|
|
"microsoft.network/loadbalancers", "microsoft.network/publicipaddresses",
|
|
"microsoft.network/applicationgateways", "microsoft.network/networksecuritygroups",
|
|
"microsoft.network/privatednszones", "microsoft.network/dnszones",
|
|
"compute.googleapis.com/network", "compute.googleapis.com/subnetwork", "compute.googleapis.com/address",
|
|
"compute.googleapis.com/forwardingrule")
|
|
HOSTING = ("microsoft.compute/disks", "microsoft.compute/virtualmachines", "microsoft.compute/virtualmachinescalesets",
|
|
"microsoft.containerservice/managedclusters", "microsoft.web/serverfarms",
|
|
"compute.googleapis.com/disk", "compute.googleapis.com/instance", "container.googleapis.com/cluster")
|
|
SHARED = ("microsoft.keyvault/vaults", "microsoft.managedidentity/userassignedidentities",
|
|
"secretmanager.googleapis.com/secret", "cloudkms.googleapis.com/cryptokey")
|
|
MESSAGING = ("microsoft.servicebus/namespaces", "microsoft.eventhub/namespaces", "microsoft.eventgrid/topics",
|
|
"microsoft.eventgrid/systemtopics", "pubsub.googleapis.com/topic")
|
|
CLOUD = {"confidence": 0.8, "validation": "auto_approved"} # the provider's own record of the link (D3)
|
|
CMDB = {"confidence": 0.7, "validation": "sme_review_recommended"} # what the CMDB says, not seen in the cloud
|
|
CONFIRMED = 0.95 # a cloud edge the CMDB also records
|
|
MATCH = {"cloud_resource_id": {"confidence": 0.95, "validation": "auto_approved"},
|
|
"name": {"confidence": 0.7, "validation": "sme_review_recommended"}}
|
|
|
|
|
|
def key(provider: str, rid: Any) -> str:
|
|
rid = str(rid or "").strip().rstrip("/")
|
|
return rid.lower() if provider == "azure" else rid
|
|
|
|
|
|
def type_of(provider: str, rid: str) -> str:
|
|
"""A resource's type from its id: microsoft.network/virtualnetworks/subnets, compute.googleapis.com/disk."""
|
|
if provider == "azure":
|
|
tail = rid.split("/providers/")[-1].split("/") if "/providers/" in rid else []
|
|
if len(tail) < 3:
|
|
return ""
|
|
return "/".join([tail[0]] + tail[1::2][:max(1, (len(tail) - 1) // 2)]).lower()
|
|
parts = rid.lstrip("/").split("/")
|
|
if len(parts) < 3:
|
|
return ""
|
|
collection = parts[-2]
|
|
singular = collection[:-1] if collection.endswith("s") else collection
|
|
return f"{parts[0]}/{singular}".lower()
|
|
|
|
|
|
def container(provider: str, rid: str) -> Optional[str]:
|
|
if provider == "azure":
|
|
found = AZURE_CONTAINER.match(rid.strip())
|
|
return found.group(1).lower() if found else None
|
|
found = GCP_PROJECT.search(rid)
|
|
return f"projects/{found.group(1)}" if found else None
|
|
|
|
|
|
def edge_type(provider: str, rtype: str) -> str:
|
|
if rtype in NETWORK or rtype.endswith(("/subnets", "/subnetwork")):
|
|
return "network_attachment"
|
|
if rtype in sc.DATA_STORES.get(provider, ()):
|
|
return "app_to_db"
|
|
if rtype in HOSTING:
|
|
return "hosted_on"
|
|
if rtype in SHARED:
|
|
return "app_to_shared_service"
|
|
if rtype in MESSAGING:
|
|
return "middleware"
|
|
return "unclassified"
|
|
|
|
|
|
def _dependency_id(dep: Any) -> str:
|
|
if isinstance(dep, dict):
|
|
return str(dep.get("id") or dep.get("target") or dep.get("resource_id") or dep.get("scope_entry_id") or "")
|
|
return str(dep or "")
|
|
|
|
|
|
def build(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
provider = payload["provider"]
|
|
records = [r for r in payload.get("records") or [] if r.get("scope_entry_id")]
|
|
by_resource = {key(provider, r["scope_entry_id"]): r for r in records}
|
|
record_containers = {container(provider, r["scope_entry_id"]) for r in records}
|
|
nodes: Dict[str, Dict[str, Any]] = {}
|
|
edges: List[Dict[str, Any]] = []
|
|
folded: List[Dict[str, Any]] = []
|
|
seen: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
|
|
|
|
for r in records:
|
|
nodes[r["canonical_id"]] = {"id": r["canonical_id"], "canonical_id": r["canonical_id"], "kind": "resource",
|
|
"name": r.get("hostname"), "resource_id": r["scope_entry_id"]}
|
|
|
|
def edge(source: str, target: str, kind: str, evidence: List[Dict[str, str]], weight: Dict[str, Any]
|
|
) -> Dict[str, Any]:
|
|
found = seen.get((source, target, kind))
|
|
if found:
|
|
found["evidence"] += [e for e in evidence if e not in found["evidence"]]
|
|
return found
|
|
made = {"edge_id": f"E-{len(edges) + 1:03d}", "source": source, "target": target, "type": kind,
|
|
"protocol_port": None, "evidence": list(evidence), **weight}
|
|
edges.append(made)
|
|
seen[(source, target, kind)] = made
|
|
return made
|
|
|
|
# resource -> resource / external: what the scan recorded
|
|
for finding in payload.get("findings") or []:
|
|
source = by_resource.get(key(provider, finding.get("scope_entry_id")))
|
|
if not source:
|
|
continue
|
|
evidence = [{"kind": "config", "ref": finding.get("evidence_ref") or finding["scope_entry_id"]}]
|
|
for dep in finding.get("dependencies") or []:
|
|
rid = _dependency_id(dep)
|
|
if not rid or key(provider, rid) == key(provider, source["scope_entry_id"]):
|
|
continue
|
|
rtype = type_of(provider, rid)
|
|
target = by_resource.get(key(provider, rid))
|
|
if target:
|
|
edge(source["canonical_id"], target["canonical_id"], edge_type(provider, rtype), evidence, CLOUD)
|
|
continue
|
|
if rtype in sc.FOLDED.get(provider, {}):
|
|
folded.append({"source": source["canonical_id"], "target": rid, "type": rtype,
|
|
"rule": sc.FOLDED[provider][rtype], "evidence_ref": evidence[0]["ref"]})
|
|
continue
|
|
node = nodes.setdefault(rid, {"id": rid, "canonical_id": rid, "kind": "external",
|
|
"name": rid.rstrip("/").split("/")[-1], "type": rtype,
|
|
"container": container(provider, rid),
|
|
"reason": ("not_inventoried" if container(provider, rid) in record_containers
|
|
else "outside_scope_root"),
|
|
"referenced_by": []})
|
|
if source["canonical_id"] not in node["referenced_by"]:
|
|
node["referenced_by"].append(source["canonical_id"])
|
|
edge(source["canonical_id"], rid, edge_type(provider, rtype), evidence, CLOUD)
|
|
|
|
# ci nodes, ci -> resource (represents), ci -> ci (relationships)
|
|
represents: Dict[str, str] = {}
|
|
for ci in payload.get("cmdb_cis") or []:
|
|
node_id = f"cmdb:{ci['ci_ref']}"
|
|
nodes[node_id] = {"id": node_id, "canonical_id": node_id, "kind": "ci", "ci_ref": ci["ci_ref"],
|
|
"name": ci.get("name"), "ci_class": ci.get("ci_class"), "lifecycle": ci.get("lifecycle"),
|
|
"result": ci.get("result"), "evidence_ref": ci.get("evidence_ref")}
|
|
target = ci.get("matched_canonical_id")
|
|
if target and target in nodes:
|
|
represents[node_id] = target
|
|
edge(node_id, target, "represents", [{"kind": "reconciliation", "ref": ci.get("evidence_ref") or node_id}],
|
|
MATCH.get(ci.get("match_basis") or "name", MATCH["name"]))
|
|
cloud_pairs: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
|
|
for e in edges:
|
|
if nodes.get(e["source"], {}).get("kind") == "resource" and nodes.get(e["target"], {}).get("kind") == "resource":
|
|
cloud_pairs.setdefault((e["source"], e["target"]), []).append(e)
|
|
comparison: List[Dict[str, Any]] = []
|
|
matched_pairs = set()
|
|
for rel in payload.get("cmdb_relationships") or []:
|
|
parent, child = f"cmdb:{rel['parent_ref']}", f"cmdb:{rel['child_ref']}"
|
|
if parent not in nodes or child not in nodes:
|
|
continue
|
|
cmdb_edge = edge(parent, child, rel["type"], [{"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}], CMDB)
|
|
pair = (represents.get(parent), represents.get(child))
|
|
item = {"cmdb_edge": cmdb_edge["edge_id"], "parent_ci": rel["parent_ref"], "child_ci": rel["child_ref"],
|
|
"source": pair[0], "target": pair[1],
|
|
"source_resource": nodes.get(pair[0] or "", {}).get("resource_id"),
|
|
"target_resource": nodes.get(pair[1] or "", {}).get("resource_id")}
|
|
if not all(pair):
|
|
comparison.append({**item, "result": "not_comparable",
|
|
"reason": "an end has no matched resource, so the cloud can't confirm or deny it"})
|
|
elif pair in cloud_pairs:
|
|
matched_pairs.add(pair)
|
|
for cloud in cloud_pairs[pair]:
|
|
cloud["evidence"].append({"kind": "cmdb_relationship", "ref": rel["evidence_ref"]})
|
|
cloud["confidence"] = max(cloud["confidence"], CONFIRMED)
|
|
comparison.append({**item, "result": "confirmed", "cloud_edges": [c["edge_id"] for c in cloud_pairs[pair]]})
|
|
else:
|
|
comparison.append({**item, "result": "cmdb_only"})
|
|
represented = set(represents.values())
|
|
if payload.get("cmdb_cis"):
|
|
for pair, cloud in cloud_pairs.items():
|
|
if pair[0] in represented and pair[1] in represented and pair not in matched_pairs:
|
|
comparison.append({"result": "discovered_only", "cloud_edges": [c["edge_id"] for c in cloud],
|
|
"source": pair[0], "target": pair[1],
|
|
"source_resource": nodes[pair[0]].get("resource_id"),
|
|
"target_resource": nodes[pair[1]].get("resource_id"),
|
|
"parent_ci": next((n[5:] for n, r in represents.items() if r == pair[0]), None),
|
|
"child_ci": next((n[5:] for n, r in represents.items() if r == pair[1]), None)})
|
|
|
|
# fan-in and fan-out over dependency edges (represents is a link, not a dependency)
|
|
dependency_edges = [e for e in edges if e["type"] != "represents"]
|
|
for node in nodes.values():
|
|
node["fan_out"] = sum(1 for e in dependency_edges if e["source"] == node["id"])
|
|
node["fan_in"] = sum(1 for e in dependency_edges if e["target"] == node["id"])
|
|
node["depended_on"] = node["fan_in"] > 0
|
|
node["retire_constraint"] = "cannot_retire_without_mitigation" if node["depended_on"] else "none"
|
|
orphans = [n["id"] for n in nodes.values() if n["kind"] == "resource" and not n["fan_in"] and not n["fan_out"]]
|
|
counts = {kind: sum(1 for n in nodes.values() if n["kind"] == kind) for kind in ("resource", "external", "ci")}
|
|
recorded = sum(len(f.get("dependencies") or []) for f in payload.get("findings") or [])
|
|
by_result = {r: sum(1 for c in comparison if c["result"] == r)
|
|
for r in ("confirmed", "cmdb_only", "discovered_only", "not_comparable")}
|
|
note = (f"Configured links only; traffic isn't known. {recorded} dependencies were recorded by the scan: "
|
|
f"{sum(1 for e in dependency_edges if nodes[e['source']]['kind'] == 'resource')} edges, "
|
|
f"{counts['external']} outside the scope as external nodes, {len(folded)} to a resource's own components. "
|
|
f"{counts['ci']} CMDB CIs, {len(payload.get('cmdb_relationships') or [])} CMDB relationships: "
|
|
+ ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in by_result.items()) + ".")
|
|
return {"graph_id": f"GRAPH-{payload.get('run_id') or payload['engagement_id']}",
|
|
"engagement_id": payload["engagement_id"], "edges": edges, "nodes": list(nodes.values()),
|
|
"orphans": orphans, "validation_queue": [e["edge_id"] for e in edges if e["validation"] != "auto_approved"],
|
|
"folded_dependencies": folded, "dangling": list(payload.get("cmdb_dangling") or []),
|
|
"graph_comparison": comparison, "note": note}
|