Files
Jonathan Boniface 9e503885cc ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

221 lines
13 KiB
Python

"""A live run's dependency graph, built in code (ADR 0019): one graph, three kinds of node.
- `resource`: a scope entry the hyperscaler showed (its inventory record).
- `external`: a resource a scope entry depends on that lies outside the scope root, or wasn't inventoried.
- `ci`: a CMDB CI the reconciliation judged (ADR 0018).
Every edge runs parent (`source`) to child (`target`), whatever the kinds:
- resource -> resource or external: a dependency the scan recorded in a finding's `dependencies`;
- ci -> ci: a relationship the CMDB records;
- ci -> resource (`represents`): the reconciliation's match of a CI to its resource.
The comparison walks this one graph: a CMDB relationship whose two CIs represent two resources the cloud also links,
in the same direction, is `confirmed`; one the cloud doesn't show is `cmdb_only`; a cloud dependency between two
represented resources the CMDB doesn't record is `discovered_only`. No model is called.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional, Tuple
from app import scope as sc
AZURE_CONTAINER = re.compile(r"^(/subscriptions/[^/]+(?:/resourcegroups/[^/]+)?)", re.I)
GCP_PROJECT = re.compile(r"/projects/([^/]+)(?:/|$)")
NETWORK = ("microsoft.network/virtualnetworks", "microsoft.network/virtualnetworks/subnets",
"microsoft.network/loadbalancers", "microsoft.network/publicipaddresses",
"microsoft.network/applicationgateways", "microsoft.network/networksecuritygroups",
"microsoft.network/privatednszones", "microsoft.network/dnszones",
"compute.googleapis.com/network", "compute.googleapis.com/subnetwork", "compute.googleapis.com/address",
"compute.googleapis.com/forwardingrule")
HOSTING = ("microsoft.compute/disks", "microsoft.compute/virtualmachines", "microsoft.compute/virtualmachinescalesets",
"microsoft.containerservice/managedclusters", "microsoft.web/serverfarms",
"compute.googleapis.com/disk", "compute.googleapis.com/instance", "container.googleapis.com/cluster")
SHARED = ("microsoft.keyvault/vaults", "microsoft.managedidentity/userassignedidentities",
"secretmanager.googleapis.com/secret", "cloudkms.googleapis.com/cryptokey")
MESSAGING = ("microsoft.servicebus/namespaces", "microsoft.eventhub/namespaces", "microsoft.eventgrid/topics",
"microsoft.eventgrid/systemtopics", "pubsub.googleapis.com/topic")
CLOUD = {"confidence": 0.8, "validation": "auto_approved"} # the provider's own record of the link (D3)
CMDB = {"confidence": 0.7, "validation": "sme_review_recommended"} # what the CMDB says, not seen in the cloud
CONFIRMED = 0.95 # a cloud edge the CMDB also records
MATCH = {"cloud_resource_id": {"confidence": 0.95, "validation": "auto_approved"},
"name": {"confidence": 0.7, "validation": "sme_review_recommended"}}
def key(provider: str, rid: Any) -> str:
rid = str(rid or "").strip().rstrip("/")
return rid.lower() if provider == "azure" else rid
def type_of(provider: str, rid: str) -> str:
"""A resource's type from its id: microsoft.network/virtualnetworks/subnets, compute.googleapis.com/disk."""
if provider == "azure":
tail = rid.split("/providers/")[-1].split("/") if "/providers/" in rid else []
if len(tail) < 3:
return ""
return "/".join([tail[0]] + tail[1::2][:max(1, (len(tail) - 1) // 2)]).lower()
parts = rid.lstrip("/").split("/")
if len(parts) < 3:
return ""
collection = parts[-2]
singular = collection[:-1] if collection.endswith("s") else collection
return f"{parts[0]}/{singular}".lower()
def container(provider: str, rid: str) -> Optional[str]:
if provider == "azure":
found = AZURE_CONTAINER.match(rid.strip())
return found.group(1).lower() if found else None
found = GCP_PROJECT.search(rid)
return f"projects/{found.group(1)}" if found else None
def edge_type(provider: str, rtype: str) -> str:
if rtype in NETWORK or rtype.endswith(("/subnets", "/subnetwork")):
return "network_attachment"
if rtype in sc.DATA_STORES.get(provider, ()):
return "app_to_db"
if rtype in HOSTING:
return "hosted_on"
if rtype in SHARED:
return "app_to_shared_service"
if rtype in MESSAGING:
return "middleware"
return "unclassified"
def _dependency_id(dep: Any) -> str:
if isinstance(dep, dict):
return str(dep.get("id") or dep.get("target") or dep.get("resource_id") or dep.get("scope_entry_id") or "")
return str(dep or "")
def build(payload: Dict[str, Any]) -> Dict[str, Any]:
provider = payload["provider"]
records = [r for r in payload.get("records") or [] if r.get("scope_entry_id")]
by_resource = {key(provider, r["scope_entry_id"]): r for r in records}
record_containers = {container(provider, r["scope_entry_id"]) for r in records}
nodes: Dict[str, Dict[str, Any]] = {}
edges: List[Dict[str, Any]] = []
folded: List[Dict[str, Any]] = []
seen: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
for r in records:
nodes[r["canonical_id"]] = {"id": r["canonical_id"], "canonical_id": r["canonical_id"], "kind": "resource",
"name": r.get("hostname"), "resource_id": r["scope_entry_id"]}
def edge(source: str, target: str, kind: str, evidence: List[Dict[str, str]], weight: Dict[str, Any]
) -> Dict[str, Any]:
found = seen.get((source, target, kind))
if found:
found["evidence"] += [e for e in evidence if e not in found["evidence"]]
return found
made = {"edge_id": f"E-{len(edges) + 1:03d}", "source": source, "target": target, "type": kind,
"protocol_port": None, "evidence": list(evidence), **weight}
edges.append(made)
seen[(source, target, kind)] = made
return made
# resource -> resource / external: what the scan recorded
for finding in payload.get("findings") or []:
source = by_resource.get(key(provider, finding.get("scope_entry_id")))
if not source:
continue
evidence = [{"kind": "config", "ref": finding.get("evidence_ref") or finding["scope_entry_id"]}]
for dep in finding.get("dependencies") or []:
rid = _dependency_id(dep)
if not rid or key(provider, rid) == key(provider, source["scope_entry_id"]):
continue
rtype = type_of(provider, rid)
target = by_resource.get(key(provider, rid))
if target:
edge(source["canonical_id"], target["canonical_id"], edge_type(provider, rtype), evidence, CLOUD)
continue
if rtype in sc.FOLDED.get(provider, {}):
folded.append({"source": source["canonical_id"], "target": rid, "type": rtype,
"rule": sc.FOLDED[provider][rtype], "evidence_ref": evidence[0]["ref"]})
continue
node = nodes.setdefault(rid, {"id": rid, "canonical_id": rid, "kind": "external",
"name": rid.rstrip("/").split("/")[-1], "type": rtype,
"container": container(provider, rid),
"reason": ("not_inventoried" if container(provider, rid) in record_containers
else "outside_scope_root"),
"referenced_by": []})
if source["canonical_id"] not in node["referenced_by"]:
node["referenced_by"].append(source["canonical_id"])
edge(source["canonical_id"], rid, edge_type(provider, rtype), evidence, CLOUD)
# ci nodes, ci -> resource (represents), ci -> ci (relationships)
represents: Dict[str, str] = {}
for ci in payload.get("cmdb_cis") or []:
node_id = f"cmdb:{ci['ci_ref']}"
nodes[node_id] = {"id": node_id, "canonical_id": node_id, "kind": "ci", "ci_ref": ci["ci_ref"],
"name": ci.get("name"), "ci_class": ci.get("ci_class"), "lifecycle": ci.get("lifecycle"),
"result": ci.get("result"), "evidence_ref": ci.get("evidence_ref")}
target = ci.get("matched_canonical_id")
if target and target in nodes:
represents[node_id] = target
edge(node_id, target, "represents", [{"kind": "reconciliation", "ref": ci.get("evidence_ref") or node_id}],
MATCH.get(ci.get("match_basis") or "name", MATCH["name"]))
cloud_pairs: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
for e in edges:
if nodes.get(e["source"], {}).get("kind") == "resource" and nodes.get(e["target"], {}).get("kind") == "resource":
cloud_pairs.setdefault((e["source"], e["target"]), []).append(e)
comparison: List[Dict[str, Any]] = []
matched_pairs = set()
for rel in payload.get("cmdb_relationships") or []:
parent, child = f"cmdb:{rel['parent_ref']}", f"cmdb:{rel['child_ref']}"
if parent not in nodes or child not in nodes:
continue
cmdb_edge = edge(parent, child, rel["type"], [{"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}], CMDB)
pair = (represents.get(parent), represents.get(child))
item = {"cmdb_edge": cmdb_edge["edge_id"], "parent_ci": rel["parent_ref"], "child_ci": rel["child_ref"],
"source": pair[0], "target": pair[1],
"source_resource": nodes.get(pair[0] or "", {}).get("resource_id"),
"target_resource": nodes.get(pair[1] or "", {}).get("resource_id")}
if not all(pair):
comparison.append({**item, "result": "not_comparable",
"reason": "an end has no matched resource, so the cloud can't confirm or deny it"})
elif pair in cloud_pairs:
matched_pairs.add(pair)
for cloud in cloud_pairs[pair]:
cloud["evidence"].append({"kind": "cmdb_relationship", "ref": rel["evidence_ref"]})
cloud["confidence"] = max(cloud["confidence"], CONFIRMED)
comparison.append({**item, "result": "confirmed", "cloud_edges": [c["edge_id"] for c in cloud_pairs[pair]]})
else:
comparison.append({**item, "result": "cmdb_only"})
represented = set(represents.values())
if payload.get("cmdb_cis"):
for pair, cloud in cloud_pairs.items():
if pair[0] in represented and pair[1] in represented and pair not in matched_pairs:
comparison.append({"result": "discovered_only", "cloud_edges": [c["edge_id"] for c in cloud],
"source": pair[0], "target": pair[1],
"source_resource": nodes[pair[0]].get("resource_id"),
"target_resource": nodes[pair[1]].get("resource_id"),
"parent_ci": next((n[5:] for n, r in represents.items() if r == pair[0]), None),
"child_ci": next((n[5:] for n, r in represents.items() if r == pair[1]), None)})
# fan-in and fan-out over dependency edges (represents is a link, not a dependency)
dependency_edges = [e for e in edges if e["type"] != "represents"]
for node in nodes.values():
node["fan_out"] = sum(1 for e in dependency_edges if e["source"] == node["id"])
node["fan_in"] = sum(1 for e in dependency_edges if e["target"] == node["id"])
node["depended_on"] = node["fan_in"] > 0
node["retire_constraint"] = "cannot_retire_without_mitigation" if node["depended_on"] else "none"
orphans = [n["id"] for n in nodes.values() if n["kind"] == "resource" and not n["fan_in"] and not n["fan_out"]]
counts = {kind: sum(1 for n in nodes.values() if n["kind"] == kind) for kind in ("resource", "external", "ci")}
recorded = sum(len(f.get("dependencies") or []) for f in payload.get("findings") or [])
by_result = {r: sum(1 for c in comparison if c["result"] == r)
for r in ("confirmed", "cmdb_only", "discovered_only", "not_comparable")}
note = (f"Configured links only; traffic isn't known. {recorded} dependencies were recorded by the scan: "
f"{sum(1 for e in dependency_edges if nodes[e['source']]['kind'] == 'resource')} edges, "
f"{counts['external']} outside the scope as external nodes, {len(folded)} to a resource's own components. "
f"{counts['ci']} CMDB CIs, {len(payload.get('cmdb_relationships') or [])} CMDB relationships: "
+ ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in by_result.items()) + ".")
return {"graph_id": f"GRAPH-{payload.get('run_id') or payload['engagement_id']}",
"engagement_id": payload["engagement_id"], "edges": edges, "nodes": list(nodes.values()),
"orphans": orphans, "validation_queue": [e["edge_id"] for e in edges if e["validation"] != "auto_approved"],
"folded_dependencies": folded, "dangling": list(payload.get("cmdb_dangling") or []),
"graph_comparison": comparison, "note": note}