ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
"""A live run's applications and asset owners, grouped in code from the CMDB, tags and containers (ADR 0021).
|
||||
|
||||
The order is the product owner's (2026-11-05):
|
||||
|
||||
1. **CMDB business service.** A record matched to a CMDB CI that names a business service joins that application.
|
||||
2. **Application tag.** Otherwise, the resource's own application tag (`Application`, `app`, `application_name`,
|
||||
`business_service`, `service`, any case).
|
||||
- **Tag alias:** when every CMDB-grouped member of one application carries the same application tag, that tag
|
||||
names the same application. A tag-only record with that tag joins it, flagged for review.
|
||||
3. **Container.** Otherwise, the resource group (Azure) or project (Google Cloud), flagged for review: sharing a
|
||||
container is weak evidence of sharing an application.
|
||||
4. **Model.** Only records with none of the above are sent to the `propose_application_groups` sub-agent. Its groups
|
||||
are judged here, every one is flagged for review, and a reply that fails the checks leaves those records ungrouped.
|
||||
|
||||
No record is in two applications. An owner comes from the matched CI (`owner`, else `support_group`), else an
|
||||
`owner` tag; an application's owner is set only when all its members that have one agree.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from app.graph import container
|
||||
|
||||
APP_TAGS = ("application", "app", "application_name", "app_name", "business_service", "service")
|
||||
OWNER_TAGS = ("owner", "owned_by", "application_owner", "app_owner")
|
||||
INSTRUCTION = re.compile(r"\b(ignore|disregard)\b.{0,40}\b(instruction|prompt|rule)s?\b", re.I)
|
||||
MODEL_BATCH = 50
|
||||
|
||||
|
||||
def _tag(record: Dict[str, Any], names: Tuple[str, ...]) -> Optional[Tuple[str, str]]:
|
||||
tags = (record.get("provider_attributes") or {}).get("tags") or (record.get("provider_attributes") or {}).get("labels") or {}
|
||||
if not isinstance(tags, dict):
|
||||
return None
|
||||
lowered = {str(k).lower(): (k, v) for k, v in tags.items()}
|
||||
for name in names:
|
||||
if name in lowered and isinstance(lowered[name][1], str) and lowered[name][1].strip():
|
||||
return lowered[name][0], lowered[name][1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _evidence(record: Dict[str, Any]) -> List[str]:
|
||||
return [s["evidence_ref"] for s in record.get("sources") or [] if s.get("evidence_ref")]
|
||||
|
||||
|
||||
def _container_name(provider: str, record: Dict[str, Any]) -> Optional[str]:
|
||||
found = container(provider, record.get("scope_entry_id") or "")
|
||||
return found.rstrip("/").split("/")[-1] if found else None
|
||||
|
||||
|
||||
def owners(records: List[Dict[str, Any]], cis: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
by_record = {c["matched_canonical_id"]: c for c in cis if c.get("result") == "matched" and c.get("matched_canonical_id")}
|
||||
found = []
|
||||
for record in records:
|
||||
rid = record["canonical_id"]
|
||||
ci = by_record.get(rid)
|
||||
if ci and (ci.get("owner") or ci.get("support_group")):
|
||||
found.append({"canonical_id": rid, "owner": ci.get("owner") or ci["support_group"], "source": "cmdb",
|
||||
"ci_ref": ci["ci_ref"], "evidence_ref": ci.get("evidence_ref")})
|
||||
continue
|
||||
tag = _tag(record, OWNER_TAGS)
|
||||
if tag:
|
||||
found.append({"canonical_id": rid, "owner": tag[1], "source": "tag", "ci_ref": None,
|
||||
"evidence_ref": (_evidence(record) or [None])[0]})
|
||||
return found
|
||||
|
||||
|
||||
def _judge(reply: str, leftovers: Dict[str, Dict[str, Any]]) -> Optional[List[Dict[str, Any]]]:
|
||||
"""The model's groups, or None when any check fails."""
|
||||
try:
|
||||
groups = json.loads(reply).get("groups")
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
if not isinstance(groups, list):
|
||||
return None
|
||||
seen, judged = set(), []
|
||||
for group in groups:
|
||||
if not isinstance(group, dict):
|
||||
return None
|
||||
name, members, reason = group.get("name"), group.get("members"), group.get("reason")
|
||||
if not (isinstance(name, str) and name.strip() and len(name) <= 80 and not INSTRUCTION.search(name)):
|
||||
return None
|
||||
if not (isinstance(members, list) and members and all(m in leftovers and m not in seen for m in members)):
|
||||
return None
|
||||
seen.update(members)
|
||||
judged.append({"name": name.strip(), "members": members, "reason": str(reason or "")[:300]})
|
||||
return judged
|
||||
|
||||
|
||||
def _prompt(leftovers: Dict[str, Dict[str, Any]]) -> str:
|
||||
rows = [{"canonical_id": r["canonical_id"], "name": r.get("hostname"), "asset_class": r.get("asset_class"),
|
||||
"resource_type": (r.get("provider_attributes") or {}).get("type"), "environment": r.get("environment")}
|
||||
for r in leftovers.values()]
|
||||
return json.dumps({"assets": rows}, ensure_ascii=False)
|
||||
|
||||
|
||||
async def group(payload: Dict[str, Any], ask: Optional[Callable[[str], Awaitable[str]]] = None) -> Dict[str, Any]:
|
||||
provider = payload["provider"]
|
||||
records = [r for r in payload.get("records") or [] if r.get("canonical_id")]
|
||||
cis = payload.get("cmdb_cis") or []
|
||||
by_record = {c["matched_canonical_id"]: c for c in cis if c.get("result") == "matched" and c.get("matched_canonical_id")}
|
||||
apps: Dict[Tuple[str, str], Dict[str, Any]] = {}
|
||||
placed: Dict[str, Tuple[str, str]] = {}
|
||||
|
||||
def join(key: Tuple[str, str], record: Dict[str, Any], basis: str, evidence: List[str], review: Optional[str]):
|
||||
app = apps.setdefault(key, {"name": key[1], "basis": basis, "members": [], "evidence_refs": [],
|
||||
"review_required": False, "review_reasons": []})
|
||||
app["members"].append(record["canonical_id"])
|
||||
app["evidence_refs"] += [e for e in evidence if e and e not in app["evidence_refs"]]
|
||||
if review:
|
||||
app["review_required"] = True
|
||||
if review not in app["review_reasons"]:
|
||||
app["review_reasons"].append(review)
|
||||
placed[record["canonical_id"]] = key
|
||||
|
||||
rest = []
|
||||
for record in records: # 1. CMDB business service
|
||||
ci = by_record.get(record["canonical_id"])
|
||||
if ci and ci.get("business_service"):
|
||||
join(("cmdb", ci["business_service"]), record, "cmdb_business_service", [ci.get("evidence_ref")], None)
|
||||
else:
|
||||
rest.append(record)
|
||||
|
||||
# Tag aliases: an application whose CMDB-grouped members all carry the same application tag.
|
||||
aliases: Dict[str, Tuple[str, str]] = {}
|
||||
for key, app in apps.items():
|
||||
values = {(_tag(r, APP_TAGS) or (None, None))[1] for r in records if r["canonical_id"] in app["members"]}
|
||||
if len(values) == 1 and None not in values:
|
||||
value = values.pop()
|
||||
aliases[value] = key if value not in aliases else ("", "") # two applications claim it: no alias
|
||||
aliases = {v: k for v, k in aliases.items() if k != ("", "")}
|
||||
|
||||
leftover = []
|
||||
for record in rest: # 2. application tag, joined to a CMDB application when it is that application's alias
|
||||
tag = _tag(record, APP_TAGS)
|
||||
if tag and tag[1] in aliases:
|
||||
join(aliases[tag[1]], record, "cmdb_business_service", _evidence(record),
|
||||
f"Joined by its {tag[0]} tag '{tag[1]}', which every CMDB-grouped member of this application carries")
|
||||
elif tag:
|
||||
join(("tag", tag[1]), record, "tag", _evidence(record), None)
|
||||
else:
|
||||
leftover.append(record)
|
||||
|
||||
unplaced = {}
|
||||
for record in leftover: # 3. container
|
||||
name = _container_name(provider, record)
|
||||
if name:
|
||||
join(("container", name), record, "container", _evidence(record),
|
||||
"Grouped only by sharing a resource group or project")
|
||||
else:
|
||||
unplaced[record["canonical_id"]] = record
|
||||
|
||||
model_note = None
|
||||
if unplaced and ask is not None: # 4. the model, judged here
|
||||
ids = list(unplaced)
|
||||
for start in range(0, len(ids), MODEL_BATCH):
|
||||
batch = {i: unplaced[i] for i in ids[start:start + MODEL_BATCH]}
|
||||
try:
|
||||
judged = _judge(await ask(_prompt(batch)), batch)
|
||||
except Exception as exc: # a failed call leaves the batch ungrouped, declared
|
||||
judged, model_note = None, f"the grouping model failed: {type(exc).__name__}"
|
||||
if judged is None:
|
||||
model_note = model_note or "the grouping model's reply failed the checks, so its assets stay ungrouped"
|
||||
continue
|
||||
for proposal in judged:
|
||||
for member in proposal["members"]:
|
||||
join(("model", proposal["name"]), batch[member], "model", _evidence(batch[member]),
|
||||
"Proposed by the model from names and types only: " + (proposal["reason"] or "no reason given"))
|
||||
|
||||
asset_owners = owners(records, cis)
|
||||
owner_of = {o["canonical_id"]: o["owner"] for o in asset_owners}
|
||||
out = []
|
||||
for number, (key, app) in enumerate(sorted(apps.items(), key=lambda kv: (kv[1]["basis"], kv[0][1])), start=1):
|
||||
named = {owner_of[m] for m in app["members"] if m in owner_of}
|
||||
out.append({"application_id": f"APP-{number:03d}", "name": app["name"], "basis": app["basis"],
|
||||
"members": app["members"], "owner": named.pop() if len(named) == 1 else None,
|
||||
"owners_disagree": len(named) > 1, "business_service": key[1] if key[0] == "cmdb" else None,
|
||||
"evidence_refs": app["evidence_refs"], "review_required": app["review_required"],
|
||||
"review_reasons": app["review_reasons"]})
|
||||
ungrouped = sorted(r["canonical_id"] for r in records if r["canonical_id"] not in placed)
|
||||
reviewed = sum(1 for a in out if a["review_required"])
|
||||
note = (f"{len(out)} applications from {len(placed)} of {len(records)} records; {reviewed} need review; "
|
||||
f"{len(ungrouped)} ungrouped; {len(asset_owners)} assets with an owner.")
|
||||
if model_note:
|
||||
note += f" Note: {model_note}."
|
||||
return {"applications": out, "asset_owners": asset_owners, "ungrouped_assets": ungrouped, "applications_note": note}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""A live run's dependency graph (ADR 0019) and its applications and asset owners (ADR 0021) are built in code from
|
||||
the scan and the CMDB; the sample estate keeps the agent loop and the alz-dependency tools."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import jsonschema
|
||||
from kyndryl_agent_runtime import ask_sub_agent
|
||||
from kyndryl_agent_runtime.request_cycle import Reply
|
||||
|
||||
from app import applications, graph
|
||||
|
||||
OUTPUT_SCHEMA = json.loads((Path(__file__).resolve().parents[1] / "schemas" / "output.json").read_text())
|
||||
|
||||
|
||||
class GraphInvalid(RuntimeError):
|
||||
"""The graph broke its output schema: a Reply can't be repaired by the output contract, so the step fails."""
|
||||
|
||||
|
||||
async def build(payload: dict, ask) -> dict:
|
||||
result = graph.build(payload)
|
||||
result.update(await applications.group(payload, ask))
|
||||
errors = sorted(jsonschema.Draft202012Validator(OUTPUT_SCHEMA).iter_errors(result), key=lambda e: list(e.path))
|
||||
if errors:
|
||||
raise GraphInvalid(f"the dependency graph broke its output schema: {errors[0].message}")
|
||||
return result
|
||||
|
||||
|
||||
async def pre_process(payload: dict) -> Optional[Any]:
|
||||
if payload.get("provider", "sample") not in ("azure", "gcp"):
|
||||
return None # the Meridian sample estate: prompt.md and the alz-dependency tools, unchanged
|
||||
|
||||
async def ask(prompt: str) -> str:
|
||||
return await ask_sub_agent("propose_application_groups", prompt)
|
||||
|
||||
return Reply(json.dumps(await build(payload, ask), ensure_ascii=False))
|
||||
@@ -0,0 +1,220 @@
|
||||
"""A live run's dependency graph, built in code (ADR 0019): one graph, three kinds of node.
|
||||
|
||||
- `resource`: a scope entry the hyperscaler showed (its inventory record).
|
||||
- `external`: a resource a scope entry depends on that lies outside the scope root, or wasn't inventoried.
|
||||
- `ci`: a CMDB CI the reconciliation judged (ADR 0018).
|
||||
|
||||
Every edge runs parent (`source`) to child (`target`), whatever the kinds:
|
||||
|
||||
- resource -> resource or external: a dependency the scan recorded in a finding's `dependencies`;
|
||||
- ci -> ci: a relationship the CMDB records;
|
||||
- ci -> resource (`represents`): the reconciliation's match of a CI to its resource.
|
||||
|
||||
The comparison walks this one graph: a CMDB relationship whose two CIs represent two resources the cloud also links,
|
||||
in the same direction, is `confirmed`; one the cloud doesn't show is `cmdb_only`; a cloud dependency between two
|
||||
represented resources the CMDB doesn't record is `discovered_only`. No model is called.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app import scope as sc
|
||||
|
||||
AZURE_CONTAINER = re.compile(r"^(/subscriptions/[^/]+(?:/resourcegroups/[^/]+)?)", re.I)
|
||||
GCP_PROJECT = re.compile(r"/projects/([^/]+)(?:/|$)")
|
||||
NETWORK = ("microsoft.network/virtualnetworks", "microsoft.network/virtualnetworks/subnets",
|
||||
"microsoft.network/loadbalancers", "microsoft.network/publicipaddresses",
|
||||
"microsoft.network/applicationgateways", "microsoft.network/networksecuritygroups",
|
||||
"microsoft.network/privatednszones", "microsoft.network/dnszones",
|
||||
"compute.googleapis.com/network", "compute.googleapis.com/subnetwork", "compute.googleapis.com/address",
|
||||
"compute.googleapis.com/forwardingrule")
|
||||
HOSTING = ("microsoft.compute/disks", "microsoft.compute/virtualmachines", "microsoft.compute/virtualmachinescalesets",
|
||||
"microsoft.containerservice/managedclusters", "microsoft.web/serverfarms",
|
||||
"compute.googleapis.com/disk", "compute.googleapis.com/instance", "container.googleapis.com/cluster")
|
||||
SHARED = ("microsoft.keyvault/vaults", "microsoft.managedidentity/userassignedidentities",
|
||||
"secretmanager.googleapis.com/secret", "cloudkms.googleapis.com/cryptokey")
|
||||
MESSAGING = ("microsoft.servicebus/namespaces", "microsoft.eventhub/namespaces", "microsoft.eventgrid/topics",
|
||||
"microsoft.eventgrid/systemtopics", "pubsub.googleapis.com/topic")
|
||||
CLOUD = {"confidence": 0.8, "validation": "auto_approved"} # the provider's own record of the link (D3)
|
||||
CMDB = {"confidence": 0.7, "validation": "sme_review_recommended"} # what the CMDB says, not seen in the cloud
|
||||
CONFIRMED = 0.95 # a cloud edge the CMDB also records
|
||||
MATCH = {"cloud_resource_id": {"confidence": 0.95, "validation": "auto_approved"},
|
||||
"name": {"confidence": 0.7, "validation": "sme_review_recommended"}}
|
||||
|
||||
|
||||
def key(provider: str, rid: Any) -> str:
|
||||
rid = str(rid or "").strip().rstrip("/")
|
||||
return rid.lower() if provider == "azure" else rid
|
||||
|
||||
|
||||
def type_of(provider: str, rid: str) -> str:
|
||||
"""A resource's type from its id: microsoft.network/virtualnetworks/subnets, compute.googleapis.com/disk."""
|
||||
if provider == "azure":
|
||||
tail = rid.split("/providers/")[-1].split("/") if "/providers/" in rid else []
|
||||
if len(tail) < 3:
|
||||
return ""
|
||||
return "/".join([tail[0]] + tail[1::2][:max(1, (len(tail) - 1) // 2)]).lower()
|
||||
parts = rid.lstrip("/").split("/")
|
||||
if len(parts) < 3:
|
||||
return ""
|
||||
collection = parts[-2]
|
||||
singular = collection[:-1] if collection.endswith("s") else collection
|
||||
return f"{parts[0]}/{singular}".lower()
|
||||
|
||||
|
||||
def container(provider: str, rid: str) -> Optional[str]:
|
||||
if provider == "azure":
|
||||
found = AZURE_CONTAINER.match(rid.strip())
|
||||
return found.group(1).lower() if found else None
|
||||
found = GCP_PROJECT.search(rid)
|
||||
return f"projects/{found.group(1)}" if found else None
|
||||
|
||||
|
||||
def edge_type(provider: str, rtype: str) -> str:
|
||||
if rtype in NETWORK or rtype.endswith(("/subnets", "/subnetwork")):
|
||||
return "network_attachment"
|
||||
if rtype in sc.DATA_STORES.get(provider, ()):
|
||||
return "app_to_db"
|
||||
if rtype in HOSTING:
|
||||
return "hosted_on"
|
||||
if rtype in SHARED:
|
||||
return "app_to_shared_service"
|
||||
if rtype in MESSAGING:
|
||||
return "middleware"
|
||||
return "unclassified"
|
||||
|
||||
|
||||
def _dependency_id(dep: Any) -> str:
|
||||
if isinstance(dep, dict):
|
||||
return str(dep.get("id") or dep.get("target") or dep.get("resource_id") or dep.get("scope_entry_id") or "")
|
||||
return str(dep or "")
|
||||
|
||||
|
||||
def build(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
provider = payload["provider"]
|
||||
records = [r for r in payload.get("records") or [] if r.get("scope_entry_id")]
|
||||
by_resource = {key(provider, r["scope_entry_id"]): r for r in records}
|
||||
record_containers = {container(provider, r["scope_entry_id"]) for r in records}
|
||||
nodes: Dict[str, Dict[str, Any]] = {}
|
||||
edges: List[Dict[str, Any]] = []
|
||||
folded: List[Dict[str, Any]] = []
|
||||
seen: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
|
||||
|
||||
for r in records:
|
||||
nodes[r["canonical_id"]] = {"id": r["canonical_id"], "canonical_id": r["canonical_id"], "kind": "resource",
|
||||
"name": r.get("hostname"), "resource_id": r["scope_entry_id"]}
|
||||
|
||||
def edge(source: str, target: str, kind: str, evidence: List[Dict[str, str]], weight: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
found = seen.get((source, target, kind))
|
||||
if found:
|
||||
found["evidence"] += [e for e in evidence if e not in found["evidence"]]
|
||||
return found
|
||||
made = {"edge_id": f"E-{len(edges) + 1:03d}", "source": source, "target": target, "type": kind,
|
||||
"protocol_port": None, "evidence": list(evidence), **weight}
|
||||
edges.append(made)
|
||||
seen[(source, target, kind)] = made
|
||||
return made
|
||||
|
||||
# resource -> resource / external: what the scan recorded
|
||||
for finding in payload.get("findings") or []:
|
||||
source = by_resource.get(key(provider, finding.get("scope_entry_id")))
|
||||
if not source:
|
||||
continue
|
||||
evidence = [{"kind": "config", "ref": finding.get("evidence_ref") or finding["scope_entry_id"]}]
|
||||
for dep in finding.get("dependencies") or []:
|
||||
rid = _dependency_id(dep)
|
||||
if not rid or key(provider, rid) == key(provider, source["scope_entry_id"]):
|
||||
continue
|
||||
rtype = type_of(provider, rid)
|
||||
target = by_resource.get(key(provider, rid))
|
||||
if target:
|
||||
edge(source["canonical_id"], target["canonical_id"], edge_type(provider, rtype), evidence, CLOUD)
|
||||
continue
|
||||
if rtype in sc.FOLDED.get(provider, {}):
|
||||
folded.append({"source": source["canonical_id"], "target": rid, "type": rtype,
|
||||
"rule": sc.FOLDED[provider][rtype], "evidence_ref": evidence[0]["ref"]})
|
||||
continue
|
||||
node = nodes.setdefault(rid, {"id": rid, "canonical_id": rid, "kind": "external",
|
||||
"name": rid.rstrip("/").split("/")[-1], "type": rtype,
|
||||
"container": container(provider, rid),
|
||||
"reason": ("not_inventoried" if container(provider, rid) in record_containers
|
||||
else "outside_scope_root"),
|
||||
"referenced_by": []})
|
||||
if source["canonical_id"] not in node["referenced_by"]:
|
||||
node["referenced_by"].append(source["canonical_id"])
|
||||
edge(source["canonical_id"], rid, edge_type(provider, rtype), evidence, CLOUD)
|
||||
|
||||
# ci nodes, ci -> resource (represents), ci -> ci (relationships)
|
||||
represents: Dict[str, str] = {}
|
||||
for ci in payload.get("cmdb_cis") or []:
|
||||
node_id = f"cmdb:{ci['ci_ref']}"
|
||||
nodes[node_id] = {"id": node_id, "canonical_id": node_id, "kind": "ci", "ci_ref": ci["ci_ref"],
|
||||
"name": ci.get("name"), "ci_class": ci.get("ci_class"), "lifecycle": ci.get("lifecycle"),
|
||||
"result": ci.get("result"), "evidence_ref": ci.get("evidence_ref")}
|
||||
target = ci.get("matched_canonical_id")
|
||||
if target and target in nodes:
|
||||
represents[node_id] = target
|
||||
edge(node_id, target, "represents", [{"kind": "reconciliation", "ref": ci.get("evidence_ref") or node_id}],
|
||||
MATCH.get(ci.get("match_basis") or "name", MATCH["name"]))
|
||||
cloud_pairs: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
|
||||
for e in edges:
|
||||
if nodes.get(e["source"], {}).get("kind") == "resource" and nodes.get(e["target"], {}).get("kind") == "resource":
|
||||
cloud_pairs.setdefault((e["source"], e["target"]), []).append(e)
|
||||
comparison: List[Dict[str, Any]] = []
|
||||
matched_pairs = set()
|
||||
for rel in payload.get("cmdb_relationships") or []:
|
||||
parent, child = f"cmdb:{rel['parent_ref']}", f"cmdb:{rel['child_ref']}"
|
||||
if parent not in nodes or child not in nodes:
|
||||
continue
|
||||
cmdb_edge = edge(parent, child, rel["type"], [{"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}], CMDB)
|
||||
pair = (represents.get(parent), represents.get(child))
|
||||
item = {"cmdb_edge": cmdb_edge["edge_id"], "parent_ci": rel["parent_ref"], "child_ci": rel["child_ref"],
|
||||
"source": pair[0], "target": pair[1],
|
||||
"source_resource": nodes.get(pair[0] or "", {}).get("resource_id"),
|
||||
"target_resource": nodes.get(pair[1] or "", {}).get("resource_id")}
|
||||
if not all(pair):
|
||||
comparison.append({**item, "result": "not_comparable",
|
||||
"reason": "an end has no matched resource, so the cloud can't confirm or deny it"})
|
||||
elif pair in cloud_pairs:
|
||||
matched_pairs.add(pair)
|
||||
for cloud in cloud_pairs[pair]:
|
||||
cloud["evidence"].append({"kind": "cmdb_relationship", "ref": rel["evidence_ref"]})
|
||||
cloud["confidence"] = max(cloud["confidence"], CONFIRMED)
|
||||
comparison.append({**item, "result": "confirmed", "cloud_edges": [c["edge_id"] for c in cloud_pairs[pair]]})
|
||||
else:
|
||||
comparison.append({**item, "result": "cmdb_only"})
|
||||
represented = set(represents.values())
|
||||
if payload.get("cmdb_cis"):
|
||||
for pair, cloud in cloud_pairs.items():
|
||||
if pair[0] in represented and pair[1] in represented and pair not in matched_pairs:
|
||||
comparison.append({"result": "discovered_only", "cloud_edges": [c["edge_id"] for c in cloud],
|
||||
"source": pair[0], "target": pair[1],
|
||||
"source_resource": nodes[pair[0]].get("resource_id"),
|
||||
"target_resource": nodes[pair[1]].get("resource_id"),
|
||||
"parent_ci": next((n[5:] for n, r in represents.items() if r == pair[0]), None),
|
||||
"child_ci": next((n[5:] for n, r in represents.items() if r == pair[1]), None)})
|
||||
|
||||
# fan-in and fan-out over dependency edges (represents is a link, not a dependency)
|
||||
dependency_edges = [e for e in edges if e["type"] != "represents"]
|
||||
for node in nodes.values():
|
||||
node["fan_out"] = sum(1 for e in dependency_edges if e["source"] == node["id"])
|
||||
node["fan_in"] = sum(1 for e in dependency_edges if e["target"] == node["id"])
|
||||
node["depended_on"] = node["fan_in"] > 0
|
||||
node["retire_constraint"] = "cannot_retire_without_mitigation" if node["depended_on"] else "none"
|
||||
orphans = [n["id"] for n in nodes.values() if n["kind"] == "resource" and not n["fan_in"] and not n["fan_out"]]
|
||||
counts = {kind: sum(1 for n in nodes.values() if n["kind"] == kind) for kind in ("resource", "external", "ci")}
|
||||
recorded = sum(len(f.get("dependencies") or []) for f in payload.get("findings") or [])
|
||||
by_result = {r: sum(1 for c in comparison if c["result"] == r)
|
||||
for r in ("confirmed", "cmdb_only", "discovered_only", "not_comparable")}
|
||||
note = (f"Configured links only; traffic isn't known. {recorded} dependencies were recorded by the scan: "
|
||||
f"{sum(1 for e in dependency_edges if nodes[e['source']]['kind'] == 'resource')} edges, "
|
||||
f"{counts['external']} outside the scope as external nodes, {len(folded)} to a resource's own components. "
|
||||
f"{counts['ci']} CMDB CIs, {len(payload.get('cmdb_relationships') or [])} CMDB relationships: "
|
||||
+ ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in by_result.items()) + ".")
|
||||
return {"graph_id": f"GRAPH-{payload.get('run_id') or payload['engagement_id']}",
|
||||
"engagement_id": payload["engagement_id"], "edges": edges, "nodes": list(nodes.values()),
|
||||
"orphans": orphans, "validation_queue": [e["edge_id"] for e in edges if e["validation"] != "auto_approved"],
|
||||
"folded_dependencies": folded, "dangling": list(payload.get("cmdb_dangling") or []),
|
||||
"graph_comparison": comparison, "note": note}
|
||||
@@ -0,0 +1,111 @@
|
||||
"""A live run's scope: every resource under the connection's scope root (ADR 0017).
|
||||
|
||||
The planner plans from this listing and the critic checks coverage against its own run of it, so both copies of
|
||||
this file must stay identical (tests/test_scope.py in each agent compares them). No runtime imports.
|
||||
|
||||
Every resource listed ends in exactly one place:
|
||||
- a scope entry: an asset the run scans, inventories and reports;
|
||||
- folded: a child component recorded with its parent's rule (a disk, a network interface, a node pool), never
|
||||
dropped;
|
||||
- structure: the estate's subscriptions, resource groups, projects and folders, which are not assets.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
PAGE_SIZE = 200 # the servers' maximum page
|
||||
MAX_PAGES = 100 # 20,000 resources bounds a runaway; the largest estate seen so far is about 650
|
||||
|
||||
STRUCTURE = {
|
||||
"azure": ("microsoft.resources/subscriptions", "microsoft.resources/subscriptions/resourcegroups",
|
||||
"microsoft.management/managementgroups"),
|
||||
"gcp": ("cloudresourcemanager.googleapis.com/project", "cloudresourcemanager.googleapis.com/folder",
|
||||
"cloudresourcemanager.googleapis.com/organization"),
|
||||
}
|
||||
FOLDED = {
|
||||
"azure": {
|
||||
"microsoft.compute/disks": "a disk is part of the VM it is attached to",
|
||||
"microsoft.network/networkinterfaces": "a network interface is part of the resource it is attached to",
|
||||
"microsoft.compute/virtualmachines/extensions": "an extension is part of its VM",
|
||||
"microsoft.compute/virtualmachinescalesets/virtualmachines": "an instance is part of its scale set",
|
||||
},
|
||||
"gcp": {
|
||||
"compute.googleapis.com/disk": "a disk is part of the instance it is attached to",
|
||||
"container.googleapis.com/nodepool": "a node pool is part of its cluster",
|
||||
},
|
||||
}
|
||||
DATA_STORES = { # the types the servers' list_data_stores kinds cover; each gets a classify_data task
|
||||
"azure": ("microsoft.sql/servers", "microsoft.sql/servers/databases", "microsoft.documentdb/databaseaccounts",
|
||||
"microsoft.storage/storageaccounts", "microsoft.dbforpostgresql/flexibleservers",
|
||||
"microsoft.dbforpostgresql/servers", "microsoft.dbformysql/flexibleservers",
|
||||
"microsoft.dbformysql/servers", "microsoft.cache/redis", "microsoft.cache/redisenterprise"),
|
||||
"gcp": ("sqladmin.googleapis.com/instance", "firestore.googleapis.com/database", "storage.googleapis.com/bucket"),
|
||||
}
|
||||
|
||||
Call = Callable[[str, Dict[str, Any]], Any]
|
||||
|
||||
|
||||
def identity(provider: str, item: Dict[str, Any]) -> Tuple[str, str, str]:
|
||||
"""(id, type in lower case, location) of one listed resource."""
|
||||
if provider == "gcp":
|
||||
return str(item.get("name") or ""), str(item.get("assetType") or "").lower(), str(item.get("location") or "")
|
||||
return str(item.get("id") or ""), str(item.get("type") or "").lower(), str(item.get("location") or "")
|
||||
|
||||
|
||||
def _data(result: Any) -> Dict[str, Any]:
|
||||
if isinstance(result, str):
|
||||
import json
|
||||
result = json.loads(result)
|
||||
if not isinstance(result, dict) or result.get("success") is False:
|
||||
error = (result or {}).get("error") if isinstance(result, dict) else None
|
||||
raise RuntimeError((error or {}).get("message") or "the tool refused the call")
|
||||
return result.get("data") or {}
|
||||
|
||||
|
||||
def list_all(provider: str, call: Call) -> Tuple[List[Dict[str, Any]], List[str], List[str]]:
|
||||
"""(every resource listed, evidence refs, issues). An issue means the listing is incomplete."""
|
||||
tool = f"{provider}_list_resources"
|
||||
items: List[Dict[str, Any]] = []
|
||||
refs: List[str] = []
|
||||
token: Optional[str] = None
|
||||
for _ in range(MAX_PAGES):
|
||||
args: Dict[str, Any] = {"page_size": PAGE_SIZE}
|
||||
if token:
|
||||
args["page_token"] = token
|
||||
try:
|
||||
data = _data(call(tool, args))
|
||||
except Exception as error: # noqa: BLE001 - a refused or failed page is an incomplete listing
|
||||
return items, refs, [f"{tool}: {str(error)[:200]}"]
|
||||
items += list(data.get("items") or [])
|
||||
if data.get("evidence_ref"):
|
||||
refs.append(data["evidence_ref"])
|
||||
token = data.get("next_page_token")
|
||||
if not token:
|
||||
return items, refs, []
|
||||
return items, refs, [f"{tool}: stopped after {MAX_PAGES} pages"]
|
||||
|
||||
|
||||
def account(provider: str, items: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Every listed resource in exactly one place: entries, folded or structure. Repeats are counted once."""
|
||||
entries, folded, structure, seen = [], [], [], set()
|
||||
for item in items:
|
||||
rid, rtype, location = identity(provider, item)
|
||||
if not rid or rid in seen:
|
||||
continue
|
||||
seen.add(rid)
|
||||
if rtype in STRUCTURE.get(provider, ()):
|
||||
structure.append(rid)
|
||||
elif rtype in FOLDED.get(provider, {}):
|
||||
folded.append({"id": rid, "type": rtype, "reason": FOLDED[provider][rtype]})
|
||||
else:
|
||||
entries.append({"id": rid, "type": rtype, "location": location})
|
||||
entries.sort(key=lambda e: e["id"].lower())
|
||||
return {"listed": len(seen), "entries": entries, "folded": folded, "structure": structure}
|
||||
|
||||
|
||||
def region(location: str) -> str:
|
||||
"""A GCP zone such as europe-west2-b is its region europe-west2; anything else is used as it is."""
|
||||
parts = location.split("-")
|
||||
if len(parts) == 3 and len(parts[2]) == 1 and parts[2].isalpha():
|
||||
return "-".join(parts[:2])
|
||||
return location
|
||||
Reference in New Issue
Block a user