ALZ-F01 Discovery: package step, value scorecard, application grouping, cases

- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent f659564208
commit 9e503885cc
211 changed files with 18170 additions and 158 deletions

View File

@@ -0,0 +1,187 @@
"""A live run's applications and asset owners, grouped in code from the CMDB, tags and containers (ADR 0021).
The order is the product owner's (2026-11-05):
1. **CMDB business service.** A record matched to a CMDB CI that names a business service joins that application.
2. **Application tag.** Otherwise, the resource's own application tag (`Application`, `app`, `application_name`,
`business_service`, `service`, any case).
- **Tag alias:** when every CMDB-grouped member of one application carries the same application tag, that tag
names the same application. A tag-only record with that tag joins it, flagged for review.
3. **Container.** Otherwise, the resource group (Azure) or project (Google Cloud), flagged for review: sharing a
container is weak evidence of sharing an application.
4. **Model.** Only records with none of the above are sent to the `propose_application_groups` sub-agent. Its groups
are judged here, every one is flagged for review, and a reply that fails the checks leaves those records ungrouped.
No record is in two applications. An owner comes from the matched CI (`owner`, else `support_group`), else an
`owner` tag; an application's owner is set only when all its members that have one agree.
"""
from __future__ import annotations
import json
import re
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
from app.graph import container
APP_TAGS = ("application", "app", "application_name", "app_name", "business_service", "service")
OWNER_TAGS = ("owner", "owned_by", "application_owner", "app_owner")
INSTRUCTION = re.compile(r"\b(ignore|disregard)\b.{0,40}\b(instruction|prompt|rule)s?\b", re.I)
MODEL_BATCH = 50
def _tag(record: Dict[str, Any], names: Tuple[str, ...]) -> Optional[Tuple[str, str]]:
tags = (record.get("provider_attributes") or {}).get("tags") or (record.get("provider_attributes") or {}).get("labels") or {}
if not isinstance(tags, dict):
return None
lowered = {str(k).lower(): (k, v) for k, v in tags.items()}
for name in names:
if name in lowered and isinstance(lowered[name][1], str) and lowered[name][1].strip():
return lowered[name][0], lowered[name][1].strip()
return None
def _evidence(record: Dict[str, Any]) -> List[str]:
return [s["evidence_ref"] for s in record.get("sources") or [] if s.get("evidence_ref")]
def _container_name(provider: str, record: Dict[str, Any]) -> Optional[str]:
found = container(provider, record.get("scope_entry_id") or "")
return found.rstrip("/").split("/")[-1] if found else None
def owners(records: List[Dict[str, Any]], cis: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
by_record = {c["matched_canonical_id"]: c for c in cis if c.get("result") == "matched" and c.get("matched_canonical_id")}
found = []
for record in records:
rid = record["canonical_id"]
ci = by_record.get(rid)
if ci and (ci.get("owner") or ci.get("support_group")):
found.append({"canonical_id": rid, "owner": ci.get("owner") or ci["support_group"], "source": "cmdb",
"ci_ref": ci["ci_ref"], "evidence_ref": ci.get("evidence_ref")})
continue
tag = _tag(record, OWNER_TAGS)
if tag:
found.append({"canonical_id": rid, "owner": tag[1], "source": "tag", "ci_ref": None,
"evidence_ref": (_evidence(record) or [None])[0]})
return found
def _judge(reply: str, leftovers: Dict[str, Dict[str, Any]]) -> Optional[List[Dict[str, Any]]]:
"""The model's groups, or None when any check fails."""
try:
groups = json.loads(reply).get("groups")
except (ValueError, AttributeError):
return None
if not isinstance(groups, list):
return None
seen, judged = set(), []
for group in groups:
if not isinstance(group, dict):
return None
name, members, reason = group.get("name"), group.get("members"), group.get("reason")
if not (isinstance(name, str) and name.strip() and len(name) <= 80 and not INSTRUCTION.search(name)):
return None
if not (isinstance(members, list) and members and all(m in leftovers and m not in seen for m in members)):
return None
seen.update(members)
judged.append({"name": name.strip(), "members": members, "reason": str(reason or "")[:300]})
return judged
def _prompt(leftovers: Dict[str, Dict[str, Any]]) -> str:
rows = [{"canonical_id": r["canonical_id"], "name": r.get("hostname"), "asset_class": r.get("asset_class"),
"resource_type": (r.get("provider_attributes") or {}).get("type"), "environment": r.get("environment")}
for r in leftovers.values()]
return json.dumps({"assets": rows}, ensure_ascii=False)
async def group(payload: Dict[str, Any], ask: Optional[Callable[[str], Awaitable[str]]] = None) -> Dict[str, Any]:
provider = payload["provider"]
records = [r for r in payload.get("records") or [] if r.get("canonical_id")]
cis = payload.get("cmdb_cis") or []
by_record = {c["matched_canonical_id"]: c for c in cis if c.get("result") == "matched" and c.get("matched_canonical_id")}
apps: Dict[Tuple[str, str], Dict[str, Any]] = {}
placed: Dict[str, Tuple[str, str]] = {}
def join(key: Tuple[str, str], record: Dict[str, Any], basis: str, evidence: List[str], review: Optional[str]):
app = apps.setdefault(key, {"name": key[1], "basis": basis, "members": [], "evidence_refs": [],
"review_required": False, "review_reasons": []})
app["members"].append(record["canonical_id"])
app["evidence_refs"] += [e for e in evidence if e and e not in app["evidence_refs"]]
if review:
app["review_required"] = True
if review not in app["review_reasons"]:
app["review_reasons"].append(review)
placed[record["canonical_id"]] = key
rest = []
for record in records: # 1. CMDB business service
ci = by_record.get(record["canonical_id"])
if ci and ci.get("business_service"):
join(("cmdb", ci["business_service"]), record, "cmdb_business_service", [ci.get("evidence_ref")], None)
else:
rest.append(record)
# Tag aliases: an application whose CMDB-grouped members all carry the same application tag.
aliases: Dict[str, Tuple[str, str]] = {}
for key, app in apps.items():
values = {(_tag(r, APP_TAGS) or (None, None))[1] for r in records if r["canonical_id"] in app["members"]}
if len(values) == 1 and None not in values:
value = values.pop()
aliases[value] = key if value not in aliases else ("", "") # two applications claim it: no alias
aliases = {v: k for v, k in aliases.items() if k != ("", "")}
leftover = []
for record in rest: # 2. application tag, joined to a CMDB application when it is that application's alias
tag = _tag(record, APP_TAGS)
if tag and tag[1] in aliases:
join(aliases[tag[1]], record, "cmdb_business_service", _evidence(record),
f"Joined by its {tag[0]} tag '{tag[1]}', which every CMDB-grouped member of this application carries")
elif tag:
join(("tag", tag[1]), record, "tag", _evidence(record), None)
else:
leftover.append(record)
unplaced = {}
for record in leftover: # 3. container
name = _container_name(provider, record)
if name:
join(("container", name), record, "container", _evidence(record),
"Grouped only by sharing a resource group or project")
else:
unplaced[record["canonical_id"]] = record
model_note = None
if unplaced and ask is not None: # 4. the model, judged here
ids = list(unplaced)
for start in range(0, len(ids), MODEL_BATCH):
batch = {i: unplaced[i] for i in ids[start:start + MODEL_BATCH]}
try:
judged = _judge(await ask(_prompt(batch)), batch)
except Exception as exc: # a failed call leaves the batch ungrouped, declared
judged, model_note = None, f"the grouping model failed: {type(exc).__name__}"
if judged is None:
model_note = model_note or "the grouping model's reply failed the checks, so its assets stay ungrouped"
continue
for proposal in judged:
for member in proposal["members"]:
join(("model", proposal["name"]), batch[member], "model", _evidence(batch[member]),
"Proposed by the model from names and types only: " + (proposal["reason"] or "no reason given"))
asset_owners = owners(records, cis)
owner_of = {o["canonical_id"]: o["owner"] for o in asset_owners}
out = []
for number, (key, app) in enumerate(sorted(apps.items(), key=lambda kv: (kv[1]["basis"], kv[0][1])), start=1):
named = {owner_of[m] for m in app["members"] if m in owner_of}
out.append({"application_id": f"APP-{number:03d}", "name": app["name"], "basis": app["basis"],
"members": app["members"], "owner": named.pop() if len(named) == 1 else None,
"owners_disagree": len(named) > 1, "business_service": key[1] if key[0] == "cmdb" else None,
"evidence_refs": app["evidence_refs"], "review_required": app["review_required"],
"review_reasons": app["review_reasons"]})
ungrouped = sorted(r["canonical_id"] for r in records if r["canonical_id"] not in placed)
reviewed = sum(1 for a in out if a["review_required"])
note = (f"{len(out)} applications from {len(placed)} of {len(records)} records; {reviewed} need review; "
f"{len(ungrouped)} ungrouped; {len(asset_owners)} assets with an owner.")
if model_note:
note += f" Note: {model_note}."
return {"applications": out, "asset_owners": asset_owners, "ungrouped_assets": ungrouped, "applications_note": note}

View File

@@ -0,0 +1,36 @@
"""A live run's dependency graph (ADR 0019) and its applications and asset owners (ADR 0021) are built in code from
the scan and the CMDB; the sample estate keeps the agent loop and the alz-dependency tools."""
import json
from pathlib import Path
from typing import Any, Optional
import jsonschema
from kyndryl_agent_runtime import ask_sub_agent
from kyndryl_agent_runtime.request_cycle import Reply
from app import applications, graph
OUTPUT_SCHEMA = json.loads((Path(__file__).resolve().parents[1] / "schemas" / "output.json").read_text())
class GraphInvalid(RuntimeError):
"""The graph broke its output schema: a Reply can't be repaired by the output contract, so the step fails."""
async def build(payload: dict, ask) -> dict:
result = graph.build(payload)
result.update(await applications.group(payload, ask))
errors = sorted(jsonschema.Draft202012Validator(OUTPUT_SCHEMA).iter_errors(result), key=lambda e: list(e.path))
if errors:
raise GraphInvalid(f"the dependency graph broke its output schema: {errors[0].message}")
return result
async def pre_process(payload: dict) -> Optional[Any]:
if payload.get("provider", "sample") not in ("azure", "gcp"):
return None # the Meridian sample estate: prompt.md and the alz-dependency tools, unchanged
async def ask(prompt: str) -> str:
return await ask_sub_agent("propose_application_groups", prompt)
return Reply(json.dumps(await build(payload, ask), ensure_ascii=False))

View File

@@ -0,0 +1,220 @@
"""A live run's dependency graph, built in code (ADR 0019): one graph, three kinds of node.
- `resource`: a scope entry the hyperscaler showed (its inventory record).
- `external`: a resource a scope entry depends on that lies outside the scope root, or wasn't inventoried.
- `ci`: a CMDB CI the reconciliation judged (ADR 0018).
Every edge runs parent (`source`) to child (`target`), whatever the kinds:
- resource -> resource or external: a dependency the scan recorded in a finding's `dependencies`;
- ci -> ci: a relationship the CMDB records;
- ci -> resource (`represents`): the reconciliation's match of a CI to its resource.
The comparison walks this one graph: a CMDB relationship whose two CIs represent two resources the cloud also links,
in the same direction, is `confirmed`; one the cloud doesn't show is `cmdb_only`; a cloud dependency between two
represented resources the CMDB doesn't record is `discovered_only`. No model is called.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional, Tuple
from app import scope as sc
AZURE_CONTAINER = re.compile(r"^(/subscriptions/[^/]+(?:/resourcegroups/[^/]+)?)", re.I)
GCP_PROJECT = re.compile(r"/projects/([^/]+)(?:/|$)")
NETWORK = ("microsoft.network/virtualnetworks", "microsoft.network/virtualnetworks/subnets",
"microsoft.network/loadbalancers", "microsoft.network/publicipaddresses",
"microsoft.network/applicationgateways", "microsoft.network/networksecuritygroups",
"microsoft.network/privatednszones", "microsoft.network/dnszones",
"compute.googleapis.com/network", "compute.googleapis.com/subnetwork", "compute.googleapis.com/address",
"compute.googleapis.com/forwardingrule")
HOSTING = ("microsoft.compute/disks", "microsoft.compute/virtualmachines", "microsoft.compute/virtualmachinescalesets",
"microsoft.containerservice/managedclusters", "microsoft.web/serverfarms",
"compute.googleapis.com/disk", "compute.googleapis.com/instance", "container.googleapis.com/cluster")
SHARED = ("microsoft.keyvault/vaults", "microsoft.managedidentity/userassignedidentities",
"secretmanager.googleapis.com/secret", "cloudkms.googleapis.com/cryptokey")
MESSAGING = ("microsoft.servicebus/namespaces", "microsoft.eventhub/namespaces", "microsoft.eventgrid/topics",
"microsoft.eventgrid/systemtopics", "pubsub.googleapis.com/topic")
CLOUD = {"confidence": 0.8, "validation": "auto_approved"} # the provider's own record of the link (D3)
CMDB = {"confidence": 0.7, "validation": "sme_review_recommended"} # what the CMDB says, not seen in the cloud
CONFIRMED = 0.95 # a cloud edge the CMDB also records
MATCH = {"cloud_resource_id": {"confidence": 0.95, "validation": "auto_approved"},
"name": {"confidence": 0.7, "validation": "sme_review_recommended"}}
def key(provider: str, rid: Any) -> str:
rid = str(rid or "").strip().rstrip("/")
return rid.lower() if provider == "azure" else rid
def type_of(provider: str, rid: str) -> str:
"""A resource's type from its id: microsoft.network/virtualnetworks/subnets, compute.googleapis.com/disk."""
if provider == "azure":
tail = rid.split("/providers/")[-1].split("/") if "/providers/" in rid else []
if len(tail) < 3:
return ""
return "/".join([tail[0]] + tail[1::2][:max(1, (len(tail) - 1) // 2)]).lower()
parts = rid.lstrip("/").split("/")
if len(parts) < 3:
return ""
collection = parts[-2]
singular = collection[:-1] if collection.endswith("s") else collection
return f"{parts[0]}/{singular}".lower()
def container(provider: str, rid: str) -> Optional[str]:
if provider == "azure":
found = AZURE_CONTAINER.match(rid.strip())
return found.group(1).lower() if found else None
found = GCP_PROJECT.search(rid)
return f"projects/{found.group(1)}" if found else None
def edge_type(provider: str, rtype: str) -> str:
if rtype in NETWORK or rtype.endswith(("/subnets", "/subnetwork")):
return "network_attachment"
if rtype in sc.DATA_STORES.get(provider, ()):
return "app_to_db"
if rtype in HOSTING:
return "hosted_on"
if rtype in SHARED:
return "app_to_shared_service"
if rtype in MESSAGING:
return "middleware"
return "unclassified"
def _dependency_id(dep: Any) -> str:
if isinstance(dep, dict):
return str(dep.get("id") or dep.get("target") or dep.get("resource_id") or dep.get("scope_entry_id") or "")
return str(dep or "")
def build(payload: Dict[str, Any]) -> Dict[str, Any]:
provider = payload["provider"]
records = [r for r in payload.get("records") or [] if r.get("scope_entry_id")]
by_resource = {key(provider, r["scope_entry_id"]): r for r in records}
record_containers = {container(provider, r["scope_entry_id"]) for r in records}
nodes: Dict[str, Dict[str, Any]] = {}
edges: List[Dict[str, Any]] = []
folded: List[Dict[str, Any]] = []
seen: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
for r in records:
nodes[r["canonical_id"]] = {"id": r["canonical_id"], "canonical_id": r["canonical_id"], "kind": "resource",
"name": r.get("hostname"), "resource_id": r["scope_entry_id"]}
def edge(source: str, target: str, kind: str, evidence: List[Dict[str, str]], weight: Dict[str, Any]
) -> Dict[str, Any]:
found = seen.get((source, target, kind))
if found:
found["evidence"] += [e for e in evidence if e not in found["evidence"]]
return found
made = {"edge_id": f"E-{len(edges) + 1:03d}", "source": source, "target": target, "type": kind,
"protocol_port": None, "evidence": list(evidence), **weight}
edges.append(made)
seen[(source, target, kind)] = made
return made
# resource -> resource / external: what the scan recorded
for finding in payload.get("findings") or []:
source = by_resource.get(key(provider, finding.get("scope_entry_id")))
if not source:
continue
evidence = [{"kind": "config", "ref": finding.get("evidence_ref") or finding["scope_entry_id"]}]
for dep in finding.get("dependencies") or []:
rid = _dependency_id(dep)
if not rid or key(provider, rid) == key(provider, source["scope_entry_id"]):
continue
rtype = type_of(provider, rid)
target = by_resource.get(key(provider, rid))
if target:
edge(source["canonical_id"], target["canonical_id"], edge_type(provider, rtype), evidence, CLOUD)
continue
if rtype in sc.FOLDED.get(provider, {}):
folded.append({"source": source["canonical_id"], "target": rid, "type": rtype,
"rule": sc.FOLDED[provider][rtype], "evidence_ref": evidence[0]["ref"]})
continue
node = nodes.setdefault(rid, {"id": rid, "canonical_id": rid, "kind": "external",
"name": rid.rstrip("/").split("/")[-1], "type": rtype,
"container": container(provider, rid),
"reason": ("not_inventoried" if container(provider, rid) in record_containers
else "outside_scope_root"),
"referenced_by": []})
if source["canonical_id"] not in node["referenced_by"]:
node["referenced_by"].append(source["canonical_id"])
edge(source["canonical_id"], rid, edge_type(provider, rtype), evidence, CLOUD)
# ci nodes, ci -> resource (represents), ci -> ci (relationships)
represents: Dict[str, str] = {}
for ci in payload.get("cmdb_cis") or []:
node_id = f"cmdb:{ci['ci_ref']}"
nodes[node_id] = {"id": node_id, "canonical_id": node_id, "kind": "ci", "ci_ref": ci["ci_ref"],
"name": ci.get("name"), "ci_class": ci.get("ci_class"), "lifecycle": ci.get("lifecycle"),
"result": ci.get("result"), "evidence_ref": ci.get("evidence_ref")}
target = ci.get("matched_canonical_id")
if target and target in nodes:
represents[node_id] = target
edge(node_id, target, "represents", [{"kind": "reconciliation", "ref": ci.get("evidence_ref") or node_id}],
MATCH.get(ci.get("match_basis") or "name", MATCH["name"]))
cloud_pairs: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
for e in edges:
if nodes.get(e["source"], {}).get("kind") == "resource" and nodes.get(e["target"], {}).get("kind") == "resource":
cloud_pairs.setdefault((e["source"], e["target"]), []).append(e)
comparison: List[Dict[str, Any]] = []
matched_pairs = set()
for rel in payload.get("cmdb_relationships") or []:
parent, child = f"cmdb:{rel['parent_ref']}", f"cmdb:{rel['child_ref']}"
if parent not in nodes or child not in nodes:
continue
cmdb_edge = edge(parent, child, rel["type"], [{"kind": "cmdb_relationship", "ref": rel["evidence_ref"]}], CMDB)
pair = (represents.get(parent), represents.get(child))
item = {"cmdb_edge": cmdb_edge["edge_id"], "parent_ci": rel["parent_ref"], "child_ci": rel["child_ref"],
"source": pair[0], "target": pair[1],
"source_resource": nodes.get(pair[0] or "", {}).get("resource_id"),
"target_resource": nodes.get(pair[1] or "", {}).get("resource_id")}
if not all(pair):
comparison.append({**item, "result": "not_comparable",
"reason": "an end has no matched resource, so the cloud can't confirm or deny it"})
elif pair in cloud_pairs:
matched_pairs.add(pair)
for cloud in cloud_pairs[pair]:
cloud["evidence"].append({"kind": "cmdb_relationship", "ref": rel["evidence_ref"]})
cloud["confidence"] = max(cloud["confidence"], CONFIRMED)
comparison.append({**item, "result": "confirmed", "cloud_edges": [c["edge_id"] for c in cloud_pairs[pair]]})
else:
comparison.append({**item, "result": "cmdb_only"})
represented = set(represents.values())
if payload.get("cmdb_cis"):
for pair, cloud in cloud_pairs.items():
if pair[0] in represented and pair[1] in represented and pair not in matched_pairs:
comparison.append({"result": "discovered_only", "cloud_edges": [c["edge_id"] for c in cloud],
"source": pair[0], "target": pair[1],
"source_resource": nodes[pair[0]].get("resource_id"),
"target_resource": nodes[pair[1]].get("resource_id"),
"parent_ci": next((n[5:] for n, r in represents.items() if r == pair[0]), None),
"child_ci": next((n[5:] for n, r in represents.items() if r == pair[1]), None)})
# fan-in and fan-out over dependency edges (represents is a link, not a dependency)
dependency_edges = [e for e in edges if e["type"] != "represents"]
for node in nodes.values():
node["fan_out"] = sum(1 for e in dependency_edges if e["source"] == node["id"])
node["fan_in"] = sum(1 for e in dependency_edges if e["target"] == node["id"])
node["depended_on"] = node["fan_in"] > 0
node["retire_constraint"] = "cannot_retire_without_mitigation" if node["depended_on"] else "none"
orphans = [n["id"] for n in nodes.values() if n["kind"] == "resource" and not n["fan_in"] and not n["fan_out"]]
counts = {kind: sum(1 for n in nodes.values() if n["kind"] == kind) for kind in ("resource", "external", "ci")}
recorded = sum(len(f.get("dependencies") or []) for f in payload.get("findings") or [])
by_result = {r: sum(1 for c in comparison if c["result"] == r)
for r in ("confirmed", "cmdb_only", "discovered_only", "not_comparable")}
note = (f"Configured links only; traffic isn't known. {recorded} dependencies were recorded by the scan: "
f"{sum(1 for e in dependency_edges if nodes[e['source']]['kind'] == 'resource')} edges, "
f"{counts['external']} outside the scope as external nodes, {len(folded)} to a resource's own components. "
f"{counts['ci']} CMDB CIs, {len(payload.get('cmdb_relationships') or [])} CMDB relationships: "
+ ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in by_result.items()) + ".")
return {"graph_id": f"GRAPH-{payload.get('run_id') or payload['engagement_id']}",
"engagement_id": payload["engagement_id"], "edges": edges, "nodes": list(nodes.values()),
"orphans": orphans, "validation_queue": [e["edge_id"] for e in edges if e["validation"] != "auto_approved"],
"folded_dependencies": folded, "dangling": list(payload.get("cmdb_dangling") or []),
"graph_comparison": comparison, "note": note}

View File

@@ -0,0 +1,111 @@
"""A live run's scope: every resource under the connection's scope root (ADR 0017).
The planner plans from this listing and the critic checks coverage against its own run of it, so both copies of
this file must stay identical (tests/test_scope.py in each agent compares them). No runtime imports.
Every resource listed ends in exactly one place:
- a scope entry: an asset the run scans, inventories and reports;
- folded: a child component recorded with its parent's rule (a disk, a network interface, a node pool), never
dropped;
- structure: the estate's subscriptions, resource groups, projects and folders, which are not assets.
"""
from __future__ import annotations
from typing import Any, Callable, Dict, List, Optional, Tuple
PAGE_SIZE = 200 # the servers' maximum page
MAX_PAGES = 100 # 20,000 resources bounds a runaway; the largest estate seen so far is about 650
STRUCTURE = {
"azure": ("microsoft.resources/subscriptions", "microsoft.resources/subscriptions/resourcegroups",
"microsoft.management/managementgroups"),
"gcp": ("cloudresourcemanager.googleapis.com/project", "cloudresourcemanager.googleapis.com/folder",
"cloudresourcemanager.googleapis.com/organization"),
}
FOLDED = {
"azure": {
"microsoft.compute/disks": "a disk is part of the VM it is attached to",
"microsoft.network/networkinterfaces": "a network interface is part of the resource it is attached to",
"microsoft.compute/virtualmachines/extensions": "an extension is part of its VM",
"microsoft.compute/virtualmachinescalesets/virtualmachines": "an instance is part of its scale set",
},
"gcp": {
"compute.googleapis.com/disk": "a disk is part of the instance it is attached to",
"container.googleapis.com/nodepool": "a node pool is part of its cluster",
},
}
DATA_STORES = { # the types the servers' list_data_stores kinds cover; each gets a classify_data task
"azure": ("microsoft.sql/servers", "microsoft.sql/servers/databases", "microsoft.documentdb/databaseaccounts",
"microsoft.storage/storageaccounts", "microsoft.dbforpostgresql/flexibleservers",
"microsoft.dbforpostgresql/servers", "microsoft.dbformysql/flexibleservers",
"microsoft.dbformysql/servers", "microsoft.cache/redis", "microsoft.cache/redisenterprise"),
"gcp": ("sqladmin.googleapis.com/instance", "firestore.googleapis.com/database", "storage.googleapis.com/bucket"),
}
Call = Callable[[str, Dict[str, Any]], Any]
def identity(provider: str, item: Dict[str, Any]) -> Tuple[str, str, str]:
"""(id, type in lower case, location) of one listed resource."""
if provider == "gcp":
return str(item.get("name") or ""), str(item.get("assetType") or "").lower(), str(item.get("location") or "")
return str(item.get("id") or ""), str(item.get("type") or "").lower(), str(item.get("location") or "")
def _data(result: Any) -> Dict[str, Any]:
if isinstance(result, str):
import json
result = json.loads(result)
if not isinstance(result, dict) or result.get("success") is False:
error = (result or {}).get("error") if isinstance(result, dict) else None
raise RuntimeError((error or {}).get("message") or "the tool refused the call")
return result.get("data") or {}
def list_all(provider: str, call: Call) -> Tuple[List[Dict[str, Any]], List[str], List[str]]:
"""(every resource listed, evidence refs, issues). An issue means the listing is incomplete."""
tool = f"{provider}_list_resources"
items: List[Dict[str, Any]] = []
refs: List[str] = []
token: Optional[str] = None
for _ in range(MAX_PAGES):
args: Dict[str, Any] = {"page_size": PAGE_SIZE}
if token:
args["page_token"] = token
try:
data = _data(call(tool, args))
except Exception as error: # noqa: BLE001 - a refused or failed page is an incomplete listing
return items, refs, [f"{tool}: {str(error)[:200]}"]
items += list(data.get("items") or [])
if data.get("evidence_ref"):
refs.append(data["evidence_ref"])
token = data.get("next_page_token")
if not token:
return items, refs, []
return items, refs, [f"{tool}: stopped after {MAX_PAGES} pages"]
def account(provider: str, items: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Every listed resource in exactly one place: entries, folded or structure. Repeats are counted once."""
entries, folded, structure, seen = [], [], [], set()
for item in items:
rid, rtype, location = identity(provider, item)
if not rid or rid in seen:
continue
seen.add(rid)
if rtype in STRUCTURE.get(provider, ()):
structure.append(rid)
elif rtype in FOLDED.get(provider, {}):
folded.append({"id": rid, "type": rtype, "reason": FOLDED[provider][rtype]})
else:
entries.append({"id": rid, "type": rtype, "location": location})
entries.sort(key=lambda e: e["id"].lower())
return {"listed": len(seen), "entries": entries, "folded": folded, "structure": structure}
def region(location: str) -> str:
"""A GCP zone such as europe-west2-b is its region europe-west2; anything else is used as it is."""
parts = location.split("-")
if len(parts) == 3 and len(parts[2]) == 1 and parts[2].isalpha():
return "-".join(parts[:2])
return location