Files
Jonathan Boniface 9e503885cc ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

187 lines
9.7 KiB
Python

"""A live plan built from every resource in scope (ADR 0017), against stand-in listings (no runtime)."""
import json
import sys
from pathlib import Path
import jsonschema
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import plan, scope # noqa: E402
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
RG = "/subscriptions/s/resourceGroups/crucible-kab-dev/providers/"
CRUCIBLE = [ # the eleven resources in crucible-kab-dev on 2026-11-04
("Microsoft.ContainerService/managedClusters/crucible-kab-dev-aks", "microsoft.containerservice/managedclusters"),
("Microsoft.Storage/storageAccounts/crucibkabdevexec", "microsoft.storage/storageaccounts"),
("Microsoft.DBforPostgreSQL/flexibleServers/crucible-kab-dev-pg", "microsoft.dbforpostgresql/flexibleservers"),
("Microsoft.KeyVault/vaults/kv-a", "microsoft.keyvault/vaults"),
("Microsoft.KeyVault/vaults/kv-b", "microsoft.keyvault/vaults"),
("Microsoft.ManagedIdentity/userAssignedIdentities/id-a", "microsoft.managedidentity/userassignedidentities"),
("Microsoft.ManagedIdentity/userAssignedIdentities/id-b", "microsoft.managedidentity/userassignedidentities"),
("Microsoft.Network/privateEndpoints/pe-1", "microsoft.network/privateendpoints"),
("Microsoft.Network/networkInterfaces/pe-1.nic", "Microsoft.Network/networkInterfaces"),
("Microsoft.Network/dnszones/example.com", "microsoft.network/dnszones"),
("Microsoft.EventGrid/systemTopics/topic-1", "microsoft.eventgrid/systemtopics"),
]
CONTEXT = """generated:
plan_id: PLAN-RUN-1
plan: {name: CP1 trial}
customer: {name: Kyndryl (internal)}
context:
scope: {regions: [], exclude: []}
sources: {cmdb: not available, itsm: not available}
"""
def azure_rows(rows=CRUCIBLE, location="eastus"):
return [{"id": RG + path, "type": rtype, "location": location} for path, rtype in rows]
def lister(items, page=4, fail_at=None):
calls = []
def call(tool, args):
calls.append((tool, args))
start = int(args.get("page_token") or 0)
if fail_at is not None and start >= fail_at:
return {"success": False, "error": {"code": "upstream_failed", "message": "429"}}
chunk, nxt = items[start:start + page], start + page
return {"success": True, "data": {"items": chunk, "next_page_token": str(nxt) if nxt < len(items) else None,
"evidence_ref": f"sha256:{start}"}}
return call, calls
def payload(**extra):
return {"engagement_id": "E", "run_id": "RUN-1", "provider": "azure", "include_cmdb": "no", "include_itsm": "no",
"source_documents": CONTEXT, **extra}
def test_every_resource_in_the_group_is_accounted_for():
call, calls = lister(azure_rows())
result = plan.build(payload(), call)
jsonschema.validate(result, OUTPUT)
acc = result["scope_accounting"]
assert acc["listed"] == 11 and acc["scope_entries"] == 10 and acc["structure"] == 0
assert [f["id"].split("/")[-1] for f in acc["folded"]] == ["pe-1.nic"]
scans = [t["target_ref"].split("/")[-1] for t in result["tasks"] if t["type"] == "scan_target"]
assert len(scans) == 10 and {"kv-a", "kv-b", "id-a", "id-b", "pe-1", "example.com", "topic-1"} <= set(scans)
assert result["summary"] == {"scope_entries": 10, "planned": 10, "gaps": 0, "scan_batches": 1}
assert {t for t, _ in calls} == {"azure_list_resources"} and all(a["page_size"] == 200 for _, a in calls)
def test_data_stores_get_a_classification_task_after_their_scan():
result = plan.build(payload(), lister(azure_rows())[0])
by_id = {t["task_id"]: t for t in result["tasks"]}
classify = [t for t in result["tasks"] if t["type"] == "classify_data"]
assert sorted(t["target_ref"].split("/")[-1] for t in classify) == ["crucibkabdevexec", "crucible-kab-dev-pg"]
for t in classify:
assert by_id[t["depends_on"][0]]["target_ref"] == t["target_ref"]
def test_network_and_dependency_tasks_close_the_plan():
tasks = plan.build(payload(), lister(azure_rows())[0])["tasks"]
assert [t["type"] for t in tasks[-2:]] == ["collect_network", "trace_dependencies"]
assert set(tasks[-1]["depends_on"]) == {t["task_id"] for t in tasks[:-1]}
def test_paging_reads_every_page():
rows = azure_rows([(f"Microsoft.KeyVault/vaults/kv-{i:03d}", "microsoft.keyvault/vaults") for i in range(23)])
call, calls = lister(rows, page=5)
result = plan.build(payload(), call)
assert result["scope_accounting"]["listed"] == 23 and len(calls) == 5 and result["summary"]["scan_batches"] == 3
def test_an_incomplete_listing_fails_the_step():
with pytest.raises(plan.ScopeIncomplete, match="429"):
plan.build(payload(), lister(azure_rows(), page=4, fail_at=4)[0])
def test_structure_is_never_a_scope_entry():
rows = azure_rows() + [{"id": "/subscriptions/s/resourceGroups/crucible-kab-dev",
"type": "microsoft.resources/subscriptions/resourcegroups", "location": "eastus"}]
acc = plan.build(payload(), lister(rows)[0])["scope_accounting"]
assert acc["structure"] == 1 and acc["scope_entries"] == 10
def test_the_context_narrows_and_never_widens():
ctx = CONTEXT.replace("scope: {regions: [], exclude: []}", "scope: {regions: [westeurope], exclude: [kv-a]}")
result = plan.build(payload(source_documents=ctx), lister(azure_rows())[0])
assert result["summary"]["planned"] == 0 and result["summary"]["gaps"] == 10
kv = next(g for g in result["coverage_gaps"] if g["scope_entry_id"].endswith("kv-a"))
assert "exclude" in kv["reason"]
assert not [t for t in result["tasks"] if t["type"] == "scan_target"]
def test_source_gaps_and_ids_follow_the_run():
result = plan.build(payload(include_itsm="yes"), lister(azure_rows())[0])
gaps = {g["source"]: g["reason"] for g in result["source_gaps"]}
assert gaps == {"cmdb": "not_requested", "itsm": "no_connection", "netflow": "no_connection", "schema": "no_connection"}
assert result["plan_id"] == "PLAN-RUN-1" and 'plan "CP1 trial"' in result["findings_note"]
def test_gcp_folds_node_pools_and_disks_and_uses_regions():
items = [{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1",
"assetType": "container.googleapis.com/Cluster", "location": "europe-west2"},
{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1/nodePools/np",
"assetType": "container.googleapis.com/NodePool", "location": "europe-west2"},
{"name": "//compute.googleapis.com/projects/p/zones/europe-west2-b/disks/d1",
"assetType": "compute.googleapis.com/Disk", "location": "europe-west2-b"},
{"name": "//cloudresourcemanager.googleapis.com/projects/123",
"assetType": "cloudresourcemanager.googleapis.com/Project", "location": "global"}]
ctx = CONTEXT.replace("regions: []", "regions: [europe-west2]")
result = plan.build(payload(provider="gcp", source_documents=ctx), lister(items)[0])
acc = result["scope_accounting"]
assert (acc["listed"], acc["scope_entries"], len(acc["folded"]), acc["structure"]) == (4, 1, 2, 1)
assert result["summary"]["planned"] == 1
assert result["tasks"][0]["mcp_tool"] == "gcp-discovery.gcp_get_resource_metadata"
assert scope.region("europe-west2-b") == "europe-west2"
def test_repeated_rows_are_counted_once():
rows = azure_rows() + azure_rows()[:2]
assert plan.build(payload(), lister(rows)[0])["scope_accounting"]["listed"] == 11
def test_the_critic_reads_the_same_rules():
critic = ROOT.parent / "cp1-discovery-critic-agent" / "app" / "scope.py"
assert critic.read_text() == (ROOT / "app" / "scope.py").read_text()
def cmdb_upload(context=CONTEXT, include="yes"):
ctx = context + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
return payload(include_cmdb=include, source_documents=ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n")
def test_an_uploaded_cmdb_is_a_source_and_gets_an_ingest_task():
result = plan.build(cmdb_upload(), lister(azure_rows())[0])
jsonschema.validate(result, OUTPUT)
assert "cmdb" not in {g["source"] for g in result["source_gaps"]}
[ingest] = [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
assert ingest["target_ref"] == "upload:cmdb-export.csv" and ingest["assigned_agent"] == "ALZ-DSC-04"
assert ingest["task_id"] in result["tasks"][-1]["depends_on"]
assert "cmdb-export.csv" in result["findings_note"] and result["plan_id"] == "PLAN-RUN-1"
def test_an_upload_is_ignored_when_the_cmdb_is_not_requested():
result = plan.build(cmdb_upload(include="no"), lister(azure_rows())[0])
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "not_requested"
assert not [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
def test_a_requested_cmdb_with_no_upload_is_still_a_gap():
ctx = CONTEXT + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
result = plan.build(payload(include_cmdb="yes", source_documents=ctx), lister(azure_rows())[0])
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "no_connection"
def test_a_relationship_export_is_ingested_with_its_cmdb():
ctx = CONTEXT + ("uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
" - {name: cmdb-relationships.csv, source: cmdb_relationships}\n")
text = ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n" + "\n\n---\n\n parent child\n0 CI1 CI2\n"
result = plan.build(payload(include_cmdb="yes", source_documents=text), lister(azure_rows())[0])
jsonschema.validate(result, OUTPUT)
assert [t["target_ref"] for t in result["tasks"] if t["type"].startswith("ingest_cmdb")] == [
"upload:cmdb-export.csv", "upload:cmdb-relationships.csv"]