- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
187 lines
9.7 KiB
Python
187 lines
9.7 KiB
Python
"""A live plan built from every resource in scope (ADR 0017), against stand-in listings (no runtime)."""
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jsonschema
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from app import plan, scope # noqa: E402
|
|
|
|
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
|
|
RG = "/subscriptions/s/resourceGroups/crucible-kab-dev/providers/"
|
|
CRUCIBLE = [ # the eleven resources in crucible-kab-dev on 2026-11-04
|
|
("Microsoft.ContainerService/managedClusters/crucible-kab-dev-aks", "microsoft.containerservice/managedclusters"),
|
|
("Microsoft.Storage/storageAccounts/crucibkabdevexec", "microsoft.storage/storageaccounts"),
|
|
("Microsoft.DBforPostgreSQL/flexibleServers/crucible-kab-dev-pg", "microsoft.dbforpostgresql/flexibleservers"),
|
|
("Microsoft.KeyVault/vaults/kv-a", "microsoft.keyvault/vaults"),
|
|
("Microsoft.KeyVault/vaults/kv-b", "microsoft.keyvault/vaults"),
|
|
("Microsoft.ManagedIdentity/userAssignedIdentities/id-a", "microsoft.managedidentity/userassignedidentities"),
|
|
("Microsoft.ManagedIdentity/userAssignedIdentities/id-b", "microsoft.managedidentity/userassignedidentities"),
|
|
("Microsoft.Network/privateEndpoints/pe-1", "microsoft.network/privateendpoints"),
|
|
("Microsoft.Network/networkInterfaces/pe-1.nic", "Microsoft.Network/networkInterfaces"),
|
|
("Microsoft.Network/dnszones/example.com", "microsoft.network/dnszones"),
|
|
("Microsoft.EventGrid/systemTopics/topic-1", "microsoft.eventgrid/systemtopics"),
|
|
]
|
|
CONTEXT = """generated:
|
|
plan_id: PLAN-RUN-1
|
|
plan: {name: CP1 trial}
|
|
customer: {name: Kyndryl (internal)}
|
|
context:
|
|
scope: {regions: [], exclude: []}
|
|
sources: {cmdb: not available, itsm: not available}
|
|
"""
|
|
|
|
|
|
def azure_rows(rows=CRUCIBLE, location="eastus"):
|
|
return [{"id": RG + path, "type": rtype, "location": location} for path, rtype in rows]
|
|
|
|
|
|
def lister(items, page=4, fail_at=None):
|
|
calls = []
|
|
|
|
def call(tool, args):
|
|
calls.append((tool, args))
|
|
start = int(args.get("page_token") or 0)
|
|
if fail_at is not None and start >= fail_at:
|
|
return {"success": False, "error": {"code": "upstream_failed", "message": "429"}}
|
|
chunk, nxt = items[start:start + page], start + page
|
|
return {"success": True, "data": {"items": chunk, "next_page_token": str(nxt) if nxt < len(items) else None,
|
|
"evidence_ref": f"sha256:{start}"}}
|
|
return call, calls
|
|
|
|
|
|
def payload(**extra):
|
|
return {"engagement_id": "E", "run_id": "RUN-1", "provider": "azure", "include_cmdb": "no", "include_itsm": "no",
|
|
"source_documents": CONTEXT, **extra}
|
|
|
|
|
|
def test_every_resource_in_the_group_is_accounted_for():
|
|
call, calls = lister(azure_rows())
|
|
result = plan.build(payload(), call)
|
|
jsonschema.validate(result, OUTPUT)
|
|
acc = result["scope_accounting"]
|
|
assert acc["listed"] == 11 and acc["scope_entries"] == 10 and acc["structure"] == 0
|
|
assert [f["id"].split("/")[-1] for f in acc["folded"]] == ["pe-1.nic"]
|
|
scans = [t["target_ref"].split("/")[-1] for t in result["tasks"] if t["type"] == "scan_target"]
|
|
assert len(scans) == 10 and {"kv-a", "kv-b", "id-a", "id-b", "pe-1", "example.com", "topic-1"} <= set(scans)
|
|
assert result["summary"] == {"scope_entries": 10, "planned": 10, "gaps": 0, "scan_batches": 1}
|
|
assert {t for t, _ in calls} == {"azure_list_resources"} and all(a["page_size"] == 200 for _, a in calls)
|
|
|
|
|
|
def test_data_stores_get_a_classification_task_after_their_scan():
|
|
result = plan.build(payload(), lister(azure_rows())[0])
|
|
by_id = {t["task_id"]: t for t in result["tasks"]}
|
|
classify = [t for t in result["tasks"] if t["type"] == "classify_data"]
|
|
assert sorted(t["target_ref"].split("/")[-1] for t in classify) == ["crucibkabdevexec", "crucible-kab-dev-pg"]
|
|
for t in classify:
|
|
assert by_id[t["depends_on"][0]]["target_ref"] == t["target_ref"]
|
|
|
|
|
|
def test_network_and_dependency_tasks_close_the_plan():
|
|
tasks = plan.build(payload(), lister(azure_rows())[0])["tasks"]
|
|
assert [t["type"] for t in tasks[-2:]] == ["collect_network", "trace_dependencies"]
|
|
assert set(tasks[-1]["depends_on"]) == {t["task_id"] for t in tasks[:-1]}
|
|
|
|
|
|
def test_paging_reads_every_page():
|
|
rows = azure_rows([(f"Microsoft.KeyVault/vaults/kv-{i:03d}", "microsoft.keyvault/vaults") for i in range(23)])
|
|
call, calls = lister(rows, page=5)
|
|
result = plan.build(payload(), call)
|
|
assert result["scope_accounting"]["listed"] == 23 and len(calls) == 5 and result["summary"]["scan_batches"] == 3
|
|
|
|
|
|
def test_an_incomplete_listing_fails_the_step():
|
|
with pytest.raises(plan.ScopeIncomplete, match="429"):
|
|
plan.build(payload(), lister(azure_rows(), page=4, fail_at=4)[0])
|
|
|
|
|
|
def test_structure_is_never_a_scope_entry():
|
|
rows = azure_rows() + [{"id": "/subscriptions/s/resourceGroups/crucible-kab-dev",
|
|
"type": "microsoft.resources/subscriptions/resourcegroups", "location": "eastus"}]
|
|
acc = plan.build(payload(), lister(rows)[0])["scope_accounting"]
|
|
assert acc["structure"] == 1 and acc["scope_entries"] == 10
|
|
|
|
|
|
def test_the_context_narrows_and_never_widens():
|
|
ctx = CONTEXT.replace("scope: {regions: [], exclude: []}", "scope: {regions: [westeurope], exclude: [kv-a]}")
|
|
result = plan.build(payload(source_documents=ctx), lister(azure_rows())[0])
|
|
assert result["summary"]["planned"] == 0 and result["summary"]["gaps"] == 10
|
|
kv = next(g for g in result["coverage_gaps"] if g["scope_entry_id"].endswith("kv-a"))
|
|
assert "exclude" in kv["reason"]
|
|
assert not [t for t in result["tasks"] if t["type"] == "scan_target"]
|
|
|
|
|
|
def test_source_gaps_and_ids_follow_the_run():
|
|
result = plan.build(payload(include_itsm="yes"), lister(azure_rows())[0])
|
|
gaps = {g["source"]: g["reason"] for g in result["source_gaps"]}
|
|
assert gaps == {"cmdb": "not_requested", "itsm": "no_connection", "netflow": "no_connection", "schema": "no_connection"}
|
|
assert result["plan_id"] == "PLAN-RUN-1" and 'plan "CP1 trial"' in result["findings_note"]
|
|
|
|
|
|
def test_gcp_folds_node_pools_and_disks_and_uses_regions():
|
|
items = [{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1",
|
|
"assetType": "container.googleapis.com/Cluster", "location": "europe-west2"},
|
|
{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1/nodePools/np",
|
|
"assetType": "container.googleapis.com/NodePool", "location": "europe-west2"},
|
|
{"name": "//compute.googleapis.com/projects/p/zones/europe-west2-b/disks/d1",
|
|
"assetType": "compute.googleapis.com/Disk", "location": "europe-west2-b"},
|
|
{"name": "//cloudresourcemanager.googleapis.com/projects/123",
|
|
"assetType": "cloudresourcemanager.googleapis.com/Project", "location": "global"}]
|
|
ctx = CONTEXT.replace("regions: []", "regions: [europe-west2]")
|
|
result = plan.build(payload(provider="gcp", source_documents=ctx), lister(items)[0])
|
|
acc = result["scope_accounting"]
|
|
assert (acc["listed"], acc["scope_entries"], len(acc["folded"]), acc["structure"]) == (4, 1, 2, 1)
|
|
assert result["summary"]["planned"] == 1
|
|
assert result["tasks"][0]["mcp_tool"] == "gcp-discovery.gcp_get_resource_metadata"
|
|
assert scope.region("europe-west2-b") == "europe-west2"
|
|
|
|
|
|
def test_repeated_rows_are_counted_once():
|
|
rows = azure_rows() + azure_rows()[:2]
|
|
assert plan.build(payload(), lister(rows)[0])["scope_accounting"]["listed"] == 11
|
|
|
|
|
|
def test_the_critic_reads_the_same_rules():
|
|
critic = ROOT.parent / "cp1-discovery-critic-agent" / "app" / "scope.py"
|
|
assert critic.read_text() == (ROOT / "app" / "scope.py").read_text()
|
|
|
|
|
|
def cmdb_upload(context=CONTEXT, include="yes"):
|
|
ctx = context + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
|
return payload(include_cmdb=include, source_documents=ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n")
|
|
|
|
|
|
def test_an_uploaded_cmdb_is_a_source_and_gets_an_ingest_task():
|
|
result = plan.build(cmdb_upload(), lister(azure_rows())[0])
|
|
jsonschema.validate(result, OUTPUT)
|
|
assert "cmdb" not in {g["source"] for g in result["source_gaps"]}
|
|
[ingest] = [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
|
|
assert ingest["target_ref"] == "upload:cmdb-export.csv" and ingest["assigned_agent"] == "ALZ-DSC-04"
|
|
assert ingest["task_id"] in result["tasks"][-1]["depends_on"]
|
|
assert "cmdb-export.csv" in result["findings_note"] and result["plan_id"] == "PLAN-RUN-1"
|
|
|
|
|
|
def test_an_upload_is_ignored_when_the_cmdb_is_not_requested():
|
|
result = plan.build(cmdb_upload(include="no"), lister(azure_rows())[0])
|
|
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "not_requested"
|
|
assert not [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
|
|
|
|
|
|
def test_a_requested_cmdb_with_no_upload_is_still_a_gap():
|
|
ctx = CONTEXT + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
|
result = plan.build(payload(include_cmdb="yes", source_documents=ctx), lister(azure_rows())[0])
|
|
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "no_connection"
|
|
|
|
|
|
def test_a_relationship_export_is_ingested_with_its_cmdb():
|
|
ctx = CONTEXT + ("uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
|
" - {name: cmdb-relationships.csv, source: cmdb_relationships}\n")
|
|
text = ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n" + "\n\n---\n\n parent child\n0 CI1 CI2\n"
|
|
result = plan.build(payload(include_cmdb="yes", source_documents=text), lister(azure_rows())[0])
|
|
jsonschema.validate(result, OUTPUT)
|
|
assert [t["target_ref"] for t in result["tasks"] if t["type"].startswith("ingest_cmdb")] == [
|
|
"upload:cmdb-export.csv", "upload:cmdb-relationships.csv"]
|