ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
"""A live plan built from every resource in scope (ADR 0017), against stand-in listings (no runtime)."""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from app import plan, scope # noqa: E402
|
||||
|
||||
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
|
||||
RG = "/subscriptions/s/resourceGroups/crucible-kab-dev/providers/"
|
||||
CRUCIBLE = [ # the eleven resources in crucible-kab-dev on 2026-11-04
|
||||
("Microsoft.ContainerService/managedClusters/crucible-kab-dev-aks", "microsoft.containerservice/managedclusters"),
|
||||
("Microsoft.Storage/storageAccounts/crucibkabdevexec", "microsoft.storage/storageaccounts"),
|
||||
("Microsoft.DBforPostgreSQL/flexibleServers/crucible-kab-dev-pg", "microsoft.dbforpostgresql/flexibleservers"),
|
||||
("Microsoft.KeyVault/vaults/kv-a", "microsoft.keyvault/vaults"),
|
||||
("Microsoft.KeyVault/vaults/kv-b", "microsoft.keyvault/vaults"),
|
||||
("Microsoft.ManagedIdentity/userAssignedIdentities/id-a", "microsoft.managedidentity/userassignedidentities"),
|
||||
("Microsoft.ManagedIdentity/userAssignedIdentities/id-b", "microsoft.managedidentity/userassignedidentities"),
|
||||
("Microsoft.Network/privateEndpoints/pe-1", "microsoft.network/privateendpoints"),
|
||||
("Microsoft.Network/networkInterfaces/pe-1.nic", "Microsoft.Network/networkInterfaces"),
|
||||
("Microsoft.Network/dnszones/example.com", "microsoft.network/dnszones"),
|
||||
("Microsoft.EventGrid/systemTopics/topic-1", "microsoft.eventgrid/systemtopics"),
|
||||
]
|
||||
CONTEXT = """generated:
|
||||
plan_id: PLAN-RUN-1
|
||||
plan: {name: CP1 trial}
|
||||
customer: {name: Kyndryl (internal)}
|
||||
context:
|
||||
scope: {regions: [], exclude: []}
|
||||
sources: {cmdb: not available, itsm: not available}
|
||||
"""
|
||||
|
||||
|
||||
def azure_rows(rows=CRUCIBLE, location="eastus"):
|
||||
return [{"id": RG + path, "type": rtype, "location": location} for path, rtype in rows]
|
||||
|
||||
|
||||
def lister(items, page=4, fail_at=None):
|
||||
calls = []
|
||||
|
||||
def call(tool, args):
|
||||
calls.append((tool, args))
|
||||
start = int(args.get("page_token") or 0)
|
||||
if fail_at is not None and start >= fail_at:
|
||||
return {"success": False, "error": {"code": "upstream_failed", "message": "429"}}
|
||||
chunk, nxt = items[start:start + page], start + page
|
||||
return {"success": True, "data": {"items": chunk, "next_page_token": str(nxt) if nxt < len(items) else None,
|
||||
"evidence_ref": f"sha256:{start}"}}
|
||||
return call, calls
|
||||
|
||||
|
||||
def payload(**extra):
|
||||
return {"engagement_id": "E", "run_id": "RUN-1", "provider": "azure", "include_cmdb": "no", "include_itsm": "no",
|
||||
"source_documents": CONTEXT, **extra}
|
||||
|
||||
|
||||
def test_every_resource_in_the_group_is_accounted_for():
|
||||
call, calls = lister(azure_rows())
|
||||
result = plan.build(payload(), call)
|
||||
jsonschema.validate(result, OUTPUT)
|
||||
acc = result["scope_accounting"]
|
||||
assert acc["listed"] == 11 and acc["scope_entries"] == 10 and acc["structure"] == 0
|
||||
assert [f["id"].split("/")[-1] for f in acc["folded"]] == ["pe-1.nic"]
|
||||
scans = [t["target_ref"].split("/")[-1] for t in result["tasks"] if t["type"] == "scan_target"]
|
||||
assert len(scans) == 10 and {"kv-a", "kv-b", "id-a", "id-b", "pe-1", "example.com", "topic-1"} <= set(scans)
|
||||
assert result["summary"] == {"scope_entries": 10, "planned": 10, "gaps": 0, "scan_batches": 1}
|
||||
assert {t for t, _ in calls} == {"azure_list_resources"} and all(a["page_size"] == 200 for _, a in calls)
|
||||
|
||||
|
||||
def test_data_stores_get_a_classification_task_after_their_scan():
|
||||
result = plan.build(payload(), lister(azure_rows())[0])
|
||||
by_id = {t["task_id"]: t for t in result["tasks"]}
|
||||
classify = [t for t in result["tasks"] if t["type"] == "classify_data"]
|
||||
assert sorted(t["target_ref"].split("/")[-1] for t in classify) == ["crucibkabdevexec", "crucible-kab-dev-pg"]
|
||||
for t in classify:
|
||||
assert by_id[t["depends_on"][0]]["target_ref"] == t["target_ref"]
|
||||
|
||||
|
||||
def test_network_and_dependency_tasks_close_the_plan():
|
||||
tasks = plan.build(payload(), lister(azure_rows())[0])["tasks"]
|
||||
assert [t["type"] for t in tasks[-2:]] == ["collect_network", "trace_dependencies"]
|
||||
assert set(tasks[-1]["depends_on"]) == {t["task_id"] for t in tasks[:-1]}
|
||||
|
||||
|
||||
def test_paging_reads_every_page():
|
||||
rows = azure_rows([(f"Microsoft.KeyVault/vaults/kv-{i:03d}", "microsoft.keyvault/vaults") for i in range(23)])
|
||||
call, calls = lister(rows, page=5)
|
||||
result = plan.build(payload(), call)
|
||||
assert result["scope_accounting"]["listed"] == 23 and len(calls) == 5 and result["summary"]["scan_batches"] == 3
|
||||
|
||||
|
||||
def test_an_incomplete_listing_fails_the_step():
|
||||
with pytest.raises(plan.ScopeIncomplete, match="429"):
|
||||
plan.build(payload(), lister(azure_rows(), page=4, fail_at=4)[0])
|
||||
|
||||
|
||||
def test_structure_is_never_a_scope_entry():
|
||||
rows = azure_rows() + [{"id": "/subscriptions/s/resourceGroups/crucible-kab-dev",
|
||||
"type": "microsoft.resources/subscriptions/resourcegroups", "location": "eastus"}]
|
||||
acc = plan.build(payload(), lister(rows)[0])["scope_accounting"]
|
||||
assert acc["structure"] == 1 and acc["scope_entries"] == 10
|
||||
|
||||
|
||||
def test_the_context_narrows_and_never_widens():
|
||||
ctx = CONTEXT.replace("scope: {regions: [], exclude: []}", "scope: {regions: [westeurope], exclude: [kv-a]}")
|
||||
result = plan.build(payload(source_documents=ctx), lister(azure_rows())[0])
|
||||
assert result["summary"]["planned"] == 0 and result["summary"]["gaps"] == 10
|
||||
kv = next(g for g in result["coverage_gaps"] if g["scope_entry_id"].endswith("kv-a"))
|
||||
assert "exclude" in kv["reason"]
|
||||
assert not [t for t in result["tasks"] if t["type"] == "scan_target"]
|
||||
|
||||
|
||||
def test_source_gaps_and_ids_follow_the_run():
|
||||
result = plan.build(payload(include_itsm="yes"), lister(azure_rows())[0])
|
||||
gaps = {g["source"]: g["reason"] for g in result["source_gaps"]}
|
||||
assert gaps == {"cmdb": "not_requested", "itsm": "no_connection", "netflow": "no_connection", "schema": "no_connection"}
|
||||
assert result["plan_id"] == "PLAN-RUN-1" and 'plan "CP1 trial"' in result["findings_note"]
|
||||
|
||||
|
||||
def test_gcp_folds_node_pools_and_disks_and_uses_regions():
|
||||
items = [{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1",
|
||||
"assetType": "container.googleapis.com/Cluster", "location": "europe-west2"},
|
||||
{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1/nodePools/np",
|
||||
"assetType": "container.googleapis.com/NodePool", "location": "europe-west2"},
|
||||
{"name": "//compute.googleapis.com/projects/p/zones/europe-west2-b/disks/d1",
|
||||
"assetType": "compute.googleapis.com/Disk", "location": "europe-west2-b"},
|
||||
{"name": "//cloudresourcemanager.googleapis.com/projects/123",
|
||||
"assetType": "cloudresourcemanager.googleapis.com/Project", "location": "global"}]
|
||||
ctx = CONTEXT.replace("regions: []", "regions: [europe-west2]")
|
||||
result = plan.build(payload(provider="gcp", source_documents=ctx), lister(items)[0])
|
||||
acc = result["scope_accounting"]
|
||||
assert (acc["listed"], acc["scope_entries"], len(acc["folded"]), acc["structure"]) == (4, 1, 2, 1)
|
||||
assert result["summary"]["planned"] == 1
|
||||
assert result["tasks"][0]["mcp_tool"] == "gcp-discovery.gcp_get_resource_metadata"
|
||||
assert scope.region("europe-west2-b") == "europe-west2"
|
||||
|
||||
|
||||
def test_repeated_rows_are_counted_once():
|
||||
rows = azure_rows() + azure_rows()[:2]
|
||||
assert plan.build(payload(), lister(rows)[0])["scope_accounting"]["listed"] == 11
|
||||
|
||||
|
||||
def test_the_critic_reads_the_same_rules():
|
||||
critic = ROOT.parent / "cp1-discovery-critic-agent" / "app" / "scope.py"
|
||||
assert critic.read_text() == (ROOT / "app" / "scope.py").read_text()
|
||||
|
||||
|
||||
def cmdb_upload(context=CONTEXT, include="yes"):
|
||||
ctx = context + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
||||
return payload(include_cmdb=include, source_documents=ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n")
|
||||
|
||||
|
||||
def test_an_uploaded_cmdb_is_a_source_and_gets_an_ingest_task():
|
||||
result = plan.build(cmdb_upload(), lister(azure_rows())[0])
|
||||
jsonschema.validate(result, OUTPUT)
|
||||
assert "cmdb" not in {g["source"] for g in result["source_gaps"]}
|
||||
[ingest] = [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
|
||||
assert ingest["target_ref"] == "upload:cmdb-export.csv" and ingest["assigned_agent"] == "ALZ-DSC-04"
|
||||
assert ingest["task_id"] in result["tasks"][-1]["depends_on"]
|
||||
assert "cmdb-export.csv" in result["findings_note"] and result["plan_id"] == "PLAN-RUN-1"
|
||||
|
||||
|
||||
def test_an_upload_is_ignored_when_the_cmdb_is_not_requested():
|
||||
result = plan.build(cmdb_upload(include="no"), lister(azure_rows())[0])
|
||||
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "not_requested"
|
||||
assert not [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
|
||||
|
||||
|
||||
def test_a_requested_cmdb_with_no_upload_is_still_a_gap():
|
||||
ctx = CONTEXT + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
||||
result = plan.build(payload(include_cmdb="yes", source_documents=ctx), lister(azure_rows())[0])
|
||||
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "no_connection"
|
||||
|
||||
|
||||
def test_a_relationship_export_is_ingested_with_its_cmdb():
|
||||
ctx = CONTEXT + ("uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
|
||||
" - {name: cmdb-relationships.csv, source: cmdb_relationships}\n")
|
||||
text = ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n" + "\n\n---\n\n parent child\n0 CI1 CI2\n"
|
||||
result = plan.build(payload(include_cmdb="yes", source_documents=text), lister(azure_rows())[0])
|
||||
jsonschema.validate(result, OUTPUT)
|
||||
assert [t["target_ref"] for t in result["tasks"] if t["type"].startswith("ingest_cmdb")] == [
|
||||
"upload:cmdb-export.csv", "upload:cmdb-relationships.csv"]
|
||||
@@ -0,0 +1,113 @@
|
||||
"""The upload split into the run context and its listed files, as parse_assessment_files renders them (ADR 0018)."""
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from app import run_context as rc # noqa: E402
|
||||
|
||||
CASE = ROOT.parent / "workflows" / "cases" / "azure-crucible-kab-dev-cmdb"
|
||||
SEP = "\n\n---\n\n"
|
||||
|
||||
|
||||
def rendered_csv(path):
|
||||
return pd.read_csv(path).to_string() # what data_file_processing.parse_csv hands on
|
||||
|
||||
|
||||
def rendered_xlsx(frames):
|
||||
buffer = io.BytesIO()
|
||||
with pd.ExcelWriter(buffer) as writer:
|
||||
for name, frame in frames.items():
|
||||
frame.to_excel(writer, sheet_name=name, index=False)
|
||||
parts = []
|
||||
for name, frame in pd.read_excel(io.BytesIO(buffer.getvalue()), sheet_name=None).items():
|
||||
parts += [f"=== Sheet: {name} ===", frame.to_string(), ""]
|
||||
return "\n".join(parts) # data_file_processing.parse_excel
|
||||
|
||||
|
||||
def upload(*files):
|
||||
return SEP.join(files)
|
||||
|
||||
|
||||
FULL_CONTEXT = (CASE / "context.yaml").read_text()
|
||||
# The case lists a CI export and a relationship export; most tests here need a context with one listed upload.
|
||||
CONTEXT = FULL_CONTEXT.replace(""" - name: cmdb-relationships.csv
|
||||
source: cmdb_relationships
|
||||
format: servicenow_cmdb_rel_ci_csv
|
||||
taken: 2026-11-04
|
||||
""", "")
|
||||
assert CONTEXT != FULL_CONTEXT
|
||||
|
||||
|
||||
def test_a_context_alone_reads_as_before():
|
||||
single = (ROOT.parent / "workflows" / "cases" / "azure-crucible-kab-dev" / "context.yaml").read_text()
|
||||
out = rc.split(single)
|
||||
assert out["context"]["workflow_inputs"]["provider"] == "azure" and out["uploads"] == []
|
||||
assert rc.context_section(single)["sources"]["cmdb"] == "not available"
|
||||
|
||||
|
||||
def test_a_csv_after_the_context_leaves_the_context_readable():
|
||||
text = upload(CONTEXT, rendered_csv(CASE / "cmdb-export.csv"))
|
||||
out = rc.split(text)
|
||||
assert out["context"]["plan"]["name"].endswith("with CMDB")
|
||||
[cmdb] = rc.uploads_from(text, "cmdb")
|
||||
[table] = rc.tables(cmdb)
|
||||
assert table["sheet"] is None and "object_id" in table["header"] and len(table["rows"]) == 12
|
||||
assert table["rows"][0][0] == 0 and "CI0010001" in table["rows"][0][1]
|
||||
assert table["rows"][11][0] == 11 and "Retired" in table["rows"][11][1]
|
||||
|
||||
|
||||
def test_two_files_in_order_and_an_xlsx_has_one_table_per_sheet():
|
||||
ctx = CONTEXT.replace("uploads:\n", "uploads:\n - {name: assets.xlsx, source: cmdb}\n")
|
||||
sheets = rendered_xlsx({"Servers": pd.DataFrame({"Name": ["a", "b"], "Env": ["prod", "dev"]}),
|
||||
"Cloud": pd.DataFrame({"Id": ["/subscriptions/s/x"], "Env": ["dev"]})})
|
||||
text = upload(ctx, sheets, rendered_csv(CASE / "cmdb-export.csv"))
|
||||
names = [u["name"] for u in rc.uploads_from(text, "cmdb")]
|
||||
assert names == ["assets.xlsx", "cmdb-export.csv"]
|
||||
tables = rc.tables(rc.uploads_from(text, "cmdb")[0])
|
||||
assert [(t["sheet"], len(t["rows"])) for t in tables] == [("Servers", 2), ("Cloud", 1)]
|
||||
assert "Env" in tables[1]["header"]
|
||||
|
||||
|
||||
def test_a_separator_inside_the_last_upload_is_kept():
|
||||
text = upload(CONTEXT, "h\n0 a" + SEP + "1 b")
|
||||
assert rc.split(text)["uploads"][0]["text"] == "h\n0 a" + SEP + "1 b"
|
||||
|
||||
|
||||
def test_a_listed_upload_that_never_arrived_is_unreadable():
|
||||
[cmdb] = rc.uploads_from(CONTEXT, "cmdb")
|
||||
assert cmdb["text"] is None
|
||||
with pytest.raises(rc.UploadUnreadable, match="not uploaded"):
|
||||
rc.tables(cmdb)
|
||||
|
||||
|
||||
def test_a_platform_parse_error_is_unreadable():
|
||||
[cmdb] = rc.uploads_from(upload(CONTEXT, "CSV parsing error: bad line 3"), "cmdb")
|
||||
with pytest.raises(rc.UploadUnreadable, match="could not parse"):
|
||||
rc.tables(cmdb)
|
||||
|
||||
|
||||
def test_an_empty_export_has_no_rows():
|
||||
[cmdb] = rc.uploads_from(upload(CONTEXT, pd.DataFrame(columns=["a", "b"]).to_string()), "cmdb")
|
||||
assert rc.tables(cmdb)[0]["rows"] == []
|
||||
|
||||
|
||||
def test_a_raw_csv_is_numbered_by_position():
|
||||
[cmdb] = rc.uploads_from(upload(CONTEXT, "id,name\n123,a\n456,b"), "cmdb")
|
||||
assert rc.tables(cmdb)[0]["rows"] == [(0, "123,a"), (1, "456,b")]
|
||||
|
||||
|
||||
def test_a_text_brief_is_not_a_context():
|
||||
assert rc.split("Meridian brief\n\nSome text.")["context"] == {}
|
||||
|
||||
|
||||
def test_the_case_lists_two_cmdb_uploads_in_order():
|
||||
rels = " sys_id parent child\n0 R1 CI1 CI2"
|
||||
text = upload(FULL_CONTEXT, rendered_csv(CASE / "cmdb-export.csv"), rels)
|
||||
assert [u["name"] for u in rc.split(text)["uploads"]] == ["cmdb-export.csv", "cmdb-relationships.csv"]
|
||||
[rel] = rc.uploads_from(text, "cmdb_relationships")
|
||||
assert rc.tables(rel)[0]["rows"] == [(0, "0 R1 CI1 CI2")]
|
||||
Reference in New Issue
Block a user