ALZ-F01 Discovery: package step, value scorecard, application grouping, cases

- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent f659564208
commit 9e503885cc
211 changed files with 18170 additions and 158 deletions

View File

@@ -0,0 +1,186 @@
"""A live plan built from every resource in scope (ADR 0017), against stand-in listings (no runtime)."""
import json
import sys
from pathlib import Path
import jsonschema
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import plan, scope # noqa: E402
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
RG = "/subscriptions/s/resourceGroups/crucible-kab-dev/providers/"
CRUCIBLE = [ # the eleven resources in crucible-kab-dev on 2026-11-04
("Microsoft.ContainerService/managedClusters/crucible-kab-dev-aks", "microsoft.containerservice/managedclusters"),
("Microsoft.Storage/storageAccounts/crucibkabdevexec", "microsoft.storage/storageaccounts"),
("Microsoft.DBforPostgreSQL/flexibleServers/crucible-kab-dev-pg", "microsoft.dbforpostgresql/flexibleservers"),
("Microsoft.KeyVault/vaults/kv-a", "microsoft.keyvault/vaults"),
("Microsoft.KeyVault/vaults/kv-b", "microsoft.keyvault/vaults"),
("Microsoft.ManagedIdentity/userAssignedIdentities/id-a", "microsoft.managedidentity/userassignedidentities"),
("Microsoft.ManagedIdentity/userAssignedIdentities/id-b", "microsoft.managedidentity/userassignedidentities"),
("Microsoft.Network/privateEndpoints/pe-1", "microsoft.network/privateendpoints"),
("Microsoft.Network/networkInterfaces/pe-1.nic", "Microsoft.Network/networkInterfaces"),
("Microsoft.Network/dnszones/example.com", "microsoft.network/dnszones"),
("Microsoft.EventGrid/systemTopics/topic-1", "microsoft.eventgrid/systemtopics"),
]
CONTEXT = """generated:
plan_id: PLAN-RUN-1
plan: {name: CP1 trial}
customer: {name: Kyndryl (internal)}
context:
scope: {regions: [], exclude: []}
sources: {cmdb: not available, itsm: not available}
"""
def azure_rows(rows=CRUCIBLE, location="eastus"):
return [{"id": RG + path, "type": rtype, "location": location} for path, rtype in rows]
def lister(items, page=4, fail_at=None):
calls = []
def call(tool, args):
calls.append((tool, args))
start = int(args.get("page_token") or 0)
if fail_at is not None and start >= fail_at:
return {"success": False, "error": {"code": "upstream_failed", "message": "429"}}
chunk, nxt = items[start:start + page], start + page
return {"success": True, "data": {"items": chunk, "next_page_token": str(nxt) if nxt < len(items) else None,
"evidence_ref": f"sha256:{start}"}}
return call, calls
def payload(**extra):
return {"engagement_id": "E", "run_id": "RUN-1", "provider": "azure", "include_cmdb": "no", "include_itsm": "no",
"source_documents": CONTEXT, **extra}
def test_every_resource_in_the_group_is_accounted_for():
call, calls = lister(azure_rows())
result = plan.build(payload(), call)
jsonschema.validate(result, OUTPUT)
acc = result["scope_accounting"]
assert acc["listed"] == 11 and acc["scope_entries"] == 10 and acc["structure"] == 0
assert [f["id"].split("/")[-1] for f in acc["folded"]] == ["pe-1.nic"]
scans = [t["target_ref"].split("/")[-1] for t in result["tasks"] if t["type"] == "scan_target"]
assert len(scans) == 10 and {"kv-a", "kv-b", "id-a", "id-b", "pe-1", "example.com", "topic-1"} <= set(scans)
assert result["summary"] == {"scope_entries": 10, "planned": 10, "gaps": 0, "scan_batches": 1}
assert {t for t, _ in calls} == {"azure_list_resources"} and all(a["page_size"] == 200 for _, a in calls)
def test_data_stores_get_a_classification_task_after_their_scan():
result = plan.build(payload(), lister(azure_rows())[0])
by_id = {t["task_id"]: t for t in result["tasks"]}
classify = [t for t in result["tasks"] if t["type"] == "classify_data"]
assert sorted(t["target_ref"].split("/")[-1] for t in classify) == ["crucibkabdevexec", "crucible-kab-dev-pg"]
for t in classify:
assert by_id[t["depends_on"][0]]["target_ref"] == t["target_ref"]
def test_network_and_dependency_tasks_close_the_plan():
tasks = plan.build(payload(), lister(azure_rows())[0])["tasks"]
assert [t["type"] for t in tasks[-2:]] == ["collect_network", "trace_dependencies"]
assert set(tasks[-1]["depends_on"]) == {t["task_id"] for t in tasks[:-1]}
def test_paging_reads_every_page():
rows = azure_rows([(f"Microsoft.KeyVault/vaults/kv-{i:03d}", "microsoft.keyvault/vaults") for i in range(23)])
call, calls = lister(rows, page=5)
result = plan.build(payload(), call)
assert result["scope_accounting"]["listed"] == 23 and len(calls) == 5 and result["summary"]["scan_batches"] == 3
def test_an_incomplete_listing_fails_the_step():
with pytest.raises(plan.ScopeIncomplete, match="429"):
plan.build(payload(), lister(azure_rows(), page=4, fail_at=4)[0])
def test_structure_is_never_a_scope_entry():
rows = azure_rows() + [{"id": "/subscriptions/s/resourceGroups/crucible-kab-dev",
"type": "microsoft.resources/subscriptions/resourcegroups", "location": "eastus"}]
acc = plan.build(payload(), lister(rows)[0])["scope_accounting"]
assert acc["structure"] == 1 and acc["scope_entries"] == 10
def test_the_context_narrows_and_never_widens():
ctx = CONTEXT.replace("scope: {regions: [], exclude: []}", "scope: {regions: [westeurope], exclude: [kv-a]}")
result = plan.build(payload(source_documents=ctx), lister(azure_rows())[0])
assert result["summary"]["planned"] == 0 and result["summary"]["gaps"] == 10
kv = next(g for g in result["coverage_gaps"] if g["scope_entry_id"].endswith("kv-a"))
assert "exclude" in kv["reason"]
assert not [t for t in result["tasks"] if t["type"] == "scan_target"]
def test_source_gaps_and_ids_follow_the_run():
result = plan.build(payload(include_itsm="yes"), lister(azure_rows())[0])
gaps = {g["source"]: g["reason"] for g in result["source_gaps"]}
assert gaps == {"cmdb": "not_requested", "itsm": "no_connection", "netflow": "no_connection", "schema": "no_connection"}
assert result["plan_id"] == "PLAN-RUN-1" and 'plan "CP1 trial"' in result["findings_note"]
def test_gcp_folds_node_pools_and_disks_and_uses_regions():
items = [{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1",
"assetType": "container.googleapis.com/Cluster", "location": "europe-west2"},
{"name": "//container.googleapis.com/projects/p/locations/europe-west2/clusters/c1/nodePools/np",
"assetType": "container.googleapis.com/NodePool", "location": "europe-west2"},
{"name": "//compute.googleapis.com/projects/p/zones/europe-west2-b/disks/d1",
"assetType": "compute.googleapis.com/Disk", "location": "europe-west2-b"},
{"name": "//cloudresourcemanager.googleapis.com/projects/123",
"assetType": "cloudresourcemanager.googleapis.com/Project", "location": "global"}]
ctx = CONTEXT.replace("regions: []", "regions: [europe-west2]")
result = plan.build(payload(provider="gcp", source_documents=ctx), lister(items)[0])
acc = result["scope_accounting"]
assert (acc["listed"], acc["scope_entries"], len(acc["folded"]), acc["structure"]) == (4, 1, 2, 1)
assert result["summary"]["planned"] == 1
assert result["tasks"][0]["mcp_tool"] == "gcp-discovery.gcp_get_resource_metadata"
assert scope.region("europe-west2-b") == "europe-west2"
def test_repeated_rows_are_counted_once():
rows = azure_rows() + azure_rows()[:2]
assert plan.build(payload(), lister(rows)[0])["scope_accounting"]["listed"] == 11
def test_the_critic_reads_the_same_rules():
critic = ROOT.parent / "cp1-discovery-critic-agent" / "app" / "scope.py"
assert critic.read_text() == (ROOT / "app" / "scope.py").read_text()
def cmdb_upload(context=CONTEXT, include="yes"):
ctx = context + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
return payload(include_cmdb=include, source_documents=ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n")
def test_an_uploaded_cmdb_is_a_source_and_gets_an_ingest_task():
result = plan.build(cmdb_upload(), lister(azure_rows())[0])
jsonschema.validate(result, OUTPUT)
assert "cmdb" not in {g["source"] for g in result["source_gaps"]}
[ingest] = [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
assert ingest["target_ref"] == "upload:cmdb-export.csv" and ingest["assigned_agent"] == "ALZ-DSC-04"
assert ingest["task_id"] in result["tasks"][-1]["depends_on"]
assert "cmdb-export.csv" in result["findings_note"] and result["plan_id"] == "PLAN-RUN-1"
def test_an_upload_is_ignored_when_the_cmdb_is_not_requested():
result = plan.build(cmdb_upload(include="no"), lister(azure_rows())[0])
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "not_requested"
assert not [t for t in result["tasks"] if t["type"] == "ingest_cmdb"]
def test_a_requested_cmdb_with_no_upload_is_still_a_gap():
ctx = CONTEXT + "uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
result = plan.build(payload(include_cmdb="yes", source_documents=ctx), lister(azure_rows())[0])
assert {g["source"]: g["reason"] for g in result["source_gaps"]}["cmdb"] == "no_connection"
def test_a_relationship_export_is_ingested_with_its_cmdb():
ctx = CONTEXT + ("uploads:\n - {name: cmdb-export.csv, source: cmdb}\n"
" - {name: cmdb-relationships.csv, source: cmdb_relationships}\n")
text = ctx + "\n\n---\n\n sys_id name\n0 CI1 a\n" + "\n\n---\n\n parent child\n0 CI1 CI2\n"
result = plan.build(payload(include_cmdb="yes", source_documents=text), lister(azure_rows())[0])
jsonschema.validate(result, OUTPUT)
assert [t["target_ref"] for t in result["tasks"] if t["type"].startswith("ingest_cmdb")] == [
"upload:cmdb-export.csv", "upload:cmdb-relationships.csv"]

View File

@@ -0,0 +1,113 @@
"""The upload split into the run context and its listed files, as parse_assessment_files renders them (ADR 0018)."""
import io
import sys
from pathlib import Path
import pandas as pd
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import run_context as rc # noqa: E402
CASE = ROOT.parent / "workflows" / "cases" / "azure-crucible-kab-dev-cmdb"
SEP = "\n\n---\n\n"
def rendered_csv(path):
return pd.read_csv(path).to_string() # what data_file_processing.parse_csv hands on
def rendered_xlsx(frames):
buffer = io.BytesIO()
with pd.ExcelWriter(buffer) as writer:
for name, frame in frames.items():
frame.to_excel(writer, sheet_name=name, index=False)
parts = []
for name, frame in pd.read_excel(io.BytesIO(buffer.getvalue()), sheet_name=None).items():
parts += [f"=== Sheet: {name} ===", frame.to_string(), ""]
return "\n".join(parts) # data_file_processing.parse_excel
def upload(*files):
return SEP.join(files)
FULL_CONTEXT = (CASE / "context.yaml").read_text()
# The case lists a CI export and a relationship export; most tests here need a context with one listed upload.
CONTEXT = FULL_CONTEXT.replace(""" - name: cmdb-relationships.csv
source: cmdb_relationships
format: servicenow_cmdb_rel_ci_csv
taken: 2026-11-04
""", "")
assert CONTEXT != FULL_CONTEXT
def test_a_context_alone_reads_as_before():
single = (ROOT.parent / "workflows" / "cases" / "azure-crucible-kab-dev" / "context.yaml").read_text()
out = rc.split(single)
assert out["context"]["workflow_inputs"]["provider"] == "azure" and out["uploads"] == []
assert rc.context_section(single)["sources"]["cmdb"] == "not available"
def test_a_csv_after_the_context_leaves_the_context_readable():
text = upload(CONTEXT, rendered_csv(CASE / "cmdb-export.csv"))
out = rc.split(text)
assert out["context"]["plan"]["name"].endswith("with CMDB")
[cmdb] = rc.uploads_from(text, "cmdb")
[table] = rc.tables(cmdb)
assert table["sheet"] is None and "object_id" in table["header"] and len(table["rows"]) == 12
assert table["rows"][0][0] == 0 and "CI0010001" in table["rows"][0][1]
assert table["rows"][11][0] == 11 and "Retired" in table["rows"][11][1]
def test_two_files_in_order_and_an_xlsx_has_one_table_per_sheet():
ctx = CONTEXT.replace("uploads:\n", "uploads:\n - {name: assets.xlsx, source: cmdb}\n")
sheets = rendered_xlsx({"Servers": pd.DataFrame({"Name": ["a", "b"], "Env": ["prod", "dev"]}),
"Cloud": pd.DataFrame({"Id": ["/subscriptions/s/x"], "Env": ["dev"]})})
text = upload(ctx, sheets, rendered_csv(CASE / "cmdb-export.csv"))
names = [u["name"] for u in rc.uploads_from(text, "cmdb")]
assert names == ["assets.xlsx", "cmdb-export.csv"]
tables = rc.tables(rc.uploads_from(text, "cmdb")[0])
assert [(t["sheet"], len(t["rows"])) for t in tables] == [("Servers", 2), ("Cloud", 1)]
assert "Env" in tables[1]["header"]
def test_a_separator_inside_the_last_upload_is_kept():
text = upload(CONTEXT, "h\n0 a" + SEP + "1 b")
assert rc.split(text)["uploads"][0]["text"] == "h\n0 a" + SEP + "1 b"
def test_a_listed_upload_that_never_arrived_is_unreadable():
[cmdb] = rc.uploads_from(CONTEXT, "cmdb")
assert cmdb["text"] is None
with pytest.raises(rc.UploadUnreadable, match="not uploaded"):
rc.tables(cmdb)
def test_a_platform_parse_error_is_unreadable():
[cmdb] = rc.uploads_from(upload(CONTEXT, "CSV parsing error: bad line 3"), "cmdb")
with pytest.raises(rc.UploadUnreadable, match="could not parse"):
rc.tables(cmdb)
def test_an_empty_export_has_no_rows():
[cmdb] = rc.uploads_from(upload(CONTEXT, pd.DataFrame(columns=["a", "b"]).to_string()), "cmdb")
assert rc.tables(cmdb)[0]["rows"] == []
def test_a_raw_csv_is_numbered_by_position():
[cmdb] = rc.uploads_from(upload(CONTEXT, "id,name\n123,a\n456,b"), "cmdb")
assert rc.tables(cmdb)[0]["rows"] == [(0, "123,a"), (1, "456,b")]
def test_a_text_brief_is_not_a_context():
assert rc.split("Meridian brief\n\nSome text.")["context"] == {}
def test_the_case_lists_two_cmdb_uploads_in_order():
rels = " sys_id parent child\n0 R1 CI1 CI2"
text = upload(FULL_CONTEXT, rendered_csv(CASE / "cmdb-export.csv"), rels)
assert [u["name"] for u in rc.split(text)["uploads"]] == ["cmdb-export.csv", "cmdb-relationships.csv"]
[rel] = rc.uploads_from(text, "cmdb_relationships")
assert rc.tables(rel)[0]["rows"] == [(0, "0 R1 CI1 CI2")]