Files
Jonathan Boniface 9e503885cc ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

326 lines
16 KiB
Python

"""Uploaded CMDB exports normalised by a stand-in model, judged and reconciled in code (ADR 0018)."""
import asyncio
import io
import json
import sys
from pathlib import Path
import jsonschema
import pandas as pd
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import cmdb, live # noqa: E402
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
CASE = ROOT.parent / "workflows" / "cases" / "azure-crucible-kab-dev-cmdb"
SEP = "\n\n---\n\n"
SUB = "/subscriptions/1e06e3e6-7ab6-41c9-a29c-dd8f55646b7e/resourceGroups/crucible-kab-dev/providers/"
GROUP = [ # crucible-kab-dev on 2026-11-04: (path, type, location)
("Microsoft.ContainerService/managedClusters/crucible-kab-dev-aks", "microsoft.containerservice/managedclusters", "eastus"),
("Microsoft.Storage/storageAccounts/crucibkabdevexec", "microsoft.storage/storageaccounts", "eastus"),
("Microsoft.DBforPostgreSQL/flexibleServers/crucible-kab-dev-pg", "microsoft.dbforpostgresql/flexibleservers", "eastus"),
("Microsoft.KeyVault/vaults/crucible-kab-dev-kv", "microsoft.keyvault/vaults", "eastus"),
("Microsoft.KeyVault/vaults/crucible-kab-dev-creds", "microsoft.keyvault/vaults", "eastus"),
("Microsoft.ManagedIdentity/userAssignedIdentities/crucible-kab-dev-certmgr", "microsoft.managedidentity/userassignedidentities", "eastus"),
("Microsoft.ManagedIdentity/userAssignedIdentities/crucible-kab-dev-vault", "microsoft.managedidentity/userassignedidentities", "eastus"),
("Microsoft.Network/privateEndpoints/crucible-kab-dev-pg-pe", "microsoft.network/privateendpoints", "eastus"),
("Microsoft.Network/networkInterfaces/crucible-kab-dev-pg-pe.nic.0aa4d373", "microsoft.network/networkinterfaces", "eastus"),
("Microsoft.Network/dnszones/kab-dev.kyndemo.live", "microsoft.network/dnszones", "global"),
("Microsoft.EventGrid/systemTopics/crucibkabdevexec-7c36cb21", "microsoft.eventgrid/systemtopics", "eastus"),
]
ITEMS = [{"id": SUB + p, "type": t, "location": loc} for p, t, loc in GROUP]
RECORDS = [{"canonical_id": f"CI-{i:03d}", "scope_entry_id": item["id"], "environment": "non_production",
"facts": {"site": item["location"]}, "sources": [{"evidence_ref": f"sha256:scan{i}"}]}
for i, item in enumerate((x for x in ITEMS if "networkInterfaces" not in x["id"]), start=1)]
CSV_MAP = {"ci_ref": "sys_id", "name": "name", "cloud_resource_id": "object_id", "ci_class": "sys_class_name",
"environment_raw": "environment", "location_raw": "location", "status_raw": "operational_status",
"owner": "owned_by", "business_service": "business_service", "support_group": "support_group"}
ENV = {"dev": "non_production", "prod": "production", "test": "non_production"}
def frame_of(path):
return pd.read_csv(path)
def rendered(frame):
return frame.to_string() # data_file_processing.parse_csv
FULL_CONTEXT = (CASE / "context.yaml").read_text()
RELS_ENTRY = """ - name: cmdb-relationships.csv
source: cmdb_relationships
format: servicenow_cmdb_rel_ci_csv
taken: 2026-11-04
"""
assert RELS_ENTRY in FULL_CONTEXT
CI_ONLY_CONTEXT = FULL_CONTEXT.replace(RELS_ENTRY, "")
def source(*uploads, context=None):
"""The upload text: the context first, then the files. One file: the context without the relationship entry."""
return SEP.join([context or (FULL_CONTEXT if len(uploads) > 1 else CI_ONLY_CONTEXT), *uploads])
def model(frames, mapping, flag_injection=True, spoil=None):
"""A stand-in normaliser that knows the test data's columns (the code under test never does)."""
calls = []
async def ask(prompt):
calls.append(prompt)
request = json.loads(prompt.split("\n\n")[1])
frame = frames[(request["file"], request["sheet"])]
cis = []
for row in request["rows"]:
values = frame.iloc[row["row"]]
ci = {"row": row["row"]}
for field, column in mapping.items():
value = values.get(column)
ci[field] = None if pd.isna(value) else str(value)
status = (ci.get("status_raw") or "").lower()
ci["environment"] = ENV.get((ci.get("environment_raw") or "").lower(), "unknown")
ci["lifecycle"] = "retired" if status in ("retired", "decommissioned") else "active" if status else "unknown"
ci["injection_detected"] = flag_injection and "AI AGENT" in row["line"]
cis.append(ci)
if spoil:
spoil(cis, len(calls))
return json.dumps({"cis": cis})
ask.calls = calls
return ask
def lister(items, fail=False):
async def listing(provider):
if fail:
return items[:3], ["sha256:p0"], ["azure_list_resources: 429"]
return items, ["sha256:p0"], []
return listing
async def no_sleep(_):
return None
def payload(text, **extra):
return {"engagement_id": "E", "run_id": "RUN-1", "provider": "azure", "records": RECORDS,
"source_gaps": [{"source": "itsm"}], "source_documents": text, **extra}
def run(text, ask, listing=None, **extra):
return asyncio.run(live.build(payload(text, **extra), ask, listing or lister(ITEMS), batch_size=5,
sleep=no_sleep))
def case_run(**kwargs):
frame = frame_of(CASE / "cmdb-export.csv")
ask = model({("cmdb-export.csv", None): frame}, CSV_MAP, **kwargs)
return run(source(rendered(frame)), ask), ask
def by_ci(result):
out = {}
for d in result["drift_items"]:
out.setdefault(d["ci_ref"] or d["canonical_id"], []).append(d)
return out
def test_the_case_export_reconciles_as_planted():
result, ask = case_run()
jsonschema.validate(result, OUTPUT)
found = by_ci(result)
[loc] = found["CI0010002"]
assert (loc["type"], loc["field"], loc["discovered_value"], loc["cmdb_value"]) == ("stale_attribute", "location", "eastus", "eastus2")
[env] = found["CI0010003"]
assert (env["field"], env["discovered_value"], env["cmdb_value"]) == ("environment", "non_production", "prod")
assert found["CI0010009"][0]["type"] == "ghost" and found["CI0010009"][0]["suggested_action"] == "retire_ci"
assert found["CI0010010"][0]["type"] == "ghost" and found["CI0010010"][0]["suggested_action"] == "investigate"
orphans = {d["canonical_id"] for d in result["drift_items"] if d["type"] == "orphan"}
names = {r["canonical_id"]: r["scope_entry_id"].split("/")[-1] for r in RECORDS}
assert {names[c] for c in orphans} == {"kab-dev.kyndemo.live", "crucibkabdevexec-7c36cb21"}
assert all(d["field"] == "resource" and d["discovered_value"].startswith("/subscriptions/")
for d in result["drift_items"] if d["type"] == "orphan")
assert "CI0010011" not in found and "CI0010012" not in found
s = result["summary"]
assert (s["matched"], s["stale_attributes"], s["orphans"], s["ghosts"]) == (8, 2, 2, 2)
assert (s["retired"], s["out_of_scope"], s["unplaced"]) == (1, 1, 0)
assert result["cmdb_sources"] == [{"file": "cmdb-export.csv", "sheet": None, "kind": "cis", "rows": 12, "cis": 12, "skipped": 0}]
assert len(result["injections_found"]) == 1 and result["injections_found"][0].startswith("CI0010001 (cmdb-export.csv row 0)")
assert len(ask.calls) == 3 # 12 rows in batches of 5
assert all(ref.startswith("upload:cmdb-export.csv#:") for d in result["drift_items"] if d["ci_ref"]
for ref in d["evidence_refs"][:1])
def test_code_flags_an_injection_the_model_missed():
result, _ = case_run(flag_injection=False)
assert result["injections_found"] and "NOTE TO AI AGENT" in result["injections_found"][0]
def test_a_value_not_in_the_row_is_rejected_and_retried():
def spoil(cis, call):
if call == 1:
cis[0]["owner"] = "Somebody Invented"
result, ask = case_run(spoil=spoil)
assert "not in the row as written" in ask.calls[1] and result["summary"]["matched"] == 8
def test_a_batch_that_fails_three_tries_fails_the_step():
def spoil(cis, call):
cis.pop()
with pytest.raises(cmdb.CmdbFailed, match="failed all 3 tries.*no result"):
case_run(spoil=spoil)
def test_judgments_must_be_allowed_values():
def spoil(cis, call):
cis[0]["environment"] = "development"
with pytest.raises(cmdb.CmdbFailed, match="environment must be one of"):
case_run(spoil=spoil)
def test_no_cmdb_source_reads_nothing():
ask = model({}, CSV_MAP)
result = run(source("x"), ask, source_gaps=[{"source": "cmdb"}, {"source": "itsm"}])
jsonschema.validate(result, OUTPUT)
assert result["drift_items"] == [] and not ask.calls and "declares no CMDB" in result["note"]
def test_an_incomplete_listing_fails_the_step():
frame = frame_of(CASE / "cmdb-export.csv")
with pytest.raises(live.ScopeIncomplete, match="429"):
run(source(rendered(frame)), model({("cmdb-export.csv", None): frame}, CSV_MAP), lister(ITEMS, fail=True))
def test_an_unreadable_upload_fails_the_step():
with pytest.raises(cmdb.CmdbFailed, match="could not parse"):
run(source("CSV parsing error: line 4"), model({}, CSV_MAP))
def test_any_layout_an_xlsx_with_other_column_names_and_two_sheets():
cloud = pd.DataFrame({"Configuration Item": ["A-1", "A-2"], "Resource ID": [ITEMS[0]["id"], None],
"Hostname": ["crucible-kab-dev-aks", "crucible-kab-dev-kv"], "Env": ["prod", "dev"],
"Region": ["East US", "eastus"], "State": ["In Use", "In Use"], "Criticality": ["High", None]})
onprem = pd.DataFrame({"Configuration Item": ["B-1"], "Resource ID": [None], "Hostname": ["lon-db-01"],
"Env": ["prod"], "Region": ["LON-DC1"], "State": ["In Use"], "Criticality": ["High"]})
buffer = io.BytesIO()
with pd.ExcelWriter(buffer) as writer:
cloud.to_excel(writer, sheet_name="Cloud", index=False)
onprem.to_excel(writer, sheet_name="On-prem", index=False)
sheets = pd.read_excel(io.BytesIO(buffer.getvalue()), sheet_name=None)
text = "\n".join(p for name, f in sheets.items() for p in (f"=== Sheet: {name} ===", f.to_string(), ""))
context = CI_ONLY_CONTEXT.replace("name: cmdb-export.csv", "name: assets.xlsx")
mapping = {"ci_ref": "Configuration Item", "cloud_resource_id": "Resource ID", "name": "Hostname",
"environment_raw": "Env", "location_raw": "Region", "status_raw": "State"}
def keep_criticality(cis, call):
for ci in cis:
if ci["ci_ref"] in ("A-1", "B-1"):
ci["other_attributes"] = {"Criticality": "High"}
for ci in cis:
if ci["status_raw"] == "In Use":
ci["lifecycle"] = "active"
ask = model({("assets.xlsx", name): f for name, f in sheets.items()}, mapping, spoil=keep_criticality)
result = run(source(text, context=context), ask)
jsonschema.validate(result, OUTPUT)
assert [(s["sheet"], s["cis"]) for s in result["cmdb_sources"]] == [("Cloud", 2), ("On-prem", 1)]
found = by_ci(result)
assert [d["field"] for d in found["A-1"]] == ["environment"] # "East US" is eastus: not drift
assert "A-2" not in found # matched by its unique name
assert result["summary"]["unplaced"] == 1 # lon-db-01 has no cloud id: never a ghost
def test_an_other_attribute_must_be_a_header_column():
def spoil(cis, call):
cis[0]["other_attributes"] = {"invented_column": "dev"}
with pytest.raises(cmdb.CmdbFailed, match="not a column in the header"):
case_run(spoil=spoil)
def test_shared_files_are_the_planners():
planner = ROOT.parent / "cp1-discovery-planner-agent" / "app"
for name in ("run_context.py", "scope.py"):
assert (ROOT / "app" / name).read_text() == (planner / name).read_text()
REL_MAP = {"parent_ref": "parent", "child_ref": "child", "type_raw": "type"}
def rel_model(frame):
calls = []
async def ask(prompt):
calls.append(prompt)
request = json.loads(prompt.split("\n\n")[1])
out = []
for row in request["rows"]:
values = frame.iloc[row["row"]]
out.append({"row": row["row"], **{f: str(values[c]) for f, c in REL_MAP.items()},
"type": "app_to_db", "injection_detected": False})
return json.dumps({"relationships": out})
ask.calls = calls
return ask
def with_relationships():
cis = frame_of(CASE / "cmdb-export.csv")
rels = frame_of(CASE / "cmdb-relationships.csv")
text = source(rendered(cis), rendered(rels))
return asyncio.run(live.build(payload(text), model({("cmdb-export.csv", None): cis}, CSV_MAP), lister(ITEMS),
batch_size=5, sleep=no_sleep, ask_relationships=rel_model(rels)))
def test_relationships_resolve_by_id_or_name_and_dangling_ends_are_reported():
result = with_relationships()
jsonschema.validate(result, OUTPUT)
pairs = {(r["parent_ref"], r["child_ref"]) for r in result["relationships"]}
assert pairs == {("CI0010008", "CI0010002"), ("CI0010001", "CI0010002")} # the second child is a name
[dangling] = result["dangling_relationships"]
assert dangling["child"] == "CI0099999" and dangling["unresolved"] == ["child"]
assert {s["kind"] for s in result["cmdb_sources"]} == {"cis", "relationships"}
assert "2 relationships between CIs, 1 dangling" in result["note"]
def test_every_ci_has_a_result_for_the_graph():
results = {c["ci_ref"]: c for c in with_relationships()["cis"]}
assert len(results) == 12
assert results["CI0010002"]["result"] == "matched" and results["CI0010002"]["match_basis"] == "cloud_resource_id"
assert results["CI0010002"]["matched_canonical_id"].startswith("CI-")
assert (results["CI0010009"]["result"], results["CI0010011"]["result"], results["CI0010012"]["result"]) == \
("ghost", "out_of_scope", "retired")
def test_each_ci_result_carries_its_ownership_verbatim():
"""ADR 0021: the dependency step groups applications and names owners from these."""
result = with_relationships()
jsonschema.validate(result, OUTPUT)
ci = {c["ci_ref"]: c for c in result["cis"]}["CI0010001"]
assert (ci["owner"], ci["business_service"], ci["support_group"]) == ("Platform Engineering", "KAB Platform",
"PE-Cloud-Ops")
def test_a_relationship_end_not_in_its_row_is_rejected():
rels = frame_of(CASE / "cmdb-relationships.csv")
bad = rel_model(rels)
async def ask(prompt):
reply = json.loads(await bad(prompt))
reply["relationships"][0]["child_ref"] = "CI0010005"
return json.dumps(reply)
cis = frame_of(CASE / "cmdb-export.csv")
with pytest.raises(cmdb.CmdbFailed, match="relationships batch.*not in the row as written"):
asyncio.run(live.build(payload(source(rendered(cis), rendered(rels))),
model({("cmdb-export.csv", None): cis}, CSV_MAP), lister(ITEMS), batch_size=5,
sleep=no_sleep, ask_relationships=ask))
def test_a_ci_rows_own_parent_column_becomes_a_relationship():
def parent_of_pg(cis, call): # the private endpoint's row names crucible-kab-dev-pg (inside its object_id)
for ci in cis:
if ci["ci_ref"] == "CI0010008":
ci["related"] = [{"ref": "crucible-kab-dev-pg", "role": "child", "type": "app_to_db",
"type_raw": None}]
frame = frame_of(CASE / "cmdb-export.csv")
result = run(source(rendered(frame)), model({("cmdb-export.csv", None): frame}, CSV_MAP, spoil=parent_of_pg))
assert {(r["parent_ref"], r["child_ref"]) for r in result["relationships"]} == {("CI0010008", "CI0010002")}