ALZ-F01 Discovery: package step, value scorecard, application grouping, cases

- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent f659564208
commit 9e503885cc
211 changed files with 18170 additions and 158 deletions

View File

@@ -0,0 +1,218 @@
"""The live inventory built in batches (ADR 0015), against a stand-in sub-agent (no runtime, no model)."""
import asyncio
import json
import sys
from pathlib import Path
import jsonschema
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import inventory # noqa: E402
OUTPUT_SCHEMA = json.loads((ROOT / "schemas" / "output.json").read_text())
SUB = "/subscriptions/s/resourceGroups/g/providers/Microsoft.Compute/virtualMachines/"
CONTEXT = """generated:
run_id: R
context:
owners: {}
business_services: []
assets:
vm-1:
owner: Platform Engineering
criticality: low
environment: development
"""
def finding(i, tags=None, region="eastus2"):
attrs = {"type": "microsoft.compute/virtualmachines", "region": region}
if tags:
attrs["tags"] = tags
return {"scope_entry_id": f"{SUB}vm-{i}", "hostname": f"vm-{i}", "os": "Ubuntu 22.04", "services": [],
"dependencies": [], "migration_risks": [], "prompt_injection_detected": False, "injection_note": "",
"evidence_ref": f"sha256:{i}", "provider_attributes": attrs}
def judged(f, **extra):
return {"scope_entry_id": f["scope_entry_id"], "hostname": f["hostname"], "fqdn": "", "asset_class": "server",
"environment": "unknown", "os": {"family": "ubuntu", "version": "22.04"}, "services": [], "facts": {},
"fact_source": "provider_api", "conflicts": [], **extra}
class Model:
"""Answers each batch from its prompt; `script` overrides the answer for given call numbers."""
def __init__(self, script=None):
self.calls, self.script = [], script or {}
async def __call__(self, prompt):
self.calls.append(prompt)
request = json.loads(prompt.split("\n\n", 1)[1].split("\n\nYour previous reply", 1)[0])
action = self.script.get(len(self.calls))
if isinstance(action, Exception):
raise action
if callable(action):
return action(request)
return json.dumps({"records": [judged(f) for f in request["findings"]]})
def run(payload, model, **kw):
waits = []
async def sleep(seconds):
waits.append(seconds)
result = asyncio.run(inventory.build(payload, model, sleep=sleep, **kw))
return result, waits
def payload(n, **extra):
return {"engagement_id": "E", "run_id": "R", "provider": "azure", "findings": [finding(i) for i in range(1, n + 1)],
**extra}
def test_97_findings_make_5_batches_and_97_records():
model = Model()
result, waits = run(payload(97), model)
assert len(model.calls) == 5 and waits == []
assert [r["canonical_id"] for r in result["records"]] == [f"CI-{i:03d}" for i in range(1, 98)]
assert [r["scope_entry_id"] for r in result["records"]] == [f"{SUB}vm-{i}" for i in range(1, 98)]
jsonschema.validate(result, OUTPUT_SCHEMA)
def test_code_sets_the_copied_and_counted_fields():
p = payload(1)
p["findings"][0]["provider_attributes"]["location"] = "ignored"
record = run(p, Model())[0]["records"][0]
assert record["sources"] == [{"source_type": "provider_api", "source_ref": f"{SUB}vm-1", "evidence_ref": "sha256:1"}]
assert record["match_basis"] == ["cloud_resource_id"] and record["confidence"] == 0.85
assert record["facts"]["site"] == "eastus2"
assert record["provider_attributes"] == p["findings"][0]["provider_attributes"]
def test_a_gcp_zone_becomes_its_region():
assert inventory.site({"location": "europe-west2-b"}) == "europe-west2"
assert inventory.site({"location": "europe-west2"}) == "europe-west2"
assert inventory.site({"region": "eastus2"}) == "eastus2"
def test_bad_json_is_fixed_on_the_second_try():
model = Model({1: lambda request: "Here is the inventory: {"})
result, waits = run(payload(3), model)
assert len(result["records"]) == 3 and len(model.calls) == 2 and waits == [10]
assert "not valid JSON" in model.calls[1]
def test_a_markdown_fence_is_tolerated():
model = Model({1: lambda r: "```json\n" + json.dumps({"records": [judged(f) for f in r["findings"]]}) + "\n```"})
assert len(run(payload(2), model)[0]["records"]) == 2 and len(model.calls) == 1
def test_a_missing_record_costs_a_try_and_is_named():
model = Model({1: lambda r: json.dumps({"records": [judged(f) for f in r["findings"][1:]]})})
result, _ = run(payload(3), model)
assert len(result["records"]) == 3
assert f"{SUB}vm-1: no record" in model.calls[1]
def test_an_empty_reply_never_passes():
empty = lambda r: json.dumps({"records": []}) # noqa: E731 - what 0.5.1's retry returned
with pytest.raises(inventory.InventoryFailed, match="batch 1 of 1 .* failed all 3 tries"):
run(payload(97, ), Model({1: empty, 2: empty, 3: empty}), batch_size=100)
def test_three_failures_fail_the_step_and_later_batches_are_not_asked():
bad = lambda r: "not json" # noqa: E731
model = Model({2: bad, 3: bad, 4: bad})
with pytest.raises(inventory.InventoryFailed, match=r"batch 2 of 3 \(findings 21-40\)"):
run(payload(50), model)
assert len(model.calls) == 4 # batch 1 once, batch 2 three times, batch 3 never
def test_max_tries_is_the_total_number_of_tries():
bad = lambda r: "not json" # noqa: E731
model = Model({i: bad for i in range(1, 10)})
with pytest.raises(inventory.InventoryFailed):
run(payload(1), model, max_tries=3)
assert len(model.calls) == 3
def test_a_failed_model_call_counts_as_a_try():
model = Model({1: RuntimeError("429 token rate limit"), 2: RuntimeError("429 token rate limit")})
result, waits = run(payload(2), model)
assert len(result["records"]) == 2 and waits == [10, 30]
assert "429" in model.calls[2]
def test_an_extra_or_repeated_record_is_refused():
def extra(r):
return json.dumps({"records": [judged(f) for f in r["findings"]] + [judged(finding(99))]})
def repeated(r):
return json.dumps({"records": [judged(r["findings"][0])] * 2 + [judged(f) for f in r["findings"][1:]]})
model = Model({1: extra, 2: repeated})
run(payload(2), model)
assert "not a finding in this batch" in model.calls[1] and "more than one record" in model.calls[2]
def test_a_fact_found_nowhere_is_refused():
model = Model({1: lambda r: json.dumps({"records": [judged(f, facts={"owner": "Finance Team"})
for f in r["findings"]]})})
run(payload(1), model)
assert "facts.owner 'Finance Team' is in neither" in model.calls[1]
def test_facts_from_tags_and_the_context_file_are_kept():
p = {"engagement_id": "E", "run_id": "R", "provider": "azure", "source_documents": CONTEXT,
"findings": [finding(1, tags={"Criticality": "High"})]}
reply = judged(p["findings"][0], environment="non_production",
facts={"owner": "Platform Engineering", "criticality": "High"},
fact_source="provider_api; document:source#context.assets.vm-1",
conflicts=[{"field": "criticality", "values": [
{"value": "High", "source_ref": "provider_api"},
{"value": "low", "source_ref": "document:source#context.assets.vm-1"}]}])
model = Model({1: lambda r: json.dumps({"records": [reply]})})
result, _ = run(p, model)
record = result["records"][0]
assert record["facts"] == {"owner": "Platform Engineering", "criticality": "High", "site": "eastus2"}
assert record["confidence"] == 0.79 and result["sme_review_queue"] == ["CI-001"]
assert '"vm-1": {"owner": "Platform Engineering"' in model.calls[0] # the batch sees the context section
def test_a_bad_enum_is_refused():
model = Model({1: lambda r: json.dumps({"records": [judged(f, environment="staging") for f in r["findings"]]})})
run(payload(1), model)
assert "environment must be one of" in model.calls[1]
def test_repeated_findings_make_one_record():
p = payload(2)
p["findings"].append(dict(p["findings"][0]))
assert len(run(p, Model())[0]["records"]) == 2
def test_no_findings_is_an_empty_inventory_with_no_model_call():
model = Model()
assert run(payload(0), model)[0] == {"records": [], "sme_review_queue": []} and model.calls == []
def test_context_section_tolerates_text_that_is_not_yaml():
assert inventory.context_section("a: [unclosed") == {}
assert inventory.context_section(None) == {}
assert inventory.context_section(CONTEXT)["assets"]["vm-1"]["owner"] == "Platform Engineering"
def test_the_context_is_read_when_a_cmdb_export_follows_it():
import pandas as pd
case = Path(__file__).resolve().parents[2] / "workflows" / "cases" / "azure-crucible-kab-dev-cmdb"
text = (case / "context.yaml").read_text() + "\n\n---\n\n" + pd.read_csv(case / "cmdb-export.csv").to_string()
assert inventory.context_section(text)["scope"]["regions"] == ["eastus", "global"]
def test_the_shared_reader_is_the_planners():
root = Path(__file__).resolve().parents[1]
planner = root.parent / "cp1-discovery-planner-agent" / "app" / "run_context.py"
assert (root / "app" / "run_context.py").read_text() == planner.read_text()