ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
"""The live inventory built in batches (ADR 0015), against a stand-in sub-agent (no runtime, no model)."""
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from app import inventory # noqa: E402
|
||||
|
||||
OUTPUT_SCHEMA = json.loads((ROOT / "schemas" / "output.json").read_text())
|
||||
SUB = "/subscriptions/s/resourceGroups/g/providers/Microsoft.Compute/virtualMachines/"
|
||||
CONTEXT = """generated:
|
||||
run_id: R
|
||||
context:
|
||||
owners: {}
|
||||
business_services: []
|
||||
assets:
|
||||
vm-1:
|
||||
owner: Platform Engineering
|
||||
criticality: low
|
||||
environment: development
|
||||
"""
|
||||
|
||||
|
||||
def finding(i, tags=None, region="eastus2"):
|
||||
attrs = {"type": "microsoft.compute/virtualmachines", "region": region}
|
||||
if tags:
|
||||
attrs["tags"] = tags
|
||||
return {"scope_entry_id": f"{SUB}vm-{i}", "hostname": f"vm-{i}", "os": "Ubuntu 22.04", "services": [],
|
||||
"dependencies": [], "migration_risks": [], "prompt_injection_detected": False, "injection_note": "",
|
||||
"evidence_ref": f"sha256:{i}", "provider_attributes": attrs}
|
||||
|
||||
|
||||
def judged(f, **extra):
|
||||
return {"scope_entry_id": f["scope_entry_id"], "hostname": f["hostname"], "fqdn": "", "asset_class": "server",
|
||||
"environment": "unknown", "os": {"family": "ubuntu", "version": "22.04"}, "services": [], "facts": {},
|
||||
"fact_source": "provider_api", "conflicts": [], **extra}
|
||||
|
||||
|
||||
class Model:
|
||||
"""Answers each batch from its prompt; `script` overrides the answer for given call numbers."""
|
||||
|
||||
def __init__(self, script=None):
|
||||
self.calls, self.script = [], script or {}
|
||||
|
||||
async def __call__(self, prompt):
|
||||
self.calls.append(prompt)
|
||||
request = json.loads(prompt.split("\n\n", 1)[1].split("\n\nYour previous reply", 1)[0])
|
||||
action = self.script.get(len(self.calls))
|
||||
if isinstance(action, Exception):
|
||||
raise action
|
||||
if callable(action):
|
||||
return action(request)
|
||||
return json.dumps({"records": [judged(f) for f in request["findings"]]})
|
||||
|
||||
|
||||
def run(payload, model, **kw):
|
||||
waits = []
|
||||
|
||||
async def sleep(seconds):
|
||||
waits.append(seconds)
|
||||
|
||||
result = asyncio.run(inventory.build(payload, model, sleep=sleep, **kw))
|
||||
return result, waits
|
||||
|
||||
|
||||
def payload(n, **extra):
|
||||
return {"engagement_id": "E", "run_id": "R", "provider": "azure", "findings": [finding(i) for i in range(1, n + 1)],
|
||||
**extra}
|
||||
|
||||
|
||||
def test_97_findings_make_5_batches_and_97_records():
|
||||
model = Model()
|
||||
result, waits = run(payload(97), model)
|
||||
assert len(model.calls) == 5 and waits == []
|
||||
assert [r["canonical_id"] for r in result["records"]] == [f"CI-{i:03d}" for i in range(1, 98)]
|
||||
assert [r["scope_entry_id"] for r in result["records"]] == [f"{SUB}vm-{i}" for i in range(1, 98)]
|
||||
jsonschema.validate(result, OUTPUT_SCHEMA)
|
||||
|
||||
|
||||
def test_code_sets_the_copied_and_counted_fields():
|
||||
p = payload(1)
|
||||
p["findings"][0]["provider_attributes"]["location"] = "ignored"
|
||||
record = run(p, Model())[0]["records"][0]
|
||||
assert record["sources"] == [{"source_type": "provider_api", "source_ref": f"{SUB}vm-1", "evidence_ref": "sha256:1"}]
|
||||
assert record["match_basis"] == ["cloud_resource_id"] and record["confidence"] == 0.85
|
||||
assert record["facts"]["site"] == "eastus2"
|
||||
assert record["provider_attributes"] == p["findings"][0]["provider_attributes"]
|
||||
|
||||
|
||||
def test_a_gcp_zone_becomes_its_region():
|
||||
assert inventory.site({"location": "europe-west2-b"}) == "europe-west2"
|
||||
assert inventory.site({"location": "europe-west2"}) == "europe-west2"
|
||||
assert inventory.site({"region": "eastus2"}) == "eastus2"
|
||||
|
||||
|
||||
def test_bad_json_is_fixed_on_the_second_try():
|
||||
model = Model({1: lambda request: "Here is the inventory: {"})
|
||||
result, waits = run(payload(3), model)
|
||||
assert len(result["records"]) == 3 and len(model.calls) == 2 and waits == [10]
|
||||
assert "not valid JSON" in model.calls[1]
|
||||
|
||||
|
||||
def test_a_markdown_fence_is_tolerated():
|
||||
model = Model({1: lambda r: "```json\n" + json.dumps({"records": [judged(f) for f in r["findings"]]}) + "\n```"})
|
||||
assert len(run(payload(2), model)[0]["records"]) == 2 and len(model.calls) == 1
|
||||
|
||||
|
||||
def test_a_missing_record_costs_a_try_and_is_named():
|
||||
model = Model({1: lambda r: json.dumps({"records": [judged(f) for f in r["findings"][1:]]})})
|
||||
result, _ = run(payload(3), model)
|
||||
assert len(result["records"]) == 3
|
||||
assert f"{SUB}vm-1: no record" in model.calls[1]
|
||||
|
||||
|
||||
def test_an_empty_reply_never_passes():
|
||||
empty = lambda r: json.dumps({"records": []}) # noqa: E731 - what 0.5.1's retry returned
|
||||
with pytest.raises(inventory.InventoryFailed, match="batch 1 of 1 .* failed all 3 tries"):
|
||||
run(payload(97, ), Model({1: empty, 2: empty, 3: empty}), batch_size=100)
|
||||
|
||||
|
||||
def test_three_failures_fail_the_step_and_later_batches_are_not_asked():
|
||||
bad = lambda r: "not json" # noqa: E731
|
||||
model = Model({2: bad, 3: bad, 4: bad})
|
||||
with pytest.raises(inventory.InventoryFailed, match=r"batch 2 of 3 \(findings 21-40\)"):
|
||||
run(payload(50), model)
|
||||
assert len(model.calls) == 4 # batch 1 once, batch 2 three times, batch 3 never
|
||||
|
||||
|
||||
def test_max_tries_is_the_total_number_of_tries():
|
||||
bad = lambda r: "not json" # noqa: E731
|
||||
model = Model({i: bad for i in range(1, 10)})
|
||||
with pytest.raises(inventory.InventoryFailed):
|
||||
run(payload(1), model, max_tries=3)
|
||||
assert len(model.calls) == 3
|
||||
|
||||
|
||||
def test_a_failed_model_call_counts_as_a_try():
|
||||
model = Model({1: RuntimeError("429 token rate limit"), 2: RuntimeError("429 token rate limit")})
|
||||
result, waits = run(payload(2), model)
|
||||
assert len(result["records"]) == 2 and waits == [10, 30]
|
||||
assert "429" in model.calls[2]
|
||||
|
||||
|
||||
def test_an_extra_or_repeated_record_is_refused():
|
||||
def extra(r):
|
||||
return json.dumps({"records": [judged(f) for f in r["findings"]] + [judged(finding(99))]})
|
||||
|
||||
def repeated(r):
|
||||
return json.dumps({"records": [judged(r["findings"][0])] * 2 + [judged(f) for f in r["findings"][1:]]})
|
||||
|
||||
model = Model({1: extra, 2: repeated})
|
||||
run(payload(2), model)
|
||||
assert "not a finding in this batch" in model.calls[1] and "more than one record" in model.calls[2]
|
||||
|
||||
|
||||
def test_a_fact_found_nowhere_is_refused():
|
||||
model = Model({1: lambda r: json.dumps({"records": [judged(f, facts={"owner": "Finance Team"})
|
||||
for f in r["findings"]]})})
|
||||
run(payload(1), model)
|
||||
assert "facts.owner 'Finance Team' is in neither" in model.calls[1]
|
||||
|
||||
|
||||
def test_facts_from_tags_and_the_context_file_are_kept():
|
||||
p = {"engagement_id": "E", "run_id": "R", "provider": "azure", "source_documents": CONTEXT,
|
||||
"findings": [finding(1, tags={"Criticality": "High"})]}
|
||||
reply = judged(p["findings"][0], environment="non_production",
|
||||
facts={"owner": "Platform Engineering", "criticality": "High"},
|
||||
fact_source="provider_api; document:source#context.assets.vm-1",
|
||||
conflicts=[{"field": "criticality", "values": [
|
||||
{"value": "High", "source_ref": "provider_api"},
|
||||
{"value": "low", "source_ref": "document:source#context.assets.vm-1"}]}])
|
||||
model = Model({1: lambda r: json.dumps({"records": [reply]})})
|
||||
result, _ = run(p, model)
|
||||
record = result["records"][0]
|
||||
assert record["facts"] == {"owner": "Platform Engineering", "criticality": "High", "site": "eastus2"}
|
||||
assert record["confidence"] == 0.79 and result["sme_review_queue"] == ["CI-001"]
|
||||
assert '"vm-1": {"owner": "Platform Engineering"' in model.calls[0] # the batch sees the context section
|
||||
|
||||
|
||||
def test_a_bad_enum_is_refused():
|
||||
model = Model({1: lambda r: json.dumps({"records": [judged(f, environment="staging") for f in r["findings"]]})})
|
||||
run(payload(1), model)
|
||||
assert "environment must be one of" in model.calls[1]
|
||||
|
||||
|
||||
def test_repeated_findings_make_one_record():
|
||||
p = payload(2)
|
||||
p["findings"].append(dict(p["findings"][0]))
|
||||
assert len(run(p, Model())[0]["records"]) == 2
|
||||
|
||||
|
||||
def test_no_findings_is_an_empty_inventory_with_no_model_call():
|
||||
model = Model()
|
||||
assert run(payload(0), model)[0] == {"records": [], "sme_review_queue": []} and model.calls == []
|
||||
|
||||
|
||||
def test_context_section_tolerates_text_that_is_not_yaml():
|
||||
assert inventory.context_section("a: [unclosed") == {}
|
||||
assert inventory.context_section(None) == {}
|
||||
assert inventory.context_section(CONTEXT)["assets"]["vm-1"]["owner"] == "Platform Engineering"
|
||||
|
||||
|
||||
def test_the_context_is_read_when_a_cmdb_export_follows_it():
|
||||
import pandas as pd
|
||||
case = Path(__file__).resolve().parents[2] / "workflows" / "cases" / "azure-crucible-kab-dev-cmdb"
|
||||
text = (case / "context.yaml").read_text() + "\n\n---\n\n" + pd.read_csv(case / "cmdb-export.csv").to_string()
|
||||
assert inventory.context_section(text)["scope"]["regions"] == ["eastus", "global"]
|
||||
|
||||
|
||||
def test_the_shared_reader_is_the_planners():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
planner = root.parent / "cp1-discovery-planner-agent" / "app" / "run_context.py"
|
||||
assert (root / "app" / "run_context.py").read_text() == planner.read_text()
|
||||
Reference in New Issue
Block a user