Some checks failed
check agents / check (push) Failing after 14s
- Report (cp1-assessment-report-agent 0.2.0, step s7): a model writes the Assessment report from facts worked out in code (register, gaps, scorecard, rule traces, checks, and what each application is from the Discovery package); code sets the verdict and renders the pages. The disposition_signoff review point moves from the critic to the report. - Package (cp1-assessment-package-agent 0.1.1, step s8): after sign-off, every step's output in one package, with a register, a gaps register, the sign-off record, lineage and a sha256 per part, in code. Page step s9. - scripts/kab_register_agents.py: registers deployed agents in KAB's agent catalog (an in-place workflow update doesn't). - Runs exec_1b1257380df7 (014) and exec_374d5d577077 (015) completed on crucible-kab-dev: critic pass, report figures all in its facts, package complete with every digest recomputing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
156 lines
7.1 KiB
Python
156 lines
7.1 KiB
Python
"""The Assessment package (F02 ADR 0014), on the step outputs of run exec_49e355141382, the first passing run."""
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jsonschema
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FEATURE = ROOT.parent
|
|
sys.path.insert(0, str(ROOT))
|
|
from app import package # noqa: E402
|
|
|
|
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
|
|
INPUT = json.loads((ROOT / "schemas" / "input.json").read_text())
|
|
RUN = json.loads((ROOT / "tests" / "fixtures" / "run-exec_49e355141382.json").read_text())
|
|
|
|
|
|
def _module(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
WF = _module("build_workflow", FEATURE / "scripts" / "build_workflow.py")
|
|
def _discovery_digest():
|
|
"""Discovery's own digest function, from its source (its module imports its own `app` package)."""
|
|
import hashlib
|
|
import re
|
|
src = (FEATURE.parent / "ALZ-F01: Discovery Workflow Agent" / "cp1-discovery-package-agent" / "app" /
|
|
"package.py").read_text()
|
|
scope = {"hashlib": hashlib, "json": json, "Any": object}
|
|
exec(re.search(r"^def digest\(.*?(?=^\S)", src, re.M | re.S).group(0), scope)
|
|
return scope["digest"]
|
|
|
|
|
|
DISCOVERY_DIGEST = _discovery_digest()
|
|
|
|
|
|
def payload(**changes):
|
|
"""The step's request as the workflow wires it, from the run's real step outputs."""
|
|
steps = RUN["steps"]
|
|
steps = {**steps, WF.REPORT_STEP: {"report_markdown": "# Assessment report\n\nSigned off from this."}}
|
|
p = {name: steps[src.split(".")[0]].get(src.split(".")[1]) for name, src in WF.PACKAGE_INPUTS.items()}
|
|
p.update({"engagement_id": "ENG-AZURE-KAB-DEV-002", "run_id": "RUN-ASM-AZURE-012",
|
|
"discovery_execution_id": "exec_58ba6a71d00b",
|
|
"discovery_approval_ref": "Engagement Lead continued Discovery exec_58ba6a71d00b",
|
|
"artifact_ids": ["6aecad166bc9323ca8d4be23"],
|
|
"rate_card_document": json.dumps({"rate_card_id": "RC-AZURE-DEV-2026", "items": []}),
|
|
"agent_versions": {"cp1-assessment-critic-agent": "0.3.1"}})
|
|
p.update(changes)
|
|
return p
|
|
|
|
|
|
def row(pkg, app="APP-001"):
|
|
return next(r for r in pkg["register"]["applications"] if r["application_id"] == app)
|
|
|
|
|
|
def test_the_report_is_a_part():
|
|
pkg = package.build(payload())
|
|
assert pkg["report"] == {"report_markdown": "# Assessment report\n\nSigned off from this."}
|
|
assert "part missing: report" in package.build(payload(report_markdown=None))["manifest"]["problems"]
|
|
|
|
|
|
def test_the_wiring_and_the_inputs_table_agree():
|
|
assert set(WF.PACKAGE_INPUTS) == set(package.INPUTS)
|
|
for name, src in WF.PACKAGE_INPUTS.items():
|
|
step, field = src.split(".")
|
|
folder = dict((s, f) for s, f, _, _ in WF.STEPS)[step]
|
|
produced = json.loads((FEATURE / folder / "schemas" / "output.json").read_text())["properties"]
|
|
assert field in produced and field == package.INPUTS[name][1], (name, src)
|
|
assert name in INPUT["properties"]
|
|
|
|
|
|
def test_run_012_makes_a_complete_package():
|
|
pkg = package.build(payload(), "exec_49e355141382")
|
|
jsonschema.validate(pkg, OUTPUT)
|
|
m = pkg["manifest"]
|
|
assert m["complete"] and m["problems"] == [] and m["critic_verdict"] == "pass"
|
|
assert set(m["parts"]) == set(package.PARTS) and m["absent_parts"] == []
|
|
assert m["discovery"]["execution_id"] == "exec_58ba6a71d00b" and m["discovery"]["verified"] is True
|
|
assert m["discovery"]["artifact_ids"] == ["6aecad166bc9323ca8d4be23"]
|
|
assert m["discovery"]["run_id"] == "RUN-20261105T161838Z-a75b46"
|
|
assert m["models"] == {"critic_model": "gpt-5.4", "executor_models": ["gpt-5.6-sol"]}
|
|
assert m["signoff"]["signed_off"] == ["APP-001"] and m["signoff"]["decision"] == "continued"
|
|
|
|
|
|
def test_the_register_row_for_app_001():
|
|
r = row(package.build(payload()))
|
|
assert (r["disposition"], r["confidence"], r["signoff"], r["matched_rule"]) == ("rehost", 0.5, "signed_off", "rehost")
|
|
assert r["risk_score"] == 0.0 and r["risk_partial"] and r["risk_missing_factors"] == ["eol", "data", "change"]
|
|
assert r["dependents"] == 0 and r["target_annual"] == 11622 and r["current_annual"] is None and r["tco_indicative"]
|
|
assert r["security_findings"] == 5 and r["top_finding"] and r["assets"] == 10
|
|
assert r["evidence_ref"] == "discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-001"
|
|
|
|
|
|
def test_the_gaps_say_what_would_raise_confidence():
|
|
g = package.build(payload())["gaps"]
|
|
items = {(i["kind"], i["item"].split(" ")[0]) for i in g["items"]}
|
|
assert {("risk_factor", "eol"), ("risk_factor", "data"), ("risk_factor", "change")} <= items
|
|
assert {("confidence", "P1"), ("confidence", "P2"), ("confidence", "P3")} <= items
|
|
assert ("tco_input", "facts.annual_cost") in items and ("readiness_signal", "platform_support") in items
|
|
assert all(i["fill_from"] for i in g["items"] if i["kind"] in ("risk_factor", "confidence", "tco_input"))
|
|
p2 = next(i for i in g["items"] if i["item"].startswith("P2"))
|
|
assert p2["effect"] == "confidence -0.20"
|
|
assert g["discovery_source_gaps"] == ["itsm", "netflow", "schema"]
|
|
|
|
|
|
def test_the_digest_is_discoverys_and_recomputes():
|
|
pkg = package.build(payload())
|
|
for part, entry in pkg["manifest"]["parts"].items():
|
|
assert entry["sha256"] == package.digest(pkg[part]) == DISCOVERY_DIGEST(pkg[part])
|
|
|
|
|
|
def test_a_fail_verdict_is_never_signed_off():
|
|
pkg = package.build(payload(verdict="fail"))
|
|
assert not pkg["manifest"]["complete"] and "critic verdict is fail, not pass" in pkg["manifest"]["problems"]
|
|
assert row(pkg)["signoff"] == "not_signed_off" and pkg["manifest"]["signoff"]["signed_off"] == []
|
|
|
|
|
|
def test_a_rejected_disposition_is_marked():
|
|
rejected = [{"application_id": "APP-001", "check_id": "K-02", "reason": "x", "named_dependent": None}]
|
|
r = row(package.build(payload(rejected_dispositions=rejected)))
|
|
assert r["signoff"] == "rejected" and r["rejected_by"] == ["K-02"]
|
|
|
|
|
|
def test_an_application_below_the_threshold_but_not_in_the_queue_is_not_required():
|
|
r = row(package.build(payload(signoff_queue=[])))
|
|
assert r["signoff"] == "not_required"
|
|
|
|
|
|
def test_a_missing_part_makes_it_incomplete():
|
|
pkg = package.build(payload(scores=None, high_risk_tail=None, risk_note=None))
|
|
assert "risk" in pkg["manifest"]["absent_parts"] and "part missing: risk" in pkg["manifest"]["problems"]
|
|
assert row(pkg)["risk_score"] is None
|
|
jsonschema.validate(pkg, OUTPUT)
|
|
|
|
|
|
def test_outputs_kab_delivers_as_json_strings_are_decoded():
|
|
p = payload()
|
|
p["dispositions"] = json.dumps(p["dispositions"])
|
|
assert row(package.build(p))["disposition"] == "rehost"
|
|
|
|
|
|
def test_an_unverified_discovery_package_is_incomplete():
|
|
pkg = package.build(payload(intake_verified=False))
|
|
assert "the intake did not verify the Discovery package" in pkg["manifest"]["problems"]
|
|
|
|
|
|
def test_the_rate_card_is_identified_not_copied():
|
|
m = package.build(payload())["manifest"]
|
|
assert m["rate_card"]["rate_card_id"] and m["rate_card"]["sha256"].startswith("sha256:")
|
|
assert "rate_card_document" not in json.dumps(package.build(payload())["plan"])
|