Files
Jonathan Boniface 66c32745aa
Some checks failed
check agents / check (push) Failing after 14s
F02: Assessment report and package (ADR 0014, 0015); register agents in KAB
- Report (cp1-assessment-report-agent 0.2.0, step s7): a model writes the
  Assessment report from facts worked out in code (register, gaps, scorecard,
  rule traces, checks, and what each application is from the Discovery
  package); code sets the verdict and renders the pages. The
  disposition_signoff review point moves from the critic to the report.
- Package (cp1-assessment-package-agent 0.1.1, step s8): after sign-off, every
  step's output in one package, with a register, a gaps register, the
  sign-off record, lineage and a sha256 per part, in code. Page step s9.
- scripts/kab_register_agents.py: registers deployed agents in KAB's agent
  catalog (an in-place workflow update doesn't).
- Runs exec_1b1257380df7 (014) and exec_374d5d577077 (015) completed on
  crucible-kab-dev: critic pass, report figures all in its facts, package
  complete with every digest recomputing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 20:19:10 +00:00

305 lines
18 KiB
Python

"""The Assessment package (F02 ADR 0014): every Assessment step's output in one document, for Strategy and the later
Features, with a manifest they can check in code.
The step runs only after the Engagement Lead continued the run at `disposition_signoff`, so a package exists only for
an assessment a person let through, and continuing signs off the queue as it stands. Parts are named by meaning, not
step id (KAB renames step ids on import), and copied unchanged so their evidence references still resolve. Two parts
are worked out here, never by a model: `register` (one row per application) and `gaps` (what Discovery data would
raise each application's confidence). Each part's `sha256` is over its canonical JSON, exactly as Discovery's package
step computes it (`cp1-discovery-package-agent/app/package.py`). Standard library only.
"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
PACKAGE_VERSION = "1.0"
SIGNOFF_BELOW = 0.80 # ALZ-ASM-04: signoff_required = confidence < 0.80
# The step's input name -> (part, the producing agent's own field name). scripts/build_workflow.py wires each input
# from the step that produces it (PACKAGE_INPUTS there); tests/test_package.py checks the two agree.
INPUTS = {
# s0, the intake: the Discovery package as it verified it.
"intake_verified": ("discovery", "verified"), "discovery_run_id": ("discovery", "discovery_run_id"),
"discovery_package_version": ("discovery", "package_version"),
"discovery_critic_verdict": ("discovery", "critic_verdict"), "discovery_provider": ("discovery", "provider"),
"discovery_parts_verified": ("discovery", "parts_verified"), "discovery_absent_parts": ("discovery", "absent_parts"),
"discovery_source_gaps": ("discovery", "source_gaps"), "discovery_agent_versions": ("discovery", "agent_versions"),
# s1, the planner.
"plan_status": ("plan", "status"), "preconditions": ("plan", "preconditions"), "rate_card_id": ("plan", "rate_card_id"),
"in_cloud_estate": ("plan", "in_cloud_estate"), "application_source": ("plan", "application_source"),
"applications": ("plan", "applications"), "tasks": ("plan", "tasks"), "coverage_gaps": ("plan", "coverage_gaps"),
"plan_summary": ("plan", "summary"), "plan_note": ("plan", "findings_note"),
# s2, the readiness profiler.
"profiles": ("profiles", "profiles"), "profile_gaps": ("profiles", "gaps"), "profile_note": ("profiles", "findings_note"),
# s3, the risk scorer.
"scores": ("risk", "scores"), "high_risk_tail": ("risk", "high_risk_tail"), "risk_note": ("risk", "findings_note"),
# s4, security posture (always advisory).
"posture_status": ("security", "status"), "security_provider": ("security", "provider"),
"posture_score": ("security", "posture_score"), "controls": ("security", "controls"),
"security_findings": ("security", "findings"), "pillars": ("security", "pillars"),
"injection_flags": ("security", "injection_flags"), "security_note": ("security", "findings_note"),
# s5, dispositions and TCO.
"dispositions": ("dispositions", "dispositions"), "producer_signoff_queue": ("dispositions", "signoff_queue"),
"scoring_note": ("dispositions", "scoring_note"),
# s6, the critic.
"verdict": ("critic", "verdict"), "checks": ("critic", "checks"),
"rejected_dispositions": ("critic", "rejected_dispositions"), "signoff_queue": ("critic", "signoff_queue"),
"unverified_claims": ("critic", "unverified_claims"), "critic_model": ("critic", "critic_model"),
"executor_models": ("critic", "executor_models"), "critic_summary": ("critic", "summary"),
# s7, the report the Engagement Lead signed off from (its Markdown; the HTML is the page step's).
"report_markdown": ("report", "report_markdown"),
}
COPIED = ("discovery", "plan", "profiles", "risk", "security", "dispositions", "critic", "report")
PARTS = COPIED + ("register", "gaps")
REQUIRED = COPIED # security is required too: its `status` says `not_applicable` when there is no cloud estate
COUNTED = {"plan": "applications", "profiles": "profiles", "risk": "scores", "security": "findings",
"dispositions": "dispositions", "critic": "checks", "register": "applications", "gaps": "items"}
# Where the data that would fill a gap comes from: the Discovery agent and field, in the producers' own terms.
RISK_FACTOR_SOURCE = {
"eol": "support_status or end_of_support on the application's inventory records (ALZ-DSC-03)",
"fan_in": "a dependency map covering the application's assets (ALZ-DSC-06)",
"data": "a classification of every store above 'unclassified' (ALZ-DSC-05, needs schema access), or "
"compliance_scope on the assets (ALZ-DSC-03)",
"change": "change records for the application's CIs from the ITSM (ALZ-DSC-04 itsm_signals)",
}
SIGNAL_SOURCE = {
"platform_support": "support_status on the inventory records (ALZ-DSC-03)",
"hardware_coupling": "the asset class of each inventory record (ALZ-DSC-03)",
"data_constraints": "store classification (ALZ-DSC-05) or compliance_scope (ALZ-DSC-03)",
"dependency_exposure": "the dependency map (ALZ-DSC-06)",
"hosting_portability": "hosting on the inventory records (ALZ-DSC-03)",
"sizing": "runtime telemetry: none is collected by Discovery today",
"performance_baseline": "runtime telemetry: none is collected by Discovery today",
}
CONFIDENCE_SOURCE = {
"P1": "runtime telemetry for sizing: none is collected by Discovery today",
"P2": "the facts or signals named in the basis",
"P3": "the risk factors missing from the score (see the risk_factor items)",
"P4": "a like-for-like replacement for the appliance on the rate card",
"P5": "stronger inventory records (confidence of 0.80 or more, no conflicts)",
"P6": "evidence that the application is no longer used",
}
def digest(part: Any) -> str:
text = json.dumps(part, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
def _decode(value: Any) -> Any:
"""A step output KAB delivered as a JSON string, decoded; anything else as it came."""
for _ in range(3):
if not isinstance(value, str):
break
try:
value = json.loads(value)
except ValueError:
break
return value
def parts_from(payload: dict) -> Dict[str, Optional[dict]]:
"""The step's inputs, grouped into named parts. A part none of whose inputs arrived is absent (None)."""
parts: Dict[str, dict] = {}
for name, (part, field) in INPUTS.items():
if payload.get(name) is not None:
parts.setdefault(part, {})[field] = _decode(payload[name])
return {part: parts.get(part) for part in COPIED}
def _by_app(items: Any) -> Dict[str, dict]:
return {i["application_id"]: i for i in items or [] if isinstance(i, dict) and i.get("application_id")}
def _rate_card(document: Any) -> dict:
"""The rate card's id and sha256 (canonical JSON when it parses, else the text as given)."""
if not document:
return {"rate_card_id": None, "sha256": None}
parsed = _decode(document)
if isinstance(parsed, (dict, list)):
return {"rate_card_id": parsed.get("rate_card_id") if isinstance(parsed, dict) else None, "sha256": digest(parsed)}
return {"rate_card_id": None, "sha256": "sha256:" + hashlib.sha256(str(document).encode("utf-8")).hexdigest()}
def signoff_status(app: str, verdict: Optional[str], disposed: set, rejected: set, queue: set) -> str:
"""Each application's sign-off, as continuing at disposition_signoff left it (ADR 0014)."""
if verdict != "pass":
return "not_signed_off" # a fail verdict is never approved; the package says so and is incomplete
if app in rejected:
return "rejected"
if app not in disposed:
return "not_disposed"
return "signed_off" if app in queue else "not_required"
def register(parts: Dict[str, Optional[dict]]) -> dict:
"""One row per application: what Strategy, the Business Case, Target Architecture and Wave Planning read."""
plan, critic = parts["plan"] or {}, parts["critic"] or {}
profiles = _by_app((parts["profiles"] or {}).get("profiles"))
scores = _by_app((parts["risk"] or {}).get("scores"))
dispositions = _by_app((parts["dispositions"] or {}).get("dispositions"))
security = parts["security"] or {}
findings: Dict[str, List[dict]] = {}
for f in security.get("findings") or []:
if isinstance(f, dict):
findings.setdefault(f.get("application_id") or "", []).append(f)
rejected: Dict[str, List[str]] = {}
for r in critic.get("rejected_dispositions") or []:
if isinstance(r, dict):
rejected.setdefault(r.get("application_id"), []).append(r.get("check_id"))
verdict, queue = critic.get("verdict"), set(critic.get("signoff_queue") or [])
gaps = {g.get("application_id"): g.get("reason") for g in plan.get("coverage_gaps") or [] if isinstance(g, dict)}
rows = []
for a in plan.get("applications") or []:
app = a.get("application_id")
d, s, p = dispositions.get(app) or {}, scores.get(app) or {}, profiles.get(app) or {}
tco = d.get("tco_model") or {}
own = findings.get(app, [])
rows.append({
"application_id": app, "name": a.get("name"), "assets": len(a.get("assets") or []),
"coverage_gap": gaps.get(app),
"disposition": d.get("disposition"), "confidence": d.get("confidence"),
"interim_rule": d.get("interim_rule"), "signoff_required": d.get("signoff_required"),
"signoff": signoff_status(app, verdict, set(dispositions), set(rejected), queue),
"rejected_by": rejected.get(app, []),
"matched_rule": next((t.get("category") for t in d.get("rule_trace") or [] if t.get("matched")), None),
"risk_score": s.get("score"), "risk_partial": s.get("partial"),
"risk_missing_factors": s.get("missing_factors") or [], "high_risk": s.get("high_risk"),
"telemetry": p.get("telemetry"),
"signals_unavailable": [x.get("signal") for x in p.get("signals") or [] if x.get("available") is False],
"depends_on": len(p.get("depends_on") or []), "dependents": len(p.get("dependents") or []),
"currency": tco.get("currency"), "current_annual": tco.get("current_annual"),
"target_annual": tco.get("target_annual"), "annual_delta": tco.get("annual_delta"),
"tco_indicative": tco.get("indicative"), "tco_missing_inputs": len(tco.get("missing_inputs") or []),
"security_findings": len(own),
"top_finding": max(own, key=lambda f: f.get("rank_score") or 0).get("description") if own else None,
"evidence_ref": d.get("evidence_ref"),
})
return {"applications": rows, "posture_status": security.get("status"), "posture_score": security.get("posture_score"),
"security_note": "Security findings are advisory (ALZ-ASM-05): they changed no disposition or confidence."}
def gaps(parts: Dict[str, Optional[dict]]) -> dict:
"""What Assessment couldn't establish, per application, with where the missing data would come from."""
items: List[dict] = []
plan = parts["plan"] or {}
for g in plan.get("coverage_gaps") or []:
if isinstance(g, dict):
items.append({"application_id": g.get("application_id"), "kind": "coverage", "item": "not assessed",
"effect": "no disposition", "reason": g.get("reason"), "fill_from": None})
for s in (parts["risk"] or {}).get("scores") or []:
for factor in s.get("missing_factors") or []:
items.append({"application_id": s.get("application_id"), "kind": "risk_factor", "item": factor,
"effect": "risk score is partial", "reason": next(
(f.get("basis") for f in s.get("factors") or [] if f.get("factor") == factor), None),
"fill_from": RISK_FACTOR_SOURCE.get(factor)})
for p in (parts["profiles"] or {}).get("profiles") or []:
for x in p.get("signals") or []:
if x.get("available") is False:
items.append({"application_id": p.get("application_id"), "kind": "readiness_signal",
"item": x.get("signal"), "effect": "signal unknown", "reason": x.get("basis"),
"fill_from": SIGNAL_SOURCE.get(x.get("signal"))})
for g in (parts["profiles"] or {}).get("gaps") or []:
if isinstance(g, dict):
items.append({"application_id": g.get("application_id"), "kind": "profile", "item": "no profile",
"effect": "readiness unknown", "reason": g.get("reason"), "fill_from": None})
for d in (parts["dispositions"] or {}).get("dispositions") or []:
app = d.get("application_id")
for adj in (d.get("confidence_workings") or {}).get("adjustments") or []:
rule = str(adj.get("rule") or "")
items.append({"application_id": app, "kind": "confidence", "item": rule,
"effect": f"confidence {adj.get('delta'):+.2f}" if isinstance(adj.get("delta"), (int, float))
else "confidence lowered", "reason": adj.get("basis"),
"fill_from": CONFIDENCE_SOURCE.get(rule.split(" ")[0])})
missing = (d.get("tco_model") or {}).get("missing_inputs") or []
if missing:
fields = sorted({m.split(" ", 1)[1] if " " in m else m for m in missing})
items.append({"application_id": app, "kind": "tco_input", "item": ", ".join(fields),
"effect": f"{len(missing)} TCO inputs missing; the model is indicative",
"reason": "; ".join(missing[:5]) + (" …" if len(missing) > 5 else ""),
"fill_from": "annual_cost and a cost breakdown on the inventory records (a billing or "
"cost-management source for Discovery)"})
source_gaps = (parts["discovery"] or {}).get("source_gaps") or []
return {"items": items, "discovery_source_gaps": source_gaps,
"note": "Each item is something Assessment had to do without. Filling it in Discovery and running both "
"workflows again is how confidence rises; nothing here is a defect of the run."}
def scorecard(parts: Dict[str, Optional[dict]], reg: dict, gap: dict) -> dict:
rows = reg["applications"]
disposed = [r for r in rows if r["disposition"]]
mix: Dict[str, int] = {}
for r in disposed:
mix[r["disposition"]] = mix.get(r["disposition"], 0) + 1
kinds: Dict[str, int] = {}
for i in gap["items"]:
kinds[i["kind"]] = kinds.get(i["kind"], 0) + 1
return {
"version": "1",
"applications": len(rows), "disposed": len(disposed), "dispositions": mix,
"below_threshold": sum(1 for r in disposed if isinstance(r["confidence"], (int, float))
and r["confidence"] < SIGNOFF_BELOW),
"signed_off": sum(1 for r in rows if r["signoff"] == "signed_off"),
"rejected": sum(1 for r in rows if r["signoff"] == "rejected"),
"risk_partial": sum(1 for r in rows if r["risk_partial"]), "high_risk": sum(1 for r in rows if r["high_risk"]),
"tco_with_target": sum(1 for r in disposed if r["target_annual"] is not None),
"tco_with_current": sum(1 for r in disposed if r["current_annual"] is not None),
"security_findings": len((parts["security"] or {}).get("findings") or []),
"gaps": kinds,
}
def build(payload: dict, execution_id: str = "", now: Optional[datetime] = None) -> dict:
parts: Dict[str, Any] = parts_from(payload)
reg = register(parts)
gap = gaps(parts)
parts["register"], parts["gaps"] = reg, gap
critic = parts["critic"] or {}
verdict = critic.get("verdict")
problems = [f"part missing: {p}" for p in REQUIRED if parts[p] is None]
if verdict != "pass":
problems.append(f"critic verdict is {verdict or 'absent'}, not pass")
if (parts["discovery"] or {}).get("verified") is not True:
problems.append("the intake did not verify the Discovery package")
signed = [r["application_id"] for r in reg["applications"] if r["signoff"] == "signed_off"]
manifest = {
"package_version": PACKAGE_VERSION,
"engagement_id": payload.get("engagement_id"), "run_id": payload.get("run_id"),
"execution_id": execution_id or None,
"created_at": (now or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ"),
"critic_verdict": verdict,
"discovery": {"execution_id": payload.get("discovery_execution_id"),
"run_id": (parts["discovery"] or {}).get("discovery_run_id"),
"artifact_ids": _decode(payload.get("artifact_ids")) or [],
"package_version": (parts["discovery"] or {}).get("package_version"),
"verified": (parts["discovery"] or {}).get("verified"),
"approval_ref": payload.get("discovery_approval_ref")},
"rate_card": {**_rate_card(payload.get("rate_card_document")),
**({"rate_card_id": (parts["plan"] or {}).get("rate_card_id")}
if (parts["plan"] or {}).get("rate_card_id") else {})},
"models": {"critic_model": critic.get("critic_model"), "executor_models": critic.get("executor_models")},
"agent_versions": payload.get("agent_versions") or {},
"signoff": {
"gate": "disposition_signoff",
"decision": "continued" if verdict == "pass" else "continued with a fail verdict: not signed off",
"signed_off": signed if verdict == "pass" else [],
"basis": "This step runs only after the Engagement Lead continued the run at disposition_signoff. "
"Continuing signs off the sign-off queue as it stands (ADR 0014); the next workflow records "
"that approval as its run input assessment_approval_ref.",
},
"parts": {p: {"sha256": digest(parts[p]),
**({"items": len((parts[p] or {}).get(COUNTED[p]) or [])} if p in COUNTED else {})}
for p in PARTS if parts[p] is not None},
"absent_parts": [p for p in PARTS if parts[p] is None],
"complete": not problems,
"problems": problems,
"scorecard": scorecard(parts, reg, gap),
}
return {"engagement_id": payload.get("engagement_id"), "run_id": payload.get("run_id"),
"package_version": PACKAGE_VERSION, "manifest": manifest,
**{p: parts[p] for p in PARTS if parts[p] is not None}}