Some checks failed
check agents / check (push) Failing after 14s
- Report (cp1-assessment-report-agent 0.2.0, step s7): a model writes the Assessment report from facts worked out in code (register, gaps, scorecard, rule traces, checks, and what each application is from the Discovery package); code sets the verdict and renders the pages. The disposition_signoff review point moves from the critic to the report. - Package (cp1-assessment-package-agent 0.1.1, step s8): after sign-off, every step's output in one package, with a register, a gaps register, the sign-off record, lineage and a sha256 per part, in code. Page step s9. - scripts/kab_register_agents.py: registers deployed agents in KAB's agent catalog (an in-place workflow update doesn't). - Runs exec_1b1257380df7 (014) and exec_374d5d577077 (015) completed on crucible-kab-dev: critic pass, report figures all in its facts, package complete with every digest recomputing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
305 lines
18 KiB
Python
305 lines
18 KiB
Python
"""The Assessment package (F02 ADR 0014): every Assessment step's output in one document, for Strategy and the later
|
|
Features, with a manifest they can check in code.
|
|
|
|
The step runs only after the Engagement Lead continued the run at `disposition_signoff`, so a package exists only for
|
|
an assessment a person let through, and continuing signs off the queue as it stands. Parts are named by meaning, not
|
|
step id (KAB renames step ids on import), and copied unchanged so their evidence references still resolve. Two parts
|
|
are worked out here, never by a model: `register` (one row per application) and `gaps` (what Discovery data would
|
|
raise each application's confidence). Each part's `sha256` is over its canonical JSON, exactly as Discovery's package
|
|
step computes it (`cp1-discovery-package-agent/app/package.py`). Standard library only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
PACKAGE_VERSION = "1.0"
|
|
SIGNOFF_BELOW = 0.80 # ALZ-ASM-04: signoff_required = confidence < 0.80
|
|
|
|
# The step's input name -> (part, the producing agent's own field name). scripts/build_workflow.py wires each input
|
|
# from the step that produces it (PACKAGE_INPUTS there); tests/test_package.py checks the two agree.
|
|
INPUTS = {
|
|
# s0, the intake: the Discovery package as it verified it.
|
|
"intake_verified": ("discovery", "verified"), "discovery_run_id": ("discovery", "discovery_run_id"),
|
|
"discovery_package_version": ("discovery", "package_version"),
|
|
"discovery_critic_verdict": ("discovery", "critic_verdict"), "discovery_provider": ("discovery", "provider"),
|
|
"discovery_parts_verified": ("discovery", "parts_verified"), "discovery_absent_parts": ("discovery", "absent_parts"),
|
|
"discovery_source_gaps": ("discovery", "source_gaps"), "discovery_agent_versions": ("discovery", "agent_versions"),
|
|
# s1, the planner.
|
|
"plan_status": ("plan", "status"), "preconditions": ("plan", "preconditions"), "rate_card_id": ("plan", "rate_card_id"),
|
|
"in_cloud_estate": ("plan", "in_cloud_estate"), "application_source": ("plan", "application_source"),
|
|
"applications": ("plan", "applications"), "tasks": ("plan", "tasks"), "coverage_gaps": ("plan", "coverage_gaps"),
|
|
"plan_summary": ("plan", "summary"), "plan_note": ("plan", "findings_note"),
|
|
# s2, the readiness profiler.
|
|
"profiles": ("profiles", "profiles"), "profile_gaps": ("profiles", "gaps"), "profile_note": ("profiles", "findings_note"),
|
|
# s3, the risk scorer.
|
|
"scores": ("risk", "scores"), "high_risk_tail": ("risk", "high_risk_tail"), "risk_note": ("risk", "findings_note"),
|
|
# s4, security posture (always advisory).
|
|
"posture_status": ("security", "status"), "security_provider": ("security", "provider"),
|
|
"posture_score": ("security", "posture_score"), "controls": ("security", "controls"),
|
|
"security_findings": ("security", "findings"), "pillars": ("security", "pillars"),
|
|
"injection_flags": ("security", "injection_flags"), "security_note": ("security", "findings_note"),
|
|
# s5, dispositions and TCO.
|
|
"dispositions": ("dispositions", "dispositions"), "producer_signoff_queue": ("dispositions", "signoff_queue"),
|
|
"scoring_note": ("dispositions", "scoring_note"),
|
|
# s6, the critic.
|
|
"verdict": ("critic", "verdict"), "checks": ("critic", "checks"),
|
|
"rejected_dispositions": ("critic", "rejected_dispositions"), "signoff_queue": ("critic", "signoff_queue"),
|
|
"unverified_claims": ("critic", "unverified_claims"), "critic_model": ("critic", "critic_model"),
|
|
"executor_models": ("critic", "executor_models"), "critic_summary": ("critic", "summary"),
|
|
# s7, the report the Engagement Lead signed off from (its Markdown; the HTML is the page step's).
|
|
"report_markdown": ("report", "report_markdown"),
|
|
}
|
|
COPIED = ("discovery", "plan", "profiles", "risk", "security", "dispositions", "critic", "report")
|
|
PARTS = COPIED + ("register", "gaps")
|
|
REQUIRED = COPIED # security is required too: its `status` says `not_applicable` when there is no cloud estate
|
|
COUNTED = {"plan": "applications", "profiles": "profiles", "risk": "scores", "security": "findings",
|
|
"dispositions": "dispositions", "critic": "checks", "register": "applications", "gaps": "items"}
|
|
|
|
# Where the data that would fill a gap comes from: the Discovery agent and field, in the producers' own terms.
|
|
RISK_FACTOR_SOURCE = {
|
|
"eol": "support_status or end_of_support on the application's inventory records (ALZ-DSC-03)",
|
|
"fan_in": "a dependency map covering the application's assets (ALZ-DSC-06)",
|
|
"data": "a classification of every store above 'unclassified' (ALZ-DSC-05, needs schema access), or "
|
|
"compliance_scope on the assets (ALZ-DSC-03)",
|
|
"change": "change records for the application's CIs from the ITSM (ALZ-DSC-04 itsm_signals)",
|
|
}
|
|
SIGNAL_SOURCE = {
|
|
"platform_support": "support_status on the inventory records (ALZ-DSC-03)",
|
|
"hardware_coupling": "the asset class of each inventory record (ALZ-DSC-03)",
|
|
"data_constraints": "store classification (ALZ-DSC-05) or compliance_scope (ALZ-DSC-03)",
|
|
"dependency_exposure": "the dependency map (ALZ-DSC-06)",
|
|
"hosting_portability": "hosting on the inventory records (ALZ-DSC-03)",
|
|
"sizing": "runtime telemetry: none is collected by Discovery today",
|
|
"performance_baseline": "runtime telemetry: none is collected by Discovery today",
|
|
}
|
|
CONFIDENCE_SOURCE = {
|
|
"P1": "runtime telemetry for sizing: none is collected by Discovery today",
|
|
"P2": "the facts or signals named in the basis",
|
|
"P3": "the risk factors missing from the score (see the risk_factor items)",
|
|
"P4": "a like-for-like replacement for the appliance on the rate card",
|
|
"P5": "stronger inventory records (confidence of 0.80 or more, no conflicts)",
|
|
"P6": "evidence that the application is no longer used",
|
|
}
|
|
|
|
|
|
def digest(part: Any) -> str:
|
|
text = json.dumps(part, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _decode(value: Any) -> Any:
|
|
"""A step output KAB delivered as a JSON string, decoded; anything else as it came."""
|
|
for _ in range(3):
|
|
if not isinstance(value, str):
|
|
break
|
|
try:
|
|
value = json.loads(value)
|
|
except ValueError:
|
|
break
|
|
return value
|
|
|
|
|
|
def parts_from(payload: dict) -> Dict[str, Optional[dict]]:
|
|
"""The step's inputs, grouped into named parts. A part none of whose inputs arrived is absent (None)."""
|
|
parts: Dict[str, dict] = {}
|
|
for name, (part, field) in INPUTS.items():
|
|
if payload.get(name) is not None:
|
|
parts.setdefault(part, {})[field] = _decode(payload[name])
|
|
return {part: parts.get(part) for part in COPIED}
|
|
|
|
|
|
def _by_app(items: Any) -> Dict[str, dict]:
|
|
return {i["application_id"]: i for i in items or [] if isinstance(i, dict) and i.get("application_id")}
|
|
|
|
|
|
def _rate_card(document: Any) -> dict:
|
|
"""The rate card's id and sha256 (canonical JSON when it parses, else the text as given)."""
|
|
if not document:
|
|
return {"rate_card_id": None, "sha256": None}
|
|
parsed = _decode(document)
|
|
if isinstance(parsed, (dict, list)):
|
|
return {"rate_card_id": parsed.get("rate_card_id") if isinstance(parsed, dict) else None, "sha256": digest(parsed)}
|
|
return {"rate_card_id": None, "sha256": "sha256:" + hashlib.sha256(str(document).encode("utf-8")).hexdigest()}
|
|
|
|
|
|
def signoff_status(app: str, verdict: Optional[str], disposed: set, rejected: set, queue: set) -> str:
|
|
"""Each application's sign-off, as continuing at disposition_signoff left it (ADR 0014)."""
|
|
if verdict != "pass":
|
|
return "not_signed_off" # a fail verdict is never approved; the package says so and is incomplete
|
|
if app in rejected:
|
|
return "rejected"
|
|
if app not in disposed:
|
|
return "not_disposed"
|
|
return "signed_off" if app in queue else "not_required"
|
|
|
|
|
|
def register(parts: Dict[str, Optional[dict]]) -> dict:
|
|
"""One row per application: what Strategy, the Business Case, Target Architecture and Wave Planning read."""
|
|
plan, critic = parts["plan"] or {}, parts["critic"] or {}
|
|
profiles = _by_app((parts["profiles"] or {}).get("profiles"))
|
|
scores = _by_app((parts["risk"] or {}).get("scores"))
|
|
dispositions = _by_app((parts["dispositions"] or {}).get("dispositions"))
|
|
security = parts["security"] or {}
|
|
findings: Dict[str, List[dict]] = {}
|
|
for f in security.get("findings") or []:
|
|
if isinstance(f, dict):
|
|
findings.setdefault(f.get("application_id") or "", []).append(f)
|
|
rejected: Dict[str, List[str]] = {}
|
|
for r in critic.get("rejected_dispositions") or []:
|
|
if isinstance(r, dict):
|
|
rejected.setdefault(r.get("application_id"), []).append(r.get("check_id"))
|
|
verdict, queue = critic.get("verdict"), set(critic.get("signoff_queue") or [])
|
|
gaps = {g.get("application_id"): g.get("reason") for g in plan.get("coverage_gaps") or [] if isinstance(g, dict)}
|
|
rows = []
|
|
for a in plan.get("applications") or []:
|
|
app = a.get("application_id")
|
|
d, s, p = dispositions.get(app) or {}, scores.get(app) or {}, profiles.get(app) or {}
|
|
tco = d.get("tco_model") or {}
|
|
own = findings.get(app, [])
|
|
rows.append({
|
|
"application_id": app, "name": a.get("name"), "assets": len(a.get("assets") or []),
|
|
"coverage_gap": gaps.get(app),
|
|
"disposition": d.get("disposition"), "confidence": d.get("confidence"),
|
|
"interim_rule": d.get("interim_rule"), "signoff_required": d.get("signoff_required"),
|
|
"signoff": signoff_status(app, verdict, set(dispositions), set(rejected), queue),
|
|
"rejected_by": rejected.get(app, []),
|
|
"matched_rule": next((t.get("category") for t in d.get("rule_trace") or [] if t.get("matched")), None),
|
|
"risk_score": s.get("score"), "risk_partial": s.get("partial"),
|
|
"risk_missing_factors": s.get("missing_factors") or [], "high_risk": s.get("high_risk"),
|
|
"telemetry": p.get("telemetry"),
|
|
"signals_unavailable": [x.get("signal") for x in p.get("signals") or [] if x.get("available") is False],
|
|
"depends_on": len(p.get("depends_on") or []), "dependents": len(p.get("dependents") or []),
|
|
"currency": tco.get("currency"), "current_annual": tco.get("current_annual"),
|
|
"target_annual": tco.get("target_annual"), "annual_delta": tco.get("annual_delta"),
|
|
"tco_indicative": tco.get("indicative"), "tco_missing_inputs": len(tco.get("missing_inputs") or []),
|
|
"security_findings": len(own),
|
|
"top_finding": max(own, key=lambda f: f.get("rank_score") or 0).get("description") if own else None,
|
|
"evidence_ref": d.get("evidence_ref"),
|
|
})
|
|
return {"applications": rows, "posture_status": security.get("status"), "posture_score": security.get("posture_score"),
|
|
"security_note": "Security findings are advisory (ALZ-ASM-05): they changed no disposition or confidence."}
|
|
|
|
|
|
def gaps(parts: Dict[str, Optional[dict]]) -> dict:
|
|
"""What Assessment couldn't establish, per application, with where the missing data would come from."""
|
|
items: List[dict] = []
|
|
plan = parts["plan"] or {}
|
|
for g in plan.get("coverage_gaps") or []:
|
|
if isinstance(g, dict):
|
|
items.append({"application_id": g.get("application_id"), "kind": "coverage", "item": "not assessed",
|
|
"effect": "no disposition", "reason": g.get("reason"), "fill_from": None})
|
|
for s in (parts["risk"] or {}).get("scores") or []:
|
|
for factor in s.get("missing_factors") or []:
|
|
items.append({"application_id": s.get("application_id"), "kind": "risk_factor", "item": factor,
|
|
"effect": "risk score is partial", "reason": next(
|
|
(f.get("basis") for f in s.get("factors") or [] if f.get("factor") == factor), None),
|
|
"fill_from": RISK_FACTOR_SOURCE.get(factor)})
|
|
for p in (parts["profiles"] or {}).get("profiles") or []:
|
|
for x in p.get("signals") or []:
|
|
if x.get("available") is False:
|
|
items.append({"application_id": p.get("application_id"), "kind": "readiness_signal",
|
|
"item": x.get("signal"), "effect": "signal unknown", "reason": x.get("basis"),
|
|
"fill_from": SIGNAL_SOURCE.get(x.get("signal"))})
|
|
for g in (parts["profiles"] or {}).get("gaps") or []:
|
|
if isinstance(g, dict):
|
|
items.append({"application_id": g.get("application_id"), "kind": "profile", "item": "no profile",
|
|
"effect": "readiness unknown", "reason": g.get("reason"), "fill_from": None})
|
|
for d in (parts["dispositions"] or {}).get("dispositions") or []:
|
|
app = d.get("application_id")
|
|
for adj in (d.get("confidence_workings") or {}).get("adjustments") or []:
|
|
rule = str(adj.get("rule") or "")
|
|
items.append({"application_id": app, "kind": "confidence", "item": rule,
|
|
"effect": f"confidence {adj.get('delta'):+.2f}" if isinstance(adj.get("delta"), (int, float))
|
|
else "confidence lowered", "reason": adj.get("basis"),
|
|
"fill_from": CONFIDENCE_SOURCE.get(rule.split(" ")[0])})
|
|
missing = (d.get("tco_model") or {}).get("missing_inputs") or []
|
|
if missing:
|
|
fields = sorted({m.split(" ", 1)[1] if " " in m else m for m in missing})
|
|
items.append({"application_id": app, "kind": "tco_input", "item": ", ".join(fields),
|
|
"effect": f"{len(missing)} TCO inputs missing; the model is indicative",
|
|
"reason": "; ".join(missing[:5]) + (" …" if len(missing) > 5 else ""),
|
|
"fill_from": "annual_cost and a cost breakdown on the inventory records (a billing or "
|
|
"cost-management source for Discovery)"})
|
|
source_gaps = (parts["discovery"] or {}).get("source_gaps") or []
|
|
return {"items": items, "discovery_source_gaps": source_gaps,
|
|
"note": "Each item is something Assessment had to do without. Filling it in Discovery and running both "
|
|
"workflows again is how confidence rises; nothing here is a defect of the run."}
|
|
|
|
|
|
def scorecard(parts: Dict[str, Optional[dict]], reg: dict, gap: dict) -> dict:
|
|
rows = reg["applications"]
|
|
disposed = [r for r in rows if r["disposition"]]
|
|
mix: Dict[str, int] = {}
|
|
for r in disposed:
|
|
mix[r["disposition"]] = mix.get(r["disposition"], 0) + 1
|
|
kinds: Dict[str, int] = {}
|
|
for i in gap["items"]:
|
|
kinds[i["kind"]] = kinds.get(i["kind"], 0) + 1
|
|
return {
|
|
"version": "1",
|
|
"applications": len(rows), "disposed": len(disposed), "dispositions": mix,
|
|
"below_threshold": sum(1 for r in disposed if isinstance(r["confidence"], (int, float))
|
|
and r["confidence"] < SIGNOFF_BELOW),
|
|
"signed_off": sum(1 for r in rows if r["signoff"] == "signed_off"),
|
|
"rejected": sum(1 for r in rows if r["signoff"] == "rejected"),
|
|
"risk_partial": sum(1 for r in rows if r["risk_partial"]), "high_risk": sum(1 for r in rows if r["high_risk"]),
|
|
"tco_with_target": sum(1 for r in disposed if r["target_annual"] is not None),
|
|
"tco_with_current": sum(1 for r in disposed if r["current_annual"] is not None),
|
|
"security_findings": len((parts["security"] or {}).get("findings") or []),
|
|
"gaps": kinds,
|
|
}
|
|
|
|
|
|
def build(payload: dict, execution_id: str = "", now: Optional[datetime] = None) -> dict:
|
|
parts: Dict[str, Any] = parts_from(payload)
|
|
reg = register(parts)
|
|
gap = gaps(parts)
|
|
parts["register"], parts["gaps"] = reg, gap
|
|
critic = parts["critic"] or {}
|
|
verdict = critic.get("verdict")
|
|
problems = [f"part missing: {p}" for p in REQUIRED if parts[p] is None]
|
|
if verdict != "pass":
|
|
problems.append(f"critic verdict is {verdict or 'absent'}, not pass")
|
|
if (parts["discovery"] or {}).get("verified") is not True:
|
|
problems.append("the intake did not verify the Discovery package")
|
|
signed = [r["application_id"] for r in reg["applications"] if r["signoff"] == "signed_off"]
|
|
manifest = {
|
|
"package_version": PACKAGE_VERSION,
|
|
"engagement_id": payload.get("engagement_id"), "run_id": payload.get("run_id"),
|
|
"execution_id": execution_id or None,
|
|
"created_at": (now or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"critic_verdict": verdict,
|
|
"discovery": {"execution_id": payload.get("discovery_execution_id"),
|
|
"run_id": (parts["discovery"] or {}).get("discovery_run_id"),
|
|
"artifact_ids": _decode(payload.get("artifact_ids")) or [],
|
|
"package_version": (parts["discovery"] or {}).get("package_version"),
|
|
"verified": (parts["discovery"] or {}).get("verified"),
|
|
"approval_ref": payload.get("discovery_approval_ref")},
|
|
"rate_card": {**_rate_card(payload.get("rate_card_document")),
|
|
**({"rate_card_id": (parts["plan"] or {}).get("rate_card_id")}
|
|
if (parts["plan"] or {}).get("rate_card_id") else {})},
|
|
"models": {"critic_model": critic.get("critic_model"), "executor_models": critic.get("executor_models")},
|
|
"agent_versions": payload.get("agent_versions") or {},
|
|
"signoff": {
|
|
"gate": "disposition_signoff",
|
|
"decision": "continued" if verdict == "pass" else "continued with a fail verdict: not signed off",
|
|
"signed_off": signed if verdict == "pass" else [],
|
|
"basis": "This step runs only after the Engagement Lead continued the run at disposition_signoff. "
|
|
"Continuing signs off the sign-off queue as it stands (ADR 0014); the next workflow records "
|
|
"that approval as its run input assessment_approval_ref.",
|
|
},
|
|
"parts": {p: {"sha256": digest(parts[p]),
|
|
**({"items": len((parts[p] or {}).get(COUNTED[p]) or [])} if p in COUNTED else {})}
|
|
for p in PARTS if parts[p] is not None},
|
|
"absent_parts": [p for p in PARTS if parts[p] is None],
|
|
"complete": not problems,
|
|
"problems": problems,
|
|
"scorecard": scorecard(parts, reg, gap),
|
|
}
|
|
return {"engagement_id": payload.get("engagement_id"), "run_id": payload.get("run_id"),
|
|
"package_version": PACKAGE_VERSION, "manifest": manifest,
|
|
**{p: parts[p] for p in PARTS if parts[p] is not None}}
|