ALZ-F01 Discovery: package step, value scorecard, application grouping, cases

- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent f659564208
commit 9e503885cc
211 changed files with 18170 additions and 158 deletions

View File

@@ -0,0 +1,20 @@
"""Answers from pre_process: the package is assembled by app/package.py, so no model is called (ADR 0020)."""
import json
from typing import Any, Optional
from kyndryl_agent_runtime.request_cycle import Reply
from app import package
def _execution_id() -> str:
try:
from kyndryl_agent_runtime.context import get_workflow_execution_id
run = get_workflow_execution_id()
except Exception: # an older runtime, or a request from outside a workflow
return ""
return "" if run in (None, "unknown") else str(run)
def pre_process(payload: dict) -> Optional[Any]:
return Reply(json.dumps(package.build(payload, _execution_id())))

View File

@@ -0,0 +1,99 @@
"""The Discovery package (ADR 0020): every step's output in one document, with a manifest F02 checks in code.
The step runs after the report's review point, so a package exists only for a run the Engagement Lead let through.
Parts are named by meaning, not step id (KAB renames step ids on import), and copied unchanged so their evidence
references still resolve. Each part's `sha256` is over its canonical JSON: `json.dumps(part, sort_keys=True,
separators=(",", ":"), ensure_ascii=False)` encoded as UTF-8. No model is involved.
"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timezone
from typing import Any
from app import scorecard
PACKAGE_VERSION = "1.0"
# The step's input name -> (part, the producing agent's own field name). scripts/build_workflow.py wires each input
# from the step that produces it; this table and that wiring are checked against each other by tests/test_package.py.
INPUTS = {
"plan_id": ("plan", "plan_id"), "plan_summary": ("plan", "summary"), "coverage_gaps": ("plan", "coverage_gaps"),
"source_gaps": ("plan", "source_gaps"), "scope_accounting": ("plan", "scope_accounting"),
"scan_findings": ("scan", "findings"),
"records": ("inventory", "records"), "sme_review_queue": ("inventory", "sme_review_queue"),
"drift_items": ("cmdb", "drift_items"), "itsm_signals": ("cmdb", "itsm_signals"), "cmdb_summary": ("cmdb", "summary"),
"cmdb_cis": ("cmdb", "cis"), "cmdb_relationships": ("cmdb", "relationships"),
"cmdb_dangling": ("cmdb", "dangling_relationships"), "cmdb_sources": ("cmdb", "cmdb_sources"),
"cmdb_injections": ("cmdb", "injections_found"),
"stores": ("classification", "stores"),
"ranges": ("network", "ranges"), "flows": ("network", "flows"), "network_findings": ("network", "findings"),
"topology_summary": ("network", "topology_summary"),
"nodes": ("dependencies", "nodes"), "edges": ("dependencies", "edges"), "orphans": ("dependencies", "orphans"),
"validation_queue": ("dependencies", "validation_queue"),
"folded_dependencies": ("dependencies", "folded_dependencies"), "dangling": ("dependencies", "dangling"),
"graph_comparison": ("dependencies", "graph_comparison"),
"applications": ("applications", "applications"), "asset_owners": ("applications", "asset_owners"),
"ungrouped_assets": ("applications", "ungrouped_assets"), "applications_note": ("applications", "applications_note"),
"verdict": ("critic", "verdict"), "critic_note": ("critic", "note"),
"coverage_complete": ("critic", "coverage_complete"), "unsupported_claims": ("critic", "unsupported_claims"),
"consistency_issues": ("critic", "consistency_issues"), "obeyed_injection": ("critic", "obeyed_injection"),
"not_checked": ("critic", "not_checked"),
"waf_status": ("well_architected", "status"), "waf_pillars": ("well_architected", "pillars"),
"waf_checks": ("well_architected", "checks"), "waf_recommendations": ("well_architected", "recommendations"),
"waf_note": ("well_architected", "note"),
"report_counts": ("report", "counts"), "report_markdown": ("report", "report_markdown"),
}
PARTS = ("plan", "scan", "inventory", "cmdb", "classification", "network", "dependencies", "applications", "critic",
"well_architected", "report")
# What F02 cannot start without. The CMDB, the applications (none on the sample estate) and the Well-Architected
# review are optional: a run without a CMDB says so in source_gaps, and F02's ASM-05 reads the provider itself.
REQUIRED = ("plan", "inventory", "classification", "network", "dependencies", "critic", "report")
# How many items each part holds, for the manifest.
COUNTED = {"scan": "findings", "inventory": "records", "cmdb": "drift_items", "classification": "stores",
"network": "findings", "dependencies": "edges", "applications": "applications", "well_architected": "checks"}
def digest(part: Any) -> str:
text = json.dumps(part, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
def parts_from(payload: dict) -> dict:
"""The step's inputs, grouped into named parts. A part none of whose inputs arrived is absent (None)."""
parts: dict[str, dict] = {}
for name, (part, field) in INPUTS.items():
if payload.get(name) is not None:
parts.setdefault(part, {})[field] = payload[name]
return {part: parts.get(part) for part in PARTS}
def build(payload: dict, execution_id: str = "", now: datetime | None = None) -> dict:
parts = parts_from(payload)
present = [p for p in PARTS if parts[p] is not None]
missing = [p for p in REQUIRED if parts[p] is None]
verdict = (parts["critic"] or {}).get("verdict")
problems = [f"part missing: {p}" for p in missing]
if verdict != "pass":
problems.append(f"critic verdict is {verdict or 'absent'}, not pass")
manifest = {
"package_version": PACKAGE_VERSION,
"engagement_id": payload.get("engagement_id"),
"run_id": payload.get("run_id"),
"execution_id": execution_id or None,
"provider": payload.get("provider"),
"created_at": (now or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ"),
"critic_verdict": verdict,
"reviewed": "The package step runs only after the Engagement Lead continued past the report's review point.",
"agent_versions": payload.get("agent_versions") or {},
"source_gaps": [g.get("source") for g in (parts["plan"] or {}).get("source_gaps") or [] if isinstance(g, dict)],
"parts": {p: {"sha256": digest(parts[p]),
**({"items": len((parts[p] or {}).get(COUNTED[p]) or [])} if p in COUNTED else {})}
for p in present},
"absent_parts": [p for p in PARTS if parts[p] is None],
"complete": not problems,
"problems": problems,
"scorecard": scorecard.score(parts),
}
return {"engagement_id": payload.get("engagement_id"), "run_id": payload.get("run_id"),
"package_version": PACKAGE_VERSION, "manifest": manifest, **{p: parts[p] for p in present}}

View File

@@ -0,0 +1,163 @@
"""The value scorecard (ADR 0020): how far a Discovery run answers what its two consumers ask, counted in code.
- The Assessment workflow (F02) reads these fields for every inventory record. Each cell is `filled` (a value),
`declared_gap` (no value, and the planner declared the source that would give it as missing), `missing` (no value
and no declared reason) or `not_applicable` (the field does not apply to this record, such as a data class on a
server).
- The Engagement Lead needs the decisions the run leaves for a person, and every finding with an owner and an action.
The same function serves the package step's manifest and `scripts/evaluate_workflow_run.py`, so the scorecard a run
ships with is the scorecard its case is judged by. No model is involved.
"""
from __future__ import annotations
from collections import Counter
from typing import Any
VERSION = "2" # 2: application and owner read from the dependency step's grouping (ADR 0021)
EMPTY = (None, "", "unknown", "Unknown", [], {})
# F02 field -> the planner's source_gaps sources whose absence explains a missing value. The planner's sources are
# cmdb, itsm, netflow and schema: there is no source yet for lifecycle, cost or compliance, so those can only be
# filled or missing until a source is added (ADR 0020, phase 3).
F02_FIELDS = {
"application": ("cmdb",),
"owner": ("cmdb",),
"criticality": ("cmdb",),
"environment": (),
"support_status": (),
"end_of_support": (),
"compliance_scope": (),
"annual_cost": (),
"data_class": ("schema",),
}
STATES = ("filled", "declared_gap", "missing", "not_applicable")
def _facts(record: dict) -> dict:
return record.get("facts") if isinstance(record.get("facts"), dict) else {}
def _value(field: str, record: dict, stores: dict, grouping: dict) -> Any:
facts = _facts(record)
rid = record.get("canonical_id")
if field == "application":
return grouping["application"].get(rid) or facts.get("business_service") or facts.get("application")
if field == "owner":
return grouping["owner"].get(rid) or facts.get("owner")
if field == "environment":
return record.get("environment")
if field == "data_class":
store = stores.get(record.get("canonical_id"))
if store is None:
return "not_applicable"
return None if store.get("highest_class") in ("unclassified", "unknown", None) else store["highest_class"]
return facts.get(field)
def _cell(field: str, record: dict, stores: dict, gaps: set, grouping: dict) -> str:
value = _value(field, record, stores, grouping)
if value == "not_applicable":
return "not_applicable"
if value not in EMPTY:
return "filled"
if field == "data_class" and (stores.get(record.get("canonical_id")) or {}).get("reason"):
return "declared_gap" # unclassified with the reason the classification step gave
return "declared_gap" if gaps & set(F02_FIELDS[field]) else "missing"
def _share(part: int, whole: int) -> float:
return round(part / whole, 3) if whole else 0.0
def _grouping(applications: dict | None) -> dict:
"""Each record's application and owner from the dependency step's grouping (ADR 0021)."""
applications = applications or {}
return {"application": {m: a.get("name") for a in applications.get("applications") or [] for m in a.get("members") or []},
"owner": {o.get("canonical_id"): o.get("owner") for o in applications.get("asset_owners") or []}}
def f02_fields(records: list, stores: list, source_gaps: list, applications: dict | None = None) -> dict:
by_id = {s.get("canonical_id"): s for s in stores or [] if isinstance(s, dict)}
grouping = _grouping(applications)
gaps = {g.get("source") for g in source_gaps or [] if isinstance(g, dict)}
per_field = {field: dict.fromkeys(STATES, 0) for field in F02_FIELDS}
for record in records or []:
for field in F02_FIELDS:
per_field[field][_cell(field, record, by_id, gaps, grouping)] += 1
totals = Counter()
for counts in per_field.values():
totals.update(counts)
applicable = sum(totals[s] for s in ("filled", "declared_gap", "missing"))
return {"records": len(records or []), "cells": applicable,
"filled": _share(totals["filled"], applicable), "declared_gap": _share(totals["declared_gap"], applicable),
"missing": _share(totals["missing"], applicable),
"filled_or_declared": _share(totals["filled"] + totals["declared_gap"], applicable),
"per_field": per_field}
def f02_run(parts: dict) -> dict:
"""What F02's ASM-01 checks before it starts."""
gaps = {g.get("source") for g in (parts.get("plan") or {}).get("source_gaps") or [] if isinstance(g, dict)}
inventory = parts.get("inventory") or {}
deps = parts.get("dependencies") or {}
critic = parts.get("critic") or {}
cmdb = parts.get("cmdb") or {}
checks = {
"inventory_present": bool(inventory.get("records")),
"dependency_map_present": deps.get("edges") is not None and deps.get("nodes") is not None,
"critic_passed": critic.get("verdict") == "pass",
"classification_present": (parts.get("classification") or {}).get("stores") is not None,
"network_present": parts.get("network") is not None,
"itsm_signals_or_declared": bool(cmdb.get("itsm_signals")) or "itsm" in gaps,
"applications_listed": bool((parts.get("applications") or {}).get("applications")),
}
return {"checks": checks, "met": sum(checks.values()), "of": len(checks)}
def decisions(parts: dict) -> dict:
"""What the run leaves for a person to decide, by kind."""
kinds: Counter = Counter()
for item in (parts.get("cmdb") or {}).get("drift_items") or []:
kinds[f"cmdb_{item.get('type')}"] += 1
deps = parts.get("dependencies") or {}
kinds["edges_to_validate"] += len(deps.get("validation_queue") or [])
kinds["cmdb_relationships_dangling"] += len(deps.get("dangling") or [])
kinds["assets_for_sme_review"] += len((parts.get("inventory") or {}).get("sme_review_queue") or [])
grouped = parts.get("applications") or {}
kinds["applications_to_review"] += sum(1 for a in grouped.get("applications") or [] if a.get("review_required"))
kinds["assets_ungrouped"] += len(grouped.get("ungrouped_assets") or [])
kinds["coverage_gaps"] += len((parts.get("plan") or {}).get("coverage_gaps") or [])
critic = parts.get("critic") or {}
kinds["critic_issues"] += len(critic.get("unsupported_claims") or []) + len(critic.get("consistency_issues") or [])
kinds["well_architected_fails"] += sum(1 for c in (parts.get("well_architected") or {}).get("checks") or []
if isinstance(c, dict) and c.get("result") == "fail")
by_kind = {k: v for k, v in sorted(kinds.items()) if v}
return {"total": sum(by_kind.values()), "by_kind": by_kind}
def findings(parts: dict) -> dict:
"""Every finding a person acts on, and how many name an owner and an action."""
grouping = _grouping(parts.get("applications"))
owners = {r.get("canonical_id"): grouping["owner"].get(r.get("canonical_id")) or _facts(r).get("owner")
for r in (parts.get("inventory") or {}).get("records") or []}
rows = []
for item in (parts.get("cmdb") or {}).get("drift_items") or []:
rows.append((owners.get(item.get("canonical_id")), item.get("suggested_action")))
for item in (parts.get("network") or {}).get("findings") or []:
rows.append((None, item.get("recommended_action")))
for check in (parts.get("well_architected") or {}).get("checks") or []:
if isinstance(check, dict) and check.get("result") == "fail":
for asset in check.get("affected") or [None]:
rows.append((owners.get(asset), check.get("recommended_action")))
with_owner = sum(1 for owner, _ in rows if owner not in EMPTY)
with_action = sum(1 for _, action in rows if action not in EMPTY)
return {"total": len(rows), "with_owner": with_owner, "with_action": with_action,
"with_owner_share": _share(with_owner, len(rows)), "with_action_share": _share(with_action, len(rows))}
def score(parts: dict) -> dict:
inventory = parts.get("inventory") or {}
return {"version": VERSION,
"f02_fields": f02_fields(inventory.get("records"), (parts.get("classification") or {}).get("stores"),
(parts.get("plan") or {}).get("source_gaps"), parts.get("applications")),
"f02_run": f02_run(parts), "decisions": decisions(parts), "findings": findings(parts)}