#!/usr/bin/env python3 """Judge a Discovery workflow run against a case in workflows/cases/. No model or agent is involved. Usage: evaluate_workflow_run.py [--usage usage.json] exits 1 when a critical check fails Besides the case's checks it prints the run's value scorecard (ADR 0020): the package step's own, or, for a run without one, the same scorecard computed from the step outputs by the package agent's app/scorecard.py. A case's `value:` bar and `limits:` (checked against scripts/collect_run_usage.py's file) fail as major. """ import json import re import sys from pathlib import Path import yaml FEATURE = Path(__file__).resolve().parents[1] sys.path.insert(0, str(FEATURE / "cp1-discovery-package-agent")) sys.path.insert(0, str(FEATURE / "scripts")) from app import package # noqa: E402 import build_workflow # noqa: E402 AGENTS = {"s1": "planner", "s2": "scan", "s3": "inventory", "s4": "cmdb", "s5": "classification", "s6": "network", "s7": "dependency", "s8": "critic", "s11": "well_architected", "s9": "report", "s13": "package"} # A case's value bar: the scorecard field it reads, and whether the run's value must be at least or at most the bar. VALUE = {"f02_filled_min": (("f02_fields", "filled"), "min"), "f02_filled_or_declared_min": (("f02_fields", "filled_or_declared"), "min"), "f02_missing_max": (("f02_fields", "missing"), "max"), "f02_run_met_min": (("f02_run", "met"), "min"), "findings_with_owner_min": (("findings", "with_owner_share"), "min"), "findings_with_action_min": (("findings", "with_action_share"), "min")} def _parsed(value): while isinstance(value, str): try: value = json.loads(value) except ValueError: return value return value def numbered(steps: dict) -> list: """`kab_run_workflow.py outputs` keys each output by its step's position in the workflow (1 is the upload step), with no agent name: name them from the committed workflow's step order.""" export = json.loads(next((Path(__file__).resolve().parents[1] / "workflows").glob("*.json")).read_text()) order = (export.get("template") or export["workflow"]["template"])["steps"] return [{"id": order[int(k) - 1]["id"], "agent": order[int(k) - 1]["agent"], "output": v} for k, v in sorted(steps.items(), key=lambda kv: int(kv[0])) if int(k) <= len(order)] def outputs_by_step(steps) -> dict: """Each step's output, found by the agent's name: import renames the step ids.""" if isinstance(steps, dict) and isinstance(steps.get("steps"), dict) and all(k.isdigit() for k in steps["steps"]): steps = numbered(steps["steps"]) if isinstance(steps, dict): steps = steps.get("steps") or steps.get("outputs") or [{"id": k, **(v if isinstance(v, dict) else {"output": v})} for k, v in steps.items()] found = {} for step in steps: label = json.dumps({k: step.get(k) for k in ("id", "step_id", "agent", "agent_name", "name", "step_name")}).lower() for short, word in AGENTS.items(): if word in label and short not in found: output = _parsed(step.get("output", step.get("result", step))) if isinstance(output, dict): output = _parsed(output.get("response", output.get("output", output))) found[short] = output return found def _text(value) -> str: return json.dumps(value, ensure_ascii=False, default=str) def _matches(value, check: dict) -> tuple[bool, str]: if "equals" in check: return value == check["equals"], f"is {_text(value)[:120]}" if "contains" in check: return str(check["contains"]) in _text(value), "text not found" if "contains_all" in check: missing = [str(item) for item in check["contains_all"] if str(item) not in _text(value)] return not missing, f"missing {missing}" if "not_contains" in check: return str(check["not_contains"]) not in _text(value), "text found" if "min_items" in check: return isinstance(value, list) and len(value) >= check["min_items"], "too few items" rule = check["item"] items = value if isinstance(value, list) else [value] hits = [item for item in items if re.search(rule["match"], _text(item))] if rule.get("absent"): return not hits, f"{len(hits)} item(s) match" if not hits: return False, "no item matches" rest = {k: v for k, v in rule.items() if k != "match"} if not rest: # `match` alone: at least one item matches return True, "" bad = [item for item in hits if not _matches(item, rest)[0]] return not bad, f"{len(bad)} of {len(hits)} item(s) wrong" def _field(output: dict, path: str): """A field, or a nested one named with dots (`manifest.complete`).""" value = output for key in path.split("."): value = value.get(key) if isinstance(value, dict) else None return value def scorecard_of(found: dict) -> tuple[dict, str]: """The package's own scorecard, else the same one computed from the step outputs.""" pack = found.get("s13") if isinstance(pack, dict) and isinstance((pack.get("manifest") or {}).get("scorecard"), dict): return pack["manifest"]["scorecard"], "from the package step" wiring = dict(next(s for s in build_workflow.STEPS if s[0] == build_workflow.PACKAGE_STEP)[3]) payload = {} for name, source in wiring.items(): step, field = source.split(".") if isinstance(found.get(step), dict) and field in found[step]: payload[name] = found[step][field] return package.scorecard.score(package.parts_from(payload)), "computed from the step outputs (no package step)" def value_checks(case: dict, card: dict, usage: dict | None) -> list[tuple[str, bool, str]]: results = [] for name, bar in (case.get("value") or {}).items(): (section, key), kind = VALUE[name] got = (card.get(section) or {}).get(key) ok = got is not None and (got >= bar if kind == "min" else got <= bar) results.append((f"value {name}", ok, f"is {got}, bar {bar}")) for name, limit in (case.get("limits") or {}).items(): got = (usage or {}).get("totals", {}).get(name) if usage is None: results.append((f"limit {name}", True, "not measured (no --usage file)")) else: results.append((f"limit {name}", got is not None and got <= limit, f"is {got}, limit {limit}")) return results def print_scorecard(card: dict, where: str) -> None: fields, run, decisions, findings = card["f02_fields"], card["f02_run"], card["decisions"], card["findings"] print(f"\nValue scorecard ({where}):") print(f"- F02 fields over {fields['records']} records: {fields['filled']:.0%} filled, {fields['declared_gap']:.0%} " f"declared gap, {fields['missing']:.0%} missing") gaps = [f for f, c in fields["per_field"].items() if c["missing"]] print(f" missing in: {', '.join(gaps) or 'none'}") print(f"- F02 run checks met: {run['met']} of {run['of']}" + "".join(f"; not met: {k}" for k, v in run["checks"].items() if not v)) print(f"- Decisions for a person: {decisions['total']} {decisions['by_kind']}") print(f"- Findings: {findings['total']}, {findings['with_owner_share']:.0%} with an owner, " f"{findings['with_action_share']:.0%} with an action") def main(argv) -> int: usage_path = None if "--usage" in argv: usage_path = argv[argv.index("--usage") + 1] argv = [a for i, a in enumerate(argv) if a != "--usage" and argv[i - 1] != "--usage"] case = yaml.safe_load(Path(argv[0]).read_text()) found = outputs_by_step(json.loads(Path(argv[1]).read_text())) usage = json.loads(Path(usage_path).read_text()) if usage_path else None print(f"Case {case['id']}: {len(found)} of {len(AGENTS)} steps found\n") failed = {"critical": 0, "major": 0} for check in case["checks"]: output = found.get(check["step"]) if not isinstance(output, dict): passed, detail = False, "no output for this step" else: value = _field(output, check["field"]) passed, detail = (False, "field missing") if value is None else _matches(value, check) if not passed: failed[check["severity"]] += 1 print(f"{'pass' if passed else 'FAIL'} {check['severity']:8} {check['step']} {check['id']}" + ("" if passed else f": {detail}. {check['why']}")) card, where = scorecard_of(found) print_scorecard(card, where) extra = value_checks(case, card, usage) for name, passed, detail in extra: failed["major"] += not passed print(f"{'pass' if passed else 'FAIL'} major {name}" + ("" if passed else f": {detail}") + (f" ({detail})" if passed and "not measured" in detail else "")) total = len(case["checks"]) + len(extra) print(f"\n{total - sum(failed.values())} of {total} passed. " f"Failed: {failed['critical']} critical, {failed['major']} major.\n\nFor a person to review:") print("\n".join(f"- {item}" for item in case.get("review") or ["Nothing listed for this case."])) return 1 if failed["critical"] else 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))