- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
190 lines
9.2 KiB
Python
190 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Judge a Discovery workflow run against a case in workflows/cases/. No model or agent is involved.
|
|
|
|
Usage: evaluate_workflow_run.py <case.yaml> <steps.json> [--usage usage.json] exits 1 when a critical check fails
|
|
|
|
Besides the case's checks it prints the run's value scorecard (ADR 0020): the package step's own, or, for a run
|
|
without one, the same scorecard computed from the step outputs by the package agent's app/scorecard.py. A case's
|
|
`value:` bar and `limits:` (checked against scripts/collect_run_usage.py's file) fail as major.
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
FEATURE = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(FEATURE / "cp1-discovery-package-agent"))
|
|
sys.path.insert(0, str(FEATURE / "scripts"))
|
|
from app import package # noqa: E402
|
|
import build_workflow # noqa: E402
|
|
|
|
AGENTS = {"s1": "planner", "s2": "scan", "s3": "inventory", "s4": "cmdb", "s5": "classification", "s6": "network",
|
|
"s7": "dependency", "s8": "critic", "s11": "well_architected", "s9": "report", "s13": "package"}
|
|
# A case's value bar: the scorecard field it reads, and whether the run's value must be at least or at most the bar.
|
|
VALUE = {"f02_filled_min": (("f02_fields", "filled"), "min"),
|
|
"f02_filled_or_declared_min": (("f02_fields", "filled_or_declared"), "min"),
|
|
"f02_missing_max": (("f02_fields", "missing"), "max"),
|
|
"f02_run_met_min": (("f02_run", "met"), "min"),
|
|
"findings_with_owner_min": (("findings", "with_owner_share"), "min"),
|
|
"findings_with_action_min": (("findings", "with_action_share"), "min")}
|
|
|
|
|
|
def _parsed(value):
|
|
while isinstance(value, str):
|
|
try:
|
|
value = json.loads(value)
|
|
except ValueError:
|
|
return value
|
|
return value
|
|
|
|
|
|
def numbered(steps: dict) -> list:
|
|
"""`kab_run_workflow.py outputs` keys each output by its step's position in the workflow (1 is the upload step),
|
|
with no agent name: name them from the committed workflow's step order."""
|
|
export = json.loads(next((Path(__file__).resolve().parents[1] / "workflows").glob("*.json")).read_text())
|
|
order = (export.get("template") or export["workflow"]["template"])["steps"]
|
|
return [{"id": order[int(k) - 1]["id"], "agent": order[int(k) - 1]["agent"], "output": v}
|
|
for k, v in sorted(steps.items(), key=lambda kv: int(kv[0])) if int(k) <= len(order)]
|
|
|
|
|
|
def outputs_by_step(steps) -> dict:
|
|
"""Each step's output, found by the agent's name: import renames the step ids."""
|
|
if isinstance(steps, dict) and isinstance(steps.get("steps"), dict) and all(k.isdigit() for k in steps["steps"]):
|
|
steps = numbered(steps["steps"])
|
|
if isinstance(steps, dict):
|
|
steps = steps.get("steps") or steps.get("outputs") or [{"id": k, **(v if isinstance(v, dict) else {"output": v})}
|
|
for k, v in steps.items()]
|
|
found = {}
|
|
for step in steps:
|
|
label = json.dumps({k: step.get(k) for k in ("id", "step_id", "agent", "agent_name", "name", "step_name")}).lower()
|
|
for short, word in AGENTS.items():
|
|
if word in label and short not in found:
|
|
output = _parsed(step.get("output", step.get("result", step)))
|
|
if isinstance(output, dict):
|
|
output = _parsed(output.get("response", output.get("output", output)))
|
|
found[short] = output
|
|
return found
|
|
|
|
|
|
def _text(value) -> str:
|
|
return json.dumps(value, ensure_ascii=False, default=str)
|
|
|
|
|
|
def _matches(value, check: dict) -> tuple[bool, str]:
|
|
if "equals" in check:
|
|
return value == check["equals"], f"is {_text(value)[:120]}"
|
|
if "contains" in check:
|
|
return str(check["contains"]) in _text(value), "text not found"
|
|
if "contains_all" in check:
|
|
missing = [str(item) for item in check["contains_all"] if str(item) not in _text(value)]
|
|
return not missing, f"missing {missing}"
|
|
if "not_contains" in check:
|
|
return str(check["not_contains"]) not in _text(value), "text found"
|
|
if "min_items" in check:
|
|
return isinstance(value, list) and len(value) >= check["min_items"], "too few items"
|
|
rule = check["item"]
|
|
items = value if isinstance(value, list) else [value]
|
|
hits = [item for item in items if re.search(rule["match"], _text(item))]
|
|
if rule.get("absent"):
|
|
return not hits, f"{len(hits)} item(s) match"
|
|
if not hits:
|
|
return False, "no item matches"
|
|
rest = {k: v for k, v in rule.items() if k != "match"}
|
|
if not rest: # `match` alone: at least one item matches
|
|
return True, ""
|
|
bad = [item for item in hits if not _matches(item, rest)[0]]
|
|
return not bad, f"{len(bad)} of {len(hits)} item(s) wrong"
|
|
|
|
|
|
def _field(output: dict, path: str):
|
|
"""A field, or a nested one named with dots (`manifest.complete`)."""
|
|
value = output
|
|
for key in path.split("."):
|
|
value = value.get(key) if isinstance(value, dict) else None
|
|
return value
|
|
|
|
|
|
def scorecard_of(found: dict) -> tuple[dict, str]:
|
|
"""The package's own scorecard, else the same one computed from the step outputs."""
|
|
pack = found.get("s13")
|
|
if isinstance(pack, dict) and isinstance((pack.get("manifest") or {}).get("scorecard"), dict):
|
|
return pack["manifest"]["scorecard"], "from the package step"
|
|
wiring = dict(next(s for s in build_workflow.STEPS if s[0] == build_workflow.PACKAGE_STEP)[3])
|
|
payload = {}
|
|
for name, source in wiring.items():
|
|
step, field = source.split(".")
|
|
if isinstance(found.get(step), dict) and field in found[step]:
|
|
payload[name] = found[step][field]
|
|
return package.scorecard.score(package.parts_from(payload)), "computed from the step outputs (no package step)"
|
|
|
|
|
|
def value_checks(case: dict, card: dict, usage: dict | None) -> list[tuple[str, bool, str]]:
|
|
results = []
|
|
for name, bar in (case.get("value") or {}).items():
|
|
(section, key), kind = VALUE[name]
|
|
got = (card.get(section) or {}).get(key)
|
|
ok = got is not None and (got >= bar if kind == "min" else got <= bar)
|
|
results.append((f"value {name}", ok, f"is {got}, bar {bar}"))
|
|
for name, limit in (case.get("limits") or {}).items():
|
|
got = (usage or {}).get("totals", {}).get(name)
|
|
if usage is None:
|
|
results.append((f"limit {name}", True, "not measured (no --usage file)"))
|
|
else:
|
|
results.append((f"limit {name}", got is not None and got <= limit, f"is {got}, limit {limit}"))
|
|
return results
|
|
|
|
|
|
def print_scorecard(card: dict, where: str) -> None:
|
|
fields, run, decisions, findings = card["f02_fields"], card["f02_run"], card["decisions"], card["findings"]
|
|
print(f"\nValue scorecard ({where}):")
|
|
print(f"- F02 fields over {fields['records']} records: {fields['filled']:.0%} filled, {fields['declared_gap']:.0%} "
|
|
f"declared gap, {fields['missing']:.0%} missing")
|
|
gaps = [f for f, c in fields["per_field"].items() if c["missing"]]
|
|
print(f" missing in: {', '.join(gaps) or 'none'}")
|
|
print(f"- F02 run checks met: {run['met']} of {run['of']}"
|
|
+ "".join(f"; not met: {k}" for k, v in run["checks"].items() if not v))
|
|
print(f"- Decisions for a person: {decisions['total']} {decisions['by_kind']}")
|
|
print(f"- Findings: {findings['total']}, {findings['with_owner_share']:.0%} with an owner, "
|
|
f"{findings['with_action_share']:.0%} with an action")
|
|
|
|
|
|
def main(argv) -> int:
|
|
usage_path = None
|
|
if "--usage" in argv:
|
|
usage_path = argv[argv.index("--usage") + 1]
|
|
argv = [a for i, a in enumerate(argv) if a != "--usage" and argv[i - 1] != "--usage"]
|
|
case = yaml.safe_load(Path(argv[0]).read_text())
|
|
found = outputs_by_step(json.loads(Path(argv[1]).read_text()))
|
|
usage = json.loads(Path(usage_path).read_text()) if usage_path else None
|
|
print(f"Case {case['id']}: {len(found)} of {len(AGENTS)} steps found\n")
|
|
failed = {"critical": 0, "major": 0}
|
|
for check in case["checks"]:
|
|
output = found.get(check["step"])
|
|
if not isinstance(output, dict):
|
|
passed, detail = False, "no output for this step"
|
|
else:
|
|
value = _field(output, check["field"])
|
|
passed, detail = (False, "field missing") if value is None else _matches(value, check)
|
|
if not passed:
|
|
failed[check["severity"]] += 1
|
|
print(f"{'pass' if passed else 'FAIL'} {check['severity']:8} {check['step']} {check['id']}" +
|
|
("" if passed else f": {detail}. {check['why']}"))
|
|
card, where = scorecard_of(found)
|
|
print_scorecard(card, where)
|
|
extra = value_checks(case, card, usage)
|
|
for name, passed, detail in extra:
|
|
failed["major"] += not passed
|
|
print(f"{'pass' if passed else 'FAIL'} major {name}" + ("" if passed else f": {detail}") +
|
|
(f" ({detail})" if passed and "not measured" in detail else ""))
|
|
total = len(case["checks"]) + len(extra)
|
|
print(f"\n{total - sum(failed.values())} of {total} passed. "
|
|
f"Failed: {failed['critical']} critical, {failed['major']} major.\n\nFor a person to review:")
|
|
print("\n".join(f"- {item}" for item in case.get("review") or ["Nothing listed for this case."]))
|
|
return 1 if failed["critical"] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|