Files
cp1-agents/individual-agents/ALZ-F01: Discovery Workflow Agent/scripts/collect_run_usage.py
Jonathan Boniface 9e503885cc ALZ-F01 Discovery: package step, value scorecard, application grouping, cases
- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part
  hashed (sha256 over canonical JSON), with a manifest and a value scorecard.
- Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021):
  CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them.
- Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum
  narrower than its producer's.
- Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic
  agents; the report and Well-Architected review agents; ADRs 0012-0021.
- Workflow cases with value bars and run limits; run evaluation and usage collection scripts;
  the run context template and the field-by-field review for ADR 0016.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

67 lines
2.8 KiB
Python

#!/usr/bin/env python3
"""Write one workflow run's model usage, per agent and in total, for a case's `limits:` (ADR 0020).
Reads each agent's own `request_completed` log lines for the execution id, read-only, with kubectl. Nothing on the
cluster is changed. `seconds` is agent time: the sum of each step's request duration, not the run's wall clock, which
includes the review point's wait.
Usage: collect_run_usage.py <execution id> --out usage.json [--context crucible-kab-dev-aks] [--since 6h]
(KUBECONFIG must point at the cluster)
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_workflow # noqa: E402
def agent_lines(agent: str, execution_id: str, context: str, since: str) -> list[dict]:
logs = subprocess.run(["kubectl", "--context", context, "-n", "agents", "logs", f"deploy/{agent}", f"--since={since}"],
capture_output=True, text=True, check=False).stdout
found = []
for line in logs.splitlines():
if execution_id not in line or '"request_completed"' not in line:
continue
try:
found.append(json.loads(line))
except ValueError:
continue
return found
def collect(execution_id: str, context: str, since: str) -> dict:
agents = {}
for _, folder, _, _ in build_workflow.STEPS:
lines = agent_lines(folder, execution_id, context, since)
if lines:
agents[folder] = {"requests": len(lines),
"outcomes": sorted({str(l.get("outcome")) for l in lines}),
"tokens": sum(int(l.get("tokens") or 0) for l in lines),
"model_calls": sum(int(l.get("model_calls") or 0) for l in lines),
"seconds": round(sum(int(l.get("duration_ms") or 0) for l in lines) / 1000, 1)}
totals = {k: (round(sum(a[k] for a in agents.values()), 1) if k == "seconds" else sum(a[k] for a in agents.values()))
for k in ("tokens", "model_calls", "seconds")}
return {"execution_id": execution_id, "agents": agents, "totals": totals}
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("execution_id")
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--context", default="crucible-kab-dev-aks")
parser.add_argument("--since", default="6h")
args = parser.parse_args(argv)
usage = collect(args.execution_id, args.context, args.since)
args.out.write_text(json.dumps(usage, indent=2) + "\n")
print(f"wrote {args.out}: {len(usage['agents'])} agents, {usage['totals']}")
return 0 if usage["agents"] else 1
if __name__ == "__main__":
raise SystemExit(main())