Some checks failed
check agents / check (push) Failing after 12s
- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11 (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py, app/business_logic.py); post_process sets those checks, the rejected dispositions, the sign-off queue and the verdict. The model keeps the judgement checks; a different reading of a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue. Tests on the records of run exec_4d88027a4d26, whose model sums were wrong. - A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2 leave it out of dependents and dependencies; the disposition and critic prompts carry the risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail. - The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its app/ and tests/ when regenerating. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
268 lines
16 KiB
Python
268 lines
16 KiB
Python
"""The critic's mechanical checks, in code (F02 ADR 0011): sums, lookups and set equality, never judgement.
|
|
|
|
The model judges which rule applies (a factor's value, whether an adjustment is due, which rate maps to an asset).
|
|
This module checks only what follows from the producers' own records by arithmetic:
|
|
|
|
- K-04: every risk score from its factors' values and the fixed weights, renormalised over the available ones; the
|
|
partial flag, the missing factors, the 0.70 tail rule and the high-risk tail.
|
|
- K-05: every confidence from its workings: the 0.90 base, each adjustment's prescribed delta, the C1 cap, the result.
|
|
- K-06: every TCO model: each rate id in the rate card at its annual rate, the line items' sums, the delta.
|
|
- K-07: the disposition agent's sign-off queue is exactly its dispositions with confidence below 0.80. The critic's
|
|
own queue is those, less the dispositions it rejects.
|
|
- K-11: the records passed the input contract on arrival (schema and 7R enum), so they validate.
|
|
|
|
The rules are ALZ-ASM-03's and ALZ-ASM-04's (their prompts' Scoring and Tail, Confidence, and TCO rules). Standard
|
|
library only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
W = {"eol": 0.35, "fan_in": 0.25, "data": 0.25, "change": 0.15}
|
|
TAIL_CUTOFF = 0.70
|
|
BASE = 0.90
|
|
SIGNOFF_BELOW = 0.80
|
|
INTERIM_CAP = 0.79
|
|
INTERIM = ("repurchase", "relocate")
|
|
SEVEN_R = ("retire", "repurchase", "replatform", "refactor", "retain", "relocate", "rehost")
|
|
# Each adjustment's delta (ALZ-ASM-04 Confidence Rules). P2 is -0.10 per unknown fact, -0.20 at most in all.
|
|
DELTA = {"P1": -0.10, "P3": -0.10, "P4": -0.10, "P5": -0.10, "P6": -0.15}
|
|
P2_STEP, P2_FLOOR = -0.10, -0.20
|
|
TOL = 0.005
|
|
CODE_CHECKS = ("K-04", "K-05", "K-06", "K-07", "K-11")
|
|
REJECTS = ("K-05", "K-06", "K-11") # a failure rejects the disposition; K-04 and K-07 fail the run
|
|
|
|
|
|
def _near(a: Any, b: Any) -> bool:
|
|
if a is None or b is None:
|
|
return a is None and b is None
|
|
return abs(float(a) - float(b)) <= TOL
|
|
|
|
|
|
def _detail(record: str, expected: Any, recorded: Any, reason: str) -> Dict[str, str]:
|
|
return {"record": record, "expected": str(expected), "recorded": str(recorded), "reason": reason}
|
|
|
|
|
|
def risk(scores: List[dict], high_risk_tail: List[str]) -> List[dict]:
|
|
"""K-04 details: empty when every score recalculates."""
|
|
out = []
|
|
for s in scores or []:
|
|
app = s.get("application_id", "?")
|
|
factors = {f.get("factor"): f for f in s.get("factors") or []}
|
|
for name, f in factors.items():
|
|
if name in W and not _near(f.get("weight"), W[name]):
|
|
out.append(_detail(f"{app} {name}", f"weight {W[name]}", f"weight {f.get('weight')}", "weights are fixed"))
|
|
value = f.get("value")
|
|
expected = None if value is None else round(float(value) * W.get(name, 0), 4)
|
|
if name in W and not _near(f.get("contribution"), expected):
|
|
out.append(_detail(f"{app} {name}", f"contribution {expected}", f"contribution {f.get('contribution')}",
|
|
"contribution = value x weight, null when the value is missing"))
|
|
available = {k: f["value"] for k, f in factors.items() if k in W and f.get("value") is not None}
|
|
missing = sorted(k for k in W if k not in available)
|
|
if not available:
|
|
out.append(_detail(app, "at least one factor with a value", "none", "no factor is available to score"))
|
|
continue
|
|
score = round(sum(W[k] * float(v) for k, v in available.items()) / sum(W[k] for k in available), 2)
|
|
if not _near(s.get("score"), score):
|
|
out.append(_detail(app, f"score {score}", f"score {s.get('score')}",
|
|
f"SUM(W x value) / SUM(W) over the available factors {sorted(available)}"))
|
|
if bool(s.get("partial")) != (len(available) < 4):
|
|
out.append(_detail(app, f"partial {len(available) < 4}", f"partial {s.get('partial')}", "partial = fewer than 4 factors"))
|
|
if sorted(s.get("missing_factors") or []) != missing:
|
|
out.append(_detail(app, f"missing_factors {missing}", f"missing_factors {sorted(s.get('missing_factors') or [])}",
|
|
"the factors with no value"))
|
|
rules = s.get("tail_rules") or []
|
|
if (score >= TAIL_CUTOFF) != ("score>=0.70" in rules):
|
|
out.append(_detail(app, f"'score>=0.70' {'in' if score >= TAIL_CUTOFF else 'not in'} tail_rules", rules,
|
|
"the 0.70 tail rule"))
|
|
if bool(s.get("high_risk")) != bool(rules):
|
|
out.append(_detail(app, f"high_risk {bool(rules)}", f"high_risk {s.get('high_risk')}", "high_risk = any tail rule"))
|
|
if bool(s.get("high_risk")) != (app in (high_risk_tail or [])):
|
|
out.append(_detail(app, f"{'in' if s.get('high_risk') else 'not in'} high_risk_tail", high_risk_tail,
|
|
"the tail lists exactly the high-risk applications"))
|
|
return out
|
|
|
|
|
|
def _rule(label: Any) -> Optional[str]:
|
|
m = re.match(r"\s*(P[1-6]|C1)\b", str(label or ""))
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def confidence(dispositions: List[dict]) -> Dict[str, List[dict]]:
|
|
"""K-05 details, per application."""
|
|
out: Dict[str, List[dict]] = {}
|
|
for d in dispositions or []:
|
|
app, problems = d.get("application_id", "?"), []
|
|
w = d.get("confidence_workings") or {}
|
|
if not _near(w.get("base"), BASE):
|
|
problems.append(_detail(app, f"base {BASE}", f"base {w.get('base')}", "confidence starts at 0.90"))
|
|
total, p2, seen = 0.0, 0.0, set()
|
|
for a in w.get("adjustments") or []:
|
|
rule, delta = _rule(a.get("rule")), float(a.get("delta") or 0)
|
|
if rule is None:
|
|
problems.append(_detail(app, "an adjustment P1-P6", a.get("rule"), "not one of the confidence rules"))
|
|
continue
|
|
if rule == "P2":
|
|
if not _near(round(delta / P2_STEP) * P2_STEP, delta) or delta > 0:
|
|
problems.append(_detail(app, "P2 in steps of -0.10", delta, "P2 is -0.10 per unknown fact"))
|
|
p2 += delta
|
|
elif rule in DELTA:
|
|
if rule in seen:
|
|
problems.append(_detail(app, f"{rule} once", f"{rule} again", "each rule applies once"))
|
|
if not _near(delta, DELTA[rule]):
|
|
problems.append(_detail(app, f"{rule} {DELTA[rule]}", f"{rule} {delta}", "the rule's fixed delta"))
|
|
seen.add(rule)
|
|
total += delta
|
|
if p2 < P2_FLOOR - TOL:
|
|
problems.append(_detail(app, f"P2 at least {P2_FLOOR}", f"P2 {round(p2, 2)}", "P2 is capped at -0.20 in all"))
|
|
expected = round(max(0.0, BASE + total), 2)
|
|
if d.get("disposition") in INTERIM:
|
|
expected = min(expected, INTERIM_CAP)
|
|
if not _near(w.get("result"), expected):
|
|
problems.append(_detail(app, f"result {expected}", f"result {w.get('result')}",
|
|
"0.90 plus the adjustments, at least 0, with C1 for repurchase and relocate"))
|
|
if not _near(d.get("confidence"), w.get("result")):
|
|
problems.append(_detail(app, f"confidence {w.get('result')}", f"confidence {d.get('confidence')}",
|
|
"confidence is the workings' result"))
|
|
if bool(d.get("signoff_required")) != (float(d.get("confidence") or 0) < SIGNOFF_BELOW):
|
|
problems.append(_detail(app, f"signoff_required {float(d.get('confidence') or 0) < SIGNOFF_BELOW}",
|
|
f"signoff_required {d.get('signoff_required')}", "sign-off below 0.80"))
|
|
if problems:
|
|
out[app] = problems
|
|
return out
|
|
|
|
|
|
def _rate_card(document: Any) -> Optional[dict]:
|
|
if isinstance(document, dict):
|
|
return document
|
|
try:
|
|
card = json.loads(document) if document else None
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return card if isinstance(card, dict) else None
|
|
|
|
|
|
def tco(dispositions: List[dict], rate_card_document: Any) -> Dict[str, List[dict]]:
|
|
"""K-06 details, per application."""
|
|
card = _rate_card(rate_card_document)
|
|
rates = {i.get("rate_id"): i for i in (card or {}).get("items") or [] if isinstance(i, dict)}
|
|
out: Dict[str, List[dict]] = {}
|
|
for d in dispositions or []:
|
|
app, problems, m = d.get("application_id", "?"), [], d.get("tco_model") or {}
|
|
if card is None:
|
|
out[app] = [_detail(app, "a rate card", "none readable", "rate_card_document is not a JSON rate card")]
|
|
continue
|
|
lines = m.get("line_items") or []
|
|
for li in lines:
|
|
rid = li.get("rate_id")
|
|
if rid is None:
|
|
continue
|
|
if rid not in rates:
|
|
problems.append(_detail(f"{app} {li.get('canonical_id')}", "a rate id in the rate card", rid, "unknown rate id"))
|
|
elif not _near(li.get("target"), rates[rid].get("annual_rate")):
|
|
problems.append(_detail(f"{app} {li.get('canonical_id')}", f"target {rates[rid].get('annual_rate')}",
|
|
f"target {li.get('target')}", f"the annual rate of {rid}"))
|
|
if m.get("currency") and card.get("currency") and m["currency"] != card["currency"]:
|
|
problems.append(_detail(app, f"currency {card['currency']}", f"currency {m['currency']}", "the rate card's currency"))
|
|
disposition = d.get("disposition")
|
|
currents = [li.get("current") for li in lines]
|
|
current = None if (not lines or any(c is None for c in currents)) else round(sum(float(c) for c in currents), 2)
|
|
if disposition == "retire":
|
|
target = 0.0
|
|
elif disposition == "retain":
|
|
target = current
|
|
else:
|
|
targets = [li.get("target") for li in lines if li.get("rate_id") is not None]
|
|
target = None if not targets or any(t is None for t in targets) else round(sum(float(t) for t in targets), 2)
|
|
if not _near(m.get("current_annual"), current):
|
|
problems.append(_detail(app, f"current_annual {current}", f"current_annual {m.get('current_annual')}",
|
|
"the sum of the line items' current costs; null if any is missing"))
|
|
if current is None and lines and not m.get("missing_inputs"):
|
|
problems.append(_detail(app, "the missing costs named", "missing_inputs empty", "a missing cost is named"))
|
|
if not _near(m.get("target_annual"), target):
|
|
problems.append(_detail(app, f"target_annual {target}", f"target_annual {m.get('target_annual')}",
|
|
"0 for retire, current for retain, else the sum of the mapped rates"))
|
|
delta = None if target is None or current is None else round(target - current, 2)
|
|
if not _near(m.get("annual_delta"), delta):
|
|
problems.append(_detail(app, f"annual_delta {delta}", f"annual_delta {m.get('annual_delta')}",
|
|
"target minus current; null if either is null"))
|
|
if m.get("indicative") is not True:
|
|
problems.append(_detail(app, "indicative true", f"indicative {m.get('indicative')}", "the figure is indicative"))
|
|
if problems:
|
|
out[app] = problems
|
|
return out
|
|
|
|
|
|
def schema(dispositions: List[dict]) -> Dict[str, List[dict]]:
|
|
"""K-11 details: the input contract validated every record on arrival; the 7R enum is checked again here."""
|
|
out: Dict[str, List[dict]] = {}
|
|
for d in dispositions or []:
|
|
bad = [c for c in [d.get("disposition")] + [t.get("category") for t in d.get("rule_trace") or []] if c not in SEVEN_R]
|
|
if bad:
|
|
out[d.get("application_id", "?")] = [_detail(d.get("application_id", "?"), "a 7R category", bad, "the 7R enum")]
|
|
return out
|
|
|
|
|
|
def check(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Every code check's details, from the critic's inputs."""
|
|
dispositions = payload.get("dispositions") or []
|
|
return {"K-04": risk(payload.get("scores") or [], payload.get("high_risk_tail") or []),
|
|
"K-05": confidence(dispositions), "K-06": tco(dispositions, payload.get("rate_card_document")),
|
|
"K-11": schema(dispositions), "K-07": routing(dispositions, payload.get("signoff_queue") or []),
|
|
"confidence": {d.get("application_id"): d.get("confidence") for d in dispositions}}
|
|
|
|
|
|
def routing(dispositions: List[dict], producer_queue: List[str]) -> List[dict]:
|
|
"""K-07 details: the disposition agent routed exactly the dispositions below 0.80 (ALZ-ASM-04)."""
|
|
below = sorted(d.get("application_id") for d in dispositions or []
|
|
if d.get("confidence") is not None and float(d["confidence"]) < SIGNOFF_BELOW)
|
|
if sorted(producer_queue or []) == below:
|
|
return []
|
|
return [_detail("signoff_queue (ALZ-ASM-04)", below, sorted(producer_queue or []),
|
|
"the disposition agent's queue is its dispositions with confidence below 0.80")]
|
|
|
|
|
|
def apply(reply: Dict[str, Any], found: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""The critic's reply with the code checks, the sign-off queue and the verdict set from `found`."""
|
|
checks = {c.get("check_id"): c for c in reply.get("checks") or []}
|
|
notes = list(reply.get("unverified_claims") or [])
|
|
by_app_reject: Dict[str, List[str]] = {}
|
|
for cid in ("K-05", "K-06", "K-11"):
|
|
for app in found[cid]:
|
|
by_app_reject.setdefault(app, []).append(cid)
|
|
rejected = [r for r in reply.get("rejected_dispositions") or [] if r.get("check_id") not in CODE_CHECKS]
|
|
for app, cids in by_app_reject.items():
|
|
for cid in cids:
|
|
rejected.append({"application_id": app, "check_id": cid, "named_dependent": None,
|
|
"reason": "; ".join(f"{x['record']}: expected {x['expected']}, recorded {x['recorded']}"
|
|
for x in found[cid][app])})
|
|
rejected_apps = {r.get("application_id") for r in rejected}
|
|
queue = sorted(a for a, c in found["confidence"].items() if c is not None and float(c) < SIGNOFF_BELOW
|
|
and a not in rejected_apps)
|
|
details = {"K-04": found["K-04"], "K-05": [x for v in found["K-05"].values() for x in v],
|
|
"K-06": [x for v in found["K-06"].values() for x in v], "K-07": found["K-07"],
|
|
"K-11": [x for v in found["K-11"].values() for x in v]}
|
|
for cid in CODE_CHECKS:
|
|
model = checks.get(cid) or {}
|
|
result = "fail" if details[cid] else "pass"
|
|
if model.get("result") not in (None, result):
|
|
reading = "; ".join(x.get("reason", "") for x in model.get("details") or []) or "no detail"
|
|
notes.append(f"{cid}: the critic model's reading was {model.get('result')} ({reading[:400]}); the result is "
|
|
f"{result}, set in code from the producers' records (ADR 0011)")
|
|
checks[cid] = {"check_id": cid, "result": result, "details": details[cid]}
|
|
ordered = [checks[f"K-{i:02d}"] for i in range(1, 13) if f"K-{i:02d}" in checks]
|
|
failed = [c["check_id"] for c in ordered if c["result"] == "fail"]
|
|
unverified = [c["check_id"] for c in ordered if c["result"] == "unverified" and c["check_id"] != "K-10"]
|
|
verdict = "fail" if failed or unverified else "pass"
|
|
changed = verdict != reply.get("verdict")
|
|
summary = reply.get("summary") or ""
|
|
if changed or failed:
|
|
summary = (f"Verdict {verdict}, set in code from the checks (ADR 0011): "
|
|
+ (f"{', '.join(failed)} failed" if failed else "no check failed")
|
|
+ (f"; {', '.join(unverified)} unverified" if unverified else "")
|
|
+ ". K-04 to K-07 and K-11 are checked in code; the other checks are the critic model's.")
|
|
return {**reply, "verdict": verdict, "checks": ordered, "rejected_dispositions": rejected,
|
|
"signoff_queue": queue, "unverified_claims": notes, "summary": summary}
|