Files
Jonathan Boniface 041e199248
Some checks failed
check agents / check (push) Failing after 12s
ALZ-F02 Assessment: critic code checks (ADR 0011), represents edges (ADR 0012)
- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11
  (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py,
  app/business_logic.py); post_process sets those checks, the rejected dispositions, the
  sign-off queue and the verdict. The model keeps the judgement checks; a different reading of
  a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue.
  Tests on the records of run exec_4d88027a4d26, whose model sums were wrong.
- A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2
  leave it out of dependents and dependencies; the disposition and critic prompts carry the
  risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail.
- The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its
  app/ and tests/ when regenerating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 13:20:40 +00:00

268 lines
16 KiB
Python

"""The critic's mechanical checks, in code (F02 ADR 0011): sums, lookups and set equality, never judgement.
The model judges which rule applies (a factor's value, whether an adjustment is due, which rate maps to an asset).
This module checks only what follows from the producers' own records by arithmetic:
- K-04: every risk score from its factors' values and the fixed weights, renormalised over the available ones; the
partial flag, the missing factors, the 0.70 tail rule and the high-risk tail.
- K-05: every confidence from its workings: the 0.90 base, each adjustment's prescribed delta, the C1 cap, the result.
- K-06: every TCO model: each rate id in the rate card at its annual rate, the line items' sums, the delta.
- K-07: the disposition agent's sign-off queue is exactly its dispositions with confidence below 0.80. The critic's
own queue is those, less the dispositions it rejects.
- K-11: the records passed the input contract on arrival (schema and 7R enum), so they validate.
The rules are ALZ-ASM-03's and ALZ-ASM-04's (their prompts' Scoring and Tail, Confidence, and TCO rules). Standard
library only.
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Optional
W = {"eol": 0.35, "fan_in": 0.25, "data": 0.25, "change": 0.15}
TAIL_CUTOFF = 0.70
BASE = 0.90
SIGNOFF_BELOW = 0.80
INTERIM_CAP = 0.79
INTERIM = ("repurchase", "relocate")
SEVEN_R = ("retire", "repurchase", "replatform", "refactor", "retain", "relocate", "rehost")
# Each adjustment's delta (ALZ-ASM-04 Confidence Rules). P2 is -0.10 per unknown fact, -0.20 at most in all.
DELTA = {"P1": -0.10, "P3": -0.10, "P4": -0.10, "P5": -0.10, "P6": -0.15}
P2_STEP, P2_FLOOR = -0.10, -0.20
TOL = 0.005
CODE_CHECKS = ("K-04", "K-05", "K-06", "K-07", "K-11")
REJECTS = ("K-05", "K-06", "K-11") # a failure rejects the disposition; K-04 and K-07 fail the run
def _near(a: Any, b: Any) -> bool:
if a is None or b is None:
return a is None and b is None
return abs(float(a) - float(b)) <= TOL
def _detail(record: str, expected: Any, recorded: Any, reason: str) -> Dict[str, str]:
return {"record": record, "expected": str(expected), "recorded": str(recorded), "reason": reason}
def risk(scores: List[dict], high_risk_tail: List[str]) -> List[dict]:
"""K-04 details: empty when every score recalculates."""
out = []
for s in scores or []:
app = s.get("application_id", "?")
factors = {f.get("factor"): f for f in s.get("factors") or []}
for name, f in factors.items():
if name in W and not _near(f.get("weight"), W[name]):
out.append(_detail(f"{app} {name}", f"weight {W[name]}", f"weight {f.get('weight')}", "weights are fixed"))
value = f.get("value")
expected = None if value is None else round(float(value) * W.get(name, 0), 4)
if name in W and not _near(f.get("contribution"), expected):
out.append(_detail(f"{app} {name}", f"contribution {expected}", f"contribution {f.get('contribution')}",
"contribution = value x weight, null when the value is missing"))
available = {k: f["value"] for k, f in factors.items() if k in W and f.get("value") is not None}
missing = sorted(k for k in W if k not in available)
if not available:
out.append(_detail(app, "at least one factor with a value", "none", "no factor is available to score"))
continue
score = round(sum(W[k] * float(v) for k, v in available.items()) / sum(W[k] for k in available), 2)
if not _near(s.get("score"), score):
out.append(_detail(app, f"score {score}", f"score {s.get('score')}",
f"SUM(W x value) / SUM(W) over the available factors {sorted(available)}"))
if bool(s.get("partial")) != (len(available) < 4):
out.append(_detail(app, f"partial {len(available) < 4}", f"partial {s.get('partial')}", "partial = fewer than 4 factors"))
if sorted(s.get("missing_factors") or []) != missing:
out.append(_detail(app, f"missing_factors {missing}", f"missing_factors {sorted(s.get('missing_factors') or [])}",
"the factors with no value"))
rules = s.get("tail_rules") or []
if (score >= TAIL_CUTOFF) != ("score>=0.70" in rules):
out.append(_detail(app, f"'score>=0.70' {'in' if score >= TAIL_CUTOFF else 'not in'} tail_rules", rules,
"the 0.70 tail rule"))
if bool(s.get("high_risk")) != bool(rules):
out.append(_detail(app, f"high_risk {bool(rules)}", f"high_risk {s.get('high_risk')}", "high_risk = any tail rule"))
if bool(s.get("high_risk")) != (app in (high_risk_tail or [])):
out.append(_detail(app, f"{'in' if s.get('high_risk') else 'not in'} high_risk_tail", high_risk_tail,
"the tail lists exactly the high-risk applications"))
return out
def _rule(label: Any) -> Optional[str]:
m = re.match(r"\s*(P[1-6]|C1)\b", str(label or ""))
return m.group(1) if m else None
def confidence(dispositions: List[dict]) -> Dict[str, List[dict]]:
"""K-05 details, per application."""
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
app, problems = d.get("application_id", "?"), []
w = d.get("confidence_workings") or {}
if not _near(w.get("base"), BASE):
problems.append(_detail(app, f"base {BASE}", f"base {w.get('base')}", "confidence starts at 0.90"))
total, p2, seen = 0.0, 0.0, set()
for a in w.get("adjustments") or []:
rule, delta = _rule(a.get("rule")), float(a.get("delta") or 0)
if rule is None:
problems.append(_detail(app, "an adjustment P1-P6", a.get("rule"), "not one of the confidence rules"))
continue
if rule == "P2":
if not _near(round(delta / P2_STEP) * P2_STEP, delta) or delta > 0:
problems.append(_detail(app, "P2 in steps of -0.10", delta, "P2 is -0.10 per unknown fact"))
p2 += delta
elif rule in DELTA:
if rule in seen:
problems.append(_detail(app, f"{rule} once", f"{rule} again", "each rule applies once"))
if not _near(delta, DELTA[rule]):
problems.append(_detail(app, f"{rule} {DELTA[rule]}", f"{rule} {delta}", "the rule's fixed delta"))
seen.add(rule)
total += delta
if p2 < P2_FLOOR - TOL:
problems.append(_detail(app, f"P2 at least {P2_FLOOR}", f"P2 {round(p2, 2)}", "P2 is capped at -0.20 in all"))
expected = round(max(0.0, BASE + total), 2)
if d.get("disposition") in INTERIM:
expected = min(expected, INTERIM_CAP)
if not _near(w.get("result"), expected):
problems.append(_detail(app, f"result {expected}", f"result {w.get('result')}",
"0.90 plus the adjustments, at least 0, with C1 for repurchase and relocate"))
if not _near(d.get("confidence"), w.get("result")):
problems.append(_detail(app, f"confidence {w.get('result')}", f"confidence {d.get('confidence')}",
"confidence is the workings' result"))
if bool(d.get("signoff_required")) != (float(d.get("confidence") or 0) < SIGNOFF_BELOW):
problems.append(_detail(app, f"signoff_required {float(d.get('confidence') or 0) < SIGNOFF_BELOW}",
f"signoff_required {d.get('signoff_required')}", "sign-off below 0.80"))
if problems:
out[app] = problems
return out
def _rate_card(document: Any) -> Optional[dict]:
if isinstance(document, dict):
return document
try:
card = json.loads(document) if document else None
except (TypeError, ValueError):
return None
return card if isinstance(card, dict) else None
def tco(dispositions: List[dict], rate_card_document: Any) -> Dict[str, List[dict]]:
"""K-06 details, per application."""
card = _rate_card(rate_card_document)
rates = {i.get("rate_id"): i for i in (card or {}).get("items") or [] if isinstance(i, dict)}
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
app, problems, m = d.get("application_id", "?"), [], d.get("tco_model") or {}
if card is None:
out[app] = [_detail(app, "a rate card", "none readable", "rate_card_document is not a JSON rate card")]
continue
lines = m.get("line_items") or []
for li in lines:
rid = li.get("rate_id")
if rid is None:
continue
if rid not in rates:
problems.append(_detail(f"{app} {li.get('canonical_id')}", "a rate id in the rate card", rid, "unknown rate id"))
elif not _near(li.get("target"), rates[rid].get("annual_rate")):
problems.append(_detail(f"{app} {li.get('canonical_id')}", f"target {rates[rid].get('annual_rate')}",
f"target {li.get('target')}", f"the annual rate of {rid}"))
if m.get("currency") and card.get("currency") and m["currency"] != card["currency"]:
problems.append(_detail(app, f"currency {card['currency']}", f"currency {m['currency']}", "the rate card's currency"))
disposition = d.get("disposition")
currents = [li.get("current") for li in lines]
current = None if (not lines or any(c is None for c in currents)) else round(sum(float(c) for c in currents), 2)
if disposition == "retire":
target = 0.0
elif disposition == "retain":
target = current
else:
targets = [li.get("target") for li in lines if li.get("rate_id") is not None]
target = None if not targets or any(t is None for t in targets) else round(sum(float(t) for t in targets), 2)
if not _near(m.get("current_annual"), current):
problems.append(_detail(app, f"current_annual {current}", f"current_annual {m.get('current_annual')}",
"the sum of the line items' current costs; null if any is missing"))
if current is None and lines and not m.get("missing_inputs"):
problems.append(_detail(app, "the missing costs named", "missing_inputs empty", "a missing cost is named"))
if not _near(m.get("target_annual"), target):
problems.append(_detail(app, f"target_annual {target}", f"target_annual {m.get('target_annual')}",
"0 for retire, current for retain, else the sum of the mapped rates"))
delta = None if target is None or current is None else round(target - current, 2)
if not _near(m.get("annual_delta"), delta):
problems.append(_detail(app, f"annual_delta {delta}", f"annual_delta {m.get('annual_delta')}",
"target minus current; null if either is null"))
if m.get("indicative") is not True:
problems.append(_detail(app, "indicative true", f"indicative {m.get('indicative')}", "the figure is indicative"))
if problems:
out[app] = problems
return out
def schema(dispositions: List[dict]) -> Dict[str, List[dict]]:
"""K-11 details: the input contract validated every record on arrival; the 7R enum is checked again here."""
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
bad = [c for c in [d.get("disposition")] + [t.get("category") for t in d.get("rule_trace") or []] if c not in SEVEN_R]
if bad:
out[d.get("application_id", "?")] = [_detail(d.get("application_id", "?"), "a 7R category", bad, "the 7R enum")]
return out
def check(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Every code check's details, from the critic's inputs."""
dispositions = payload.get("dispositions") or []
return {"K-04": risk(payload.get("scores") or [], payload.get("high_risk_tail") or []),
"K-05": confidence(dispositions), "K-06": tco(dispositions, payload.get("rate_card_document")),
"K-11": schema(dispositions), "K-07": routing(dispositions, payload.get("signoff_queue") or []),
"confidence": {d.get("application_id"): d.get("confidence") for d in dispositions}}
def routing(dispositions: List[dict], producer_queue: List[str]) -> List[dict]:
"""K-07 details: the disposition agent routed exactly the dispositions below 0.80 (ALZ-ASM-04)."""
below = sorted(d.get("application_id") for d in dispositions or []
if d.get("confidence") is not None and float(d["confidence"]) < SIGNOFF_BELOW)
if sorted(producer_queue or []) == below:
return []
return [_detail("signoff_queue (ALZ-ASM-04)", below, sorted(producer_queue or []),
"the disposition agent's queue is its dispositions with confidence below 0.80")]
def apply(reply: Dict[str, Any], found: Dict[str, Any]) -> Dict[str, Any]:
"""The critic's reply with the code checks, the sign-off queue and the verdict set from `found`."""
checks = {c.get("check_id"): c for c in reply.get("checks") or []}
notes = list(reply.get("unverified_claims") or [])
by_app_reject: Dict[str, List[str]] = {}
for cid in ("K-05", "K-06", "K-11"):
for app in found[cid]:
by_app_reject.setdefault(app, []).append(cid)
rejected = [r for r in reply.get("rejected_dispositions") or [] if r.get("check_id") not in CODE_CHECKS]
for app, cids in by_app_reject.items():
for cid in cids:
rejected.append({"application_id": app, "check_id": cid, "named_dependent": None,
"reason": "; ".join(f"{x['record']}: expected {x['expected']}, recorded {x['recorded']}"
for x in found[cid][app])})
rejected_apps = {r.get("application_id") for r in rejected}
queue = sorted(a for a, c in found["confidence"].items() if c is not None and float(c) < SIGNOFF_BELOW
and a not in rejected_apps)
details = {"K-04": found["K-04"], "K-05": [x for v in found["K-05"].values() for x in v],
"K-06": [x for v in found["K-06"].values() for x in v], "K-07": found["K-07"],
"K-11": [x for v in found["K-11"].values() for x in v]}
for cid in CODE_CHECKS:
model = checks.get(cid) or {}
result = "fail" if details[cid] else "pass"
if model.get("result") not in (None, result):
reading = "; ".join(x.get("reason", "") for x in model.get("details") or []) or "no detail"
notes.append(f"{cid}: the critic model's reading was {model.get('result')} ({reading[:400]}); the result is "
f"{result}, set in code from the producers' records (ADR 0011)")
checks[cid] = {"check_id": cid, "result": result, "details": details[cid]}
ordered = [checks[f"K-{i:02d}"] for i in range(1, 13) if f"K-{i:02d}" in checks]
failed = [c["check_id"] for c in ordered if c["result"] == "fail"]
unverified = [c["check_id"] for c in ordered if c["result"] == "unverified" and c["check_id"] != "K-10"]
verdict = "fail" if failed or unverified else "pass"
changed = verdict != reply.get("verdict")
summary = reply.get("summary") or ""
if changed or failed:
summary = (f"Verdict {verdict}, set in code from the checks (ADR 0011): "
+ (f"{', '.join(failed)} failed" if failed else "no check failed")
+ (f"; {', '.join(unverified)} unverified" if unverified else "")
+ ". K-04 to K-07 and K-11 are checked in code; the other checks are the critic model's.")
return {**reply, "verdict": verdict, "checks": ordered, "rejected_dispositions": rejected,
"signoff_queue": queue, "unverified_claims": notes, "summary": summary}