Files
Jonathan Boniface 3b3ca64de0
Some checks failed
check agents / check (push) Failing after 14s
F02: K-02 lookup in code (ADR 0013), represents rule in the profiler; first passing run
- Critic 0.3.0: the hooks look up every disposition citation in the verified
  Discovery package (app/evidence.py). An unresolved citation fails K-02 and
  rejects the disposition; whether the records support the basis stays the
  model's judgement.
- Critic 0.3.1: K-02 judges a disposition's citations together (evidence_ref
  is the primary record), ADR 0013 addendum.
- Profiler 0.1.1: represents edges are not dependencies (ADR 0012, now covering
  ALZ-ASM-02).
- Scaffold: builds every prompt before deleting anything.
- Tests on Discovery run exec_58ba6a71d00b's package and runs 007 and 009 (15).
- Run exec_49e355141382 (RUN-ASM-AZURE-012) passed on crucible-kab-dev.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 15:59:01 +00:00

278 lines
16 KiB
Python

"""The critic's mechanical checks, in code (F02 ADR 0011): sums, lookups and set equality, never judgement.
The model judges which rule applies (a factor's value, whether an adjustment is due, which rate maps to an asset).
This module checks only what follows from the producers' own records by arithmetic:
- K-04: every risk score from its factors' values and the fixed weights, renormalised over the available ones; the
partial flag, the missing factors, the 0.70 tail rule and the high-risk tail.
- K-05: every confidence from its workings: the 0.90 base, each adjustment's prescribed delta, the C1 cap, the result.
- K-06: every TCO model: each rate id in the rate card at its annual rate, the line items' sums, the delta.
- K-07: the disposition agent's sign-off queue is exactly its dispositions with confidence below 0.80. The critic's
own queue is those, less the dispositions it rejects.
- K-11: the records passed the input contract on arrival (schema and 7R enum), so they validate.
- K-02, its lookup only (ADR 0013, `app/evidence.py`): a citation that doesn't resolve to a package record fails
K-02 and rejects the disposition. Whether the records support the basis stays the model's.
The rules are ALZ-ASM-03's and ALZ-ASM-04's (their prompts' Scoring and Tail, Confidence, and TCO rules). Standard
library only.
"""
from __future__ import annotations
import json
import re
from typing import Any, Dict, List, Optional
W = {"eol": 0.35, "fan_in": 0.25, "data": 0.25, "change": 0.15}
TAIL_CUTOFF = 0.70
BASE = 0.90
SIGNOFF_BELOW = 0.80
INTERIM_CAP = 0.79
INTERIM = ("repurchase", "relocate")
SEVEN_R = ("retire", "repurchase", "replatform", "refactor", "retain", "relocate", "rehost")
# Each adjustment's delta (ALZ-ASM-04 Confidence Rules). P2 is -0.10 per unknown fact, -0.20 at most in all.
DELTA = {"P1": -0.10, "P3": -0.10, "P4": -0.10, "P5": -0.10, "P6": -0.15}
P2_STEP, P2_FLOOR = -0.10, -0.20
TOL = 0.005
CODE_CHECKS = ("K-04", "K-05", "K-06", "K-07", "K-11")
REJECTS = ("K-05", "K-06", "K-11") # a failure rejects the disposition; K-04 and K-07 fail the run
def _near(a: Any, b: Any) -> bool:
if a is None or b is None:
return a is None and b is None
return abs(float(a) - float(b)) <= TOL
def _detail(record: str, expected: Any, recorded: Any, reason: str) -> Dict[str, str]:
return {"record": record, "expected": str(expected), "recorded": str(recorded), "reason": reason}
def risk(scores: List[dict], high_risk_tail: List[str]) -> List[dict]:
"""K-04 details: empty when every score recalculates."""
out = []
for s in scores or []:
app = s.get("application_id", "?")
factors = {f.get("factor"): f for f in s.get("factors") or []}
for name, f in factors.items():
if name in W and not _near(f.get("weight"), W[name]):
out.append(_detail(f"{app} {name}", f"weight {W[name]}", f"weight {f.get('weight')}", "weights are fixed"))
value = f.get("value")
expected = None if value is None else round(float(value) * W.get(name, 0), 4)
if name in W and not _near(f.get("contribution"), expected):
out.append(_detail(f"{app} {name}", f"contribution {expected}", f"contribution {f.get('contribution')}",
"contribution = value x weight, null when the value is missing"))
available = {k: f["value"] for k, f in factors.items() if k in W and f.get("value") is not None}
missing = sorted(k for k in W if k not in available)
if not available:
out.append(_detail(app, "at least one factor with a value", "none", "no factor is available to score"))
continue
score = round(sum(W[k] * float(v) for k, v in available.items()) / sum(W[k] for k in available), 2)
if not _near(s.get("score"), score):
out.append(_detail(app, f"score {score}", f"score {s.get('score')}",
f"SUM(W x value) / SUM(W) over the available factors {sorted(available)}"))
if bool(s.get("partial")) != (len(available) < 4):
out.append(_detail(app, f"partial {len(available) < 4}", f"partial {s.get('partial')}", "partial = fewer than 4 factors"))
if sorted(s.get("missing_factors") or []) != missing:
out.append(_detail(app, f"missing_factors {missing}", f"missing_factors {sorted(s.get('missing_factors') or [])}",
"the factors with no value"))
rules = s.get("tail_rules") or []
if (score >= TAIL_CUTOFF) != ("score>=0.70" in rules):
out.append(_detail(app, f"'score>=0.70' {'in' if score >= TAIL_CUTOFF else 'not in'} tail_rules", rules,
"the 0.70 tail rule"))
if bool(s.get("high_risk")) != bool(rules):
out.append(_detail(app, f"high_risk {bool(rules)}", f"high_risk {s.get('high_risk')}", "high_risk = any tail rule"))
if bool(s.get("high_risk")) != (app in (high_risk_tail or [])):
out.append(_detail(app, f"{'in' if s.get('high_risk') else 'not in'} high_risk_tail", high_risk_tail,
"the tail lists exactly the high-risk applications"))
return out
def _rule(label: Any) -> Optional[str]:
m = re.match(r"\s*(P[1-6]|C1)\b", str(label or ""))
return m.group(1) if m else None
def confidence(dispositions: List[dict]) -> Dict[str, List[dict]]:
"""K-05 details, per application."""
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
app, problems = d.get("application_id", "?"), []
w = d.get("confidence_workings") or {}
if not _near(w.get("base"), BASE):
problems.append(_detail(app, f"base {BASE}", f"base {w.get('base')}", "confidence starts at 0.90"))
total, p2, seen = 0.0, 0.0, set()
for a in w.get("adjustments") or []:
rule, delta = _rule(a.get("rule")), float(a.get("delta") or 0)
if rule is None:
problems.append(_detail(app, "an adjustment P1-P6", a.get("rule"), "not one of the confidence rules"))
continue
if rule == "P2":
if not _near(round(delta / P2_STEP) * P2_STEP, delta) or delta > 0:
problems.append(_detail(app, "P2 in steps of -0.10", delta, "P2 is -0.10 per unknown fact"))
p2 += delta
elif rule in DELTA:
if rule in seen:
problems.append(_detail(app, f"{rule} once", f"{rule} again", "each rule applies once"))
if not _near(delta, DELTA[rule]):
problems.append(_detail(app, f"{rule} {DELTA[rule]}", f"{rule} {delta}", "the rule's fixed delta"))
seen.add(rule)
total += delta
if p2 < P2_FLOOR - TOL:
problems.append(_detail(app, f"P2 at least {P2_FLOOR}", f"P2 {round(p2, 2)}", "P2 is capped at -0.20 in all"))
expected = round(max(0.0, BASE + total), 2)
if d.get("disposition") in INTERIM:
expected = min(expected, INTERIM_CAP)
if not _near(w.get("result"), expected):
problems.append(_detail(app, f"result {expected}", f"result {w.get('result')}",
"0.90 plus the adjustments, at least 0, with C1 for repurchase and relocate"))
if not _near(d.get("confidence"), w.get("result")):
problems.append(_detail(app, f"confidence {w.get('result')}", f"confidence {d.get('confidence')}",
"confidence is the workings' result"))
if bool(d.get("signoff_required")) != (float(d.get("confidence") or 0) < SIGNOFF_BELOW):
problems.append(_detail(app, f"signoff_required {float(d.get('confidence') or 0) < SIGNOFF_BELOW}",
f"signoff_required {d.get('signoff_required')}", "sign-off below 0.80"))
if problems:
out[app] = problems
return out
def _rate_card(document: Any) -> Optional[dict]:
if isinstance(document, dict):
return document
try:
card = json.loads(document) if document else None
except (TypeError, ValueError):
return None
return card if isinstance(card, dict) else None
def tco(dispositions: List[dict], rate_card_document: Any) -> Dict[str, List[dict]]:
"""K-06 details, per application."""
card = _rate_card(rate_card_document)
rates = {i.get("rate_id"): i for i in (card or {}).get("items") or [] if isinstance(i, dict)}
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
app, problems, m = d.get("application_id", "?"), [], d.get("tco_model") or {}
if card is None:
out[app] = [_detail(app, "a rate card", "none readable", "rate_card_document is not a JSON rate card")]
continue
lines = m.get("line_items") or []
for li in lines:
rid = li.get("rate_id")
if rid is None:
continue
if rid not in rates:
problems.append(_detail(f"{app} {li.get('canonical_id')}", "a rate id in the rate card", rid, "unknown rate id"))
elif not _near(li.get("target"), rates[rid].get("annual_rate")):
problems.append(_detail(f"{app} {li.get('canonical_id')}", f"target {rates[rid].get('annual_rate')}",
f"target {li.get('target')}", f"the annual rate of {rid}"))
if m.get("currency") and card.get("currency") and m["currency"] != card["currency"]:
problems.append(_detail(app, f"currency {card['currency']}", f"currency {m['currency']}", "the rate card's currency"))
disposition = d.get("disposition")
currents = [li.get("current") for li in lines]
current = None if (not lines or any(c is None for c in currents)) else round(sum(float(c) for c in currents), 2)
if disposition == "retire":
target = 0.0
elif disposition == "retain":
target = current
else:
targets = [li.get("target") for li in lines if li.get("rate_id") is not None]
target = None if not targets or any(t is None for t in targets) else round(sum(float(t) for t in targets), 2)
if not _near(m.get("current_annual"), current):
problems.append(_detail(app, f"current_annual {current}", f"current_annual {m.get('current_annual')}",
"the sum of the line items' current costs; null if any is missing"))
if current is None and lines and not m.get("missing_inputs"):
problems.append(_detail(app, "the missing costs named", "missing_inputs empty", "a missing cost is named"))
if not _near(m.get("target_annual"), target):
problems.append(_detail(app, f"target_annual {target}", f"target_annual {m.get('target_annual')}",
"0 for retire, current for retain, else the sum of the mapped rates"))
delta = None if target is None or current is None else round(target - current, 2)
if not _near(m.get("annual_delta"), delta):
problems.append(_detail(app, f"annual_delta {delta}", f"annual_delta {m.get('annual_delta')}",
"target minus current; null if either is null"))
if m.get("indicative") is not True:
problems.append(_detail(app, "indicative true", f"indicative {m.get('indicative')}", "the figure is indicative"))
if problems:
out[app] = problems
return out
def schema(dispositions: List[dict]) -> Dict[str, List[dict]]:
"""K-11 details: the input contract validated every record on arrival; the 7R enum is checked again here."""
out: Dict[str, List[dict]] = {}
for d in dispositions or []:
bad = [c for c in [d.get("disposition")] + [t.get("category") for t in d.get("rule_trace") or []] if c not in SEVEN_R]
if bad:
out[d.get("application_id", "?")] = [_detail(d.get("application_id", "?"), "a 7R category", bad, "the 7R enum")]
return out
def check(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Every code check's details, from the critic's inputs."""
dispositions = payload.get("dispositions") or []
return {"K-04": risk(payload.get("scores") or [], payload.get("high_risk_tail") or []),
"K-05": confidence(dispositions), "K-06": tco(dispositions, payload.get("rate_card_document")),
"K-11": schema(dispositions), "K-07": routing(dispositions, payload.get("signoff_queue") or []),
"confidence": {d.get("application_id"): d.get("confidence") for d in dispositions}}
def routing(dispositions: List[dict], producer_queue: List[str]) -> List[dict]:
"""K-07 details: the disposition agent routed exactly the dispositions below 0.80 (ALZ-ASM-04)."""
below = sorted(d.get("application_id") for d in dispositions or []
if d.get("confidence") is not None and float(d["confidence"]) < SIGNOFF_BELOW)
if sorted(producer_queue or []) == below:
return []
return [_detail("signoff_queue (ALZ-ASM-04)", below, sorted(producer_queue or []),
"the disposition agent's queue is its dispositions with confidence below 0.80")]
def apply(reply: Dict[str, Any], found: Dict[str, Any]) -> Dict[str, Any]:
"""The critic's reply with the code checks, the sign-off queue and the verdict set from `found`."""
checks = {c.get("check_id"): c for c in reply.get("checks") or []}
notes = list(reply.get("unverified_claims") or [])
by_app_reject: Dict[str, List[str]] = {}
for cid in ("K-05", "K-06", "K-11"):
for app in found[cid]:
by_app_reject.setdefault(app, []).append(cid)
rejected = [r for r in reply.get("rejected_dispositions") or [] if r.get("check_id") not in CODE_CHECKS]
for app, cids in by_app_reject.items():
for cid in cids:
rejected.append({"application_id": app, "check_id": cid, "named_dependent": None,
"reason": "; ".join(f"{x['record']}: expected {x['expected']}, recorded {x['recorded']}"
for x in found[cid][app])})
for app, problems in (found.get("K-02") or {}).items():
if not any(r.get("application_id") == app and r.get("check_id") == "K-02" for r in rejected):
rejected.append({"application_id": app, "check_id": "K-02", "named_dependent": None,
"reason": "; ".join(f"{x['recorded']}: {x['reason']}" for x in problems)})
if found.get("K-02"):
model = checks.get("K-02") or {}
checks["K-02"] = {"check_id": "K-02", "result": "fail",
"details": [x for v in found["K-02"].values() for x in v] + list(model.get("details") or [])}
rejected_apps = {r.get("application_id") for r in rejected}
queue = sorted(a for a, c in found["confidence"].items() if c is not None and float(c) < SIGNOFF_BELOW
and a not in rejected_apps)
details = {"K-04": found["K-04"], "K-05": [x for v in found["K-05"].values() for x in v],
"K-06": [x for v in found["K-06"].values() for x in v], "K-07": found["K-07"],
"K-11": [x for v in found["K-11"].values() for x in v]}
for cid in CODE_CHECKS:
model = checks.get(cid) or {}
result = "fail" if details[cid] else "pass"
if model.get("result") not in (None, result):
reading = "; ".join(x.get("reason", "") for x in model.get("details") or []) or "no detail"
notes.append(f"{cid}: the critic model's reading was {model.get('result')} ({reading[:400]}); the result is "
f"{result}, set in code from the producers' records (ADR 0011)")
checks[cid] = {"check_id": cid, "result": result, "details": details[cid]}
ordered = [checks[f"K-{i:02d}"] for i in range(1, 13) if f"K-{i:02d}" in checks]
failed = [c["check_id"] for c in ordered if c["result"] == "fail"]
unverified = [c["check_id"] for c in ordered if c["result"] == "unverified" and c["check_id"] != "K-10"]
verdict = "fail" if failed or unverified else "pass"
changed = verdict != reply.get("verdict")
summary = reply.get("summary") or ""
if changed or failed:
summary = (f"Verdict {verdict}, set in code from the checks (ADR 0011): "
+ (f"{', '.join(failed)} failed" if failed else "no check failed")
+ (f"; {', '.join(unverified)} unverified" if unverified else "")
+ ". K-04 to K-07, K-11 and K-02's lookup are checked in code; the other checks are the critic model's.")
return {**reply, "verdict": verdict, "checks": ordered, "rejected_dispositions": rejected,
"signoff_queue": queue, "unverified_claims": notes, "summary": summary}