Files
Jonathan Boniface 041e199248
Some checks failed
check agents / check (push) Failing after 12s
ALZ-F02 Assessment: critic code checks (ADR 0011), represents edges (ADR 0012)
- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11
  (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py,
  app/business_logic.py); post_process sets those checks, the rejected dispositions, the
  sign-off queue and the verdict. The model keeps the judgement checks; a different reading of
  a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue.
  Tests on the records of run exec_4d88027a4d26, whose model sums were wrong.
- A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2
  leave it out of dependents and dependencies; the disposition and critic prompts carry the
  risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail.
- The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its
  app/ and tests/ when regenerating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 13:20:40 +00:00

121 lines
5.2 KiB
Python

"""The critic's code checks (F02 ADR 0011), on the real records of Assessment run exec_4d88027a4d26 (no runtime).
In that run the critic model failed K-04 (it expected a score of 0.70, not 1.00), K-05 (it summed 0.90 - 0.10 - 0.20
- 0.10 to 0.60) and K-11 (a citation's form). The producers' figures were right.
"""
import copy
import json
import sys
from pathlib import Path
import jsonschema
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import arithmetic, business_logic # noqa: E402
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
RUN = json.loads((ROOT / "tests" / "fixtures" / "run-exec_4d88027a4d26.json").read_text())
INPUTS, REPLY = RUN["inputs"], RUN["reply"]
def run(inputs=None, reply=None):
inputs, reply = copy.deepcopy(inputs or INPUTS), copy.deepcopy(reply or REPLY)
business_logic.pre_process(inputs)
out = business_logic.post_process(reply)
return out if out is not None else reply
def result(reply, cid):
return next(c for c in reply["checks"] if c["check_id"] == cid)
def test_the_live_records_recalculate_and_the_verdict_is_set_in_code():
found = arithmetic.check(INPUTS)
assert found["K-04"] == [] and found["K-05"] == {} and found["K-06"] == {} and found["K-11"] == {}
out = run()
jsonschema.validate(out, OUTPUT)
assert [result(out, c)["result"] for c in ("K-04", "K-05", "K-06", "K-07", "K-11")] == ["pass"] * 5
assert result(out, "K-10")["result"] == "unverified" # the model's, kept: it never fails the run alone
assert out["verdict"] == "pass" and out["rejected_dispositions"] == [] and out["signoff_queue"] == ["APP-001"]
assert any(n.startswith("K-05: the critic model's reading was fail") for n in out["unverified_claims"])
def test_a_score_that_does_not_recalculate_fails_the_run():
inputs = copy.deepcopy(INPUTS)
inputs["scores"][0]["score"] = 0.7
out = run(inputs)
assert result(out, "K-04")["result"] == "fail" and out["verdict"] == "fail"
assert "score 1.0" in result(out, "K-04")["details"][0]["expected"]
def test_a_wrong_confidence_sum_rejects_the_disposition():
inputs = copy.deepcopy(INPUTS)
d = inputs["dispositions"][0]
d["confidence_workings"]["result"] = d["confidence"] = 0.6
out = run(inputs)
assert result(out, "K-05")["result"] == "fail" and out["verdict"] == "fail"
assert [r["check_id"] for r in out["rejected_dispositions"]] == ["K-05"]
assert out["signoff_queue"] == [] # a rejected disposition is not routed
def test_a_disposition_the_model_rejects_leaves_routing_correct():
"""Run exec_2cd7a953335b: the model rejected APP-001 under K-02, and K-07 was failed for the queue it then left."""
reply = copy.deepcopy(REPLY)
reply["rejected_dispositions"] = [{"application_id": "APP-001", "check_id": "K-02", "reason": "x", "named_dependent": None}]
for c in reply["checks"]:
if c["check_id"] == "K-02":
c["result"] = "fail"
out = run(reply=reply)
assert result(out, "K-07")["result"] == "pass" and out["signoff_queue"] == []
assert out["verdict"] == "fail" and [r["check_id"] for r in out["rejected_dispositions"]] == ["K-02"]
def test_a_producer_queue_missing_an_application_fails_k07():
inputs = copy.deepcopy(INPUTS)
inputs["signoff_queue"] = []
out = run(inputs)
assert result(out, "K-07")["result"] == "fail" and out["verdict"] == "fail"
def test_the_interim_cap_holds_repurchase_and_relocate_at_079():
d = {"application_id": "A", "disposition": "repurchase", "confidence": 0.79, "signoff_required": True,
"confidence_workings": {"base": 0.9, "adjustments": [], "cap": 0.79, "result": 0.79}}
assert arithmetic.confidence([d]) == {}
d["confidence_workings"]["result"] = d["confidence"] = 0.9
assert "result 0.79" in arithmetic.confidence([d])["A"][0]["expected"]
def test_tco_against_the_rate_card():
inputs = copy.deepcopy(INPUTS)
m = inputs["dispositions"][0]["tco_model"]
m["line_items"][0]["rate_id"] = "RC-NOPE"
m["target_annual"] = 1
problems = arithmetic.tco(inputs["dispositions"], inputs["rate_card_document"])["APP-001"]
reasons = {p["reason"] for p in problems}
assert "unknown rate id" in reasons and any("sum of the mapped rates" in r for r in reasons)
assert arithmetic.tco(INPUTS["dispositions"], "not json")["APP-001"][0]["reason"].startswith("rate_card_document")
def test_a_check_the_model_owns_still_fails_the_run():
reply = copy.deepcopy(REPLY)
for c in reply["checks"]:
if c["check_id"] == "K-08":
c["result"] = "fail"
out = run(reply=reply)
assert out["verdict"] == "fail" and result(out, "K-08")["result"] == "fail"
def test_a_refused_plan_is_left_to_the_model():
inputs = copy.deepcopy(INPUTS)
inputs["preconditions"]["failed"] = ["critic_verdict_pass"]
business_logic.pre_process(inputs)
assert business_logic.post_process(copy.deepcopy(REPLY)) is None
def test_the_model_sees_the_code_results():
inputs = copy.deepcopy(INPUTS)
business_logic.pre_process(inputs)
assert inputs["checked_in_code"]["K-04"] == "every risk score recalculates"
business_logic.post_process(copy.deepcopy(REPLY))