Some checks failed
check agents / check (push) Failing after 12s
- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11 (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py, app/business_logic.py); post_process sets those checks, the rejected dispositions, the sign-off queue and the verdict. The model keeps the judgement checks; a different reading of a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue. Tests on the records of run exec_4d88027a4d26, whose model sums were wrong. - A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2 leave it out of dependents and dependencies; the disposition and critic prompts carry the risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail. - The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its app/ and tests/ when regenerating. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
121 lines
5.2 KiB
Python
121 lines
5.2 KiB
Python
"""The critic's code checks (F02 ADR 0011), on the real records of Assessment run exec_4d88027a4d26 (no runtime).
|
|
|
|
In that run the critic model failed K-04 (it expected a score of 0.70, not 1.00), K-05 (it summed 0.90 - 0.10 - 0.20
|
|
- 0.10 to 0.60) and K-11 (a citation's form). The producers' figures were right.
|
|
"""
|
|
import copy
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jsonschema
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from app import arithmetic, business_logic # noqa: E402
|
|
|
|
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
|
|
RUN = json.loads((ROOT / "tests" / "fixtures" / "run-exec_4d88027a4d26.json").read_text())
|
|
INPUTS, REPLY = RUN["inputs"], RUN["reply"]
|
|
|
|
|
|
def run(inputs=None, reply=None):
|
|
inputs, reply = copy.deepcopy(inputs or INPUTS), copy.deepcopy(reply or REPLY)
|
|
business_logic.pre_process(inputs)
|
|
out = business_logic.post_process(reply)
|
|
return out if out is not None else reply
|
|
|
|
|
|
def result(reply, cid):
|
|
return next(c for c in reply["checks"] if c["check_id"] == cid)
|
|
|
|
|
|
def test_the_live_records_recalculate_and_the_verdict_is_set_in_code():
|
|
found = arithmetic.check(INPUTS)
|
|
assert found["K-04"] == [] and found["K-05"] == {} and found["K-06"] == {} and found["K-11"] == {}
|
|
out = run()
|
|
jsonschema.validate(out, OUTPUT)
|
|
assert [result(out, c)["result"] for c in ("K-04", "K-05", "K-06", "K-07", "K-11")] == ["pass"] * 5
|
|
assert result(out, "K-10")["result"] == "unverified" # the model's, kept: it never fails the run alone
|
|
assert out["verdict"] == "pass" and out["rejected_dispositions"] == [] and out["signoff_queue"] == ["APP-001"]
|
|
assert any(n.startswith("K-05: the critic model's reading was fail") for n in out["unverified_claims"])
|
|
|
|
|
|
def test_a_score_that_does_not_recalculate_fails_the_run():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
inputs["scores"][0]["score"] = 0.7
|
|
out = run(inputs)
|
|
assert result(out, "K-04")["result"] == "fail" and out["verdict"] == "fail"
|
|
assert "score 1.0" in result(out, "K-04")["details"][0]["expected"]
|
|
|
|
|
|
def test_a_wrong_confidence_sum_rejects_the_disposition():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
d = inputs["dispositions"][0]
|
|
d["confidence_workings"]["result"] = d["confidence"] = 0.6
|
|
out = run(inputs)
|
|
assert result(out, "K-05")["result"] == "fail" and out["verdict"] == "fail"
|
|
assert [r["check_id"] for r in out["rejected_dispositions"]] == ["K-05"]
|
|
assert out["signoff_queue"] == [] # a rejected disposition is not routed
|
|
|
|
|
|
def test_a_disposition_the_model_rejects_leaves_routing_correct():
|
|
"""Run exec_2cd7a953335b: the model rejected APP-001 under K-02, and K-07 was failed for the queue it then left."""
|
|
reply = copy.deepcopy(REPLY)
|
|
reply["rejected_dispositions"] = [{"application_id": "APP-001", "check_id": "K-02", "reason": "x", "named_dependent": None}]
|
|
for c in reply["checks"]:
|
|
if c["check_id"] == "K-02":
|
|
c["result"] = "fail"
|
|
out = run(reply=reply)
|
|
assert result(out, "K-07")["result"] == "pass" and out["signoff_queue"] == []
|
|
assert out["verdict"] == "fail" and [r["check_id"] for r in out["rejected_dispositions"]] == ["K-02"]
|
|
|
|
|
|
def test_a_producer_queue_missing_an_application_fails_k07():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
inputs["signoff_queue"] = []
|
|
out = run(inputs)
|
|
assert result(out, "K-07")["result"] == "fail" and out["verdict"] == "fail"
|
|
|
|
|
|
def test_the_interim_cap_holds_repurchase_and_relocate_at_079():
|
|
d = {"application_id": "A", "disposition": "repurchase", "confidence": 0.79, "signoff_required": True,
|
|
"confidence_workings": {"base": 0.9, "adjustments": [], "cap": 0.79, "result": 0.79}}
|
|
assert arithmetic.confidence([d]) == {}
|
|
d["confidence_workings"]["result"] = d["confidence"] = 0.9
|
|
assert "result 0.79" in arithmetic.confidence([d])["A"][0]["expected"]
|
|
|
|
|
|
def test_tco_against_the_rate_card():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
m = inputs["dispositions"][0]["tco_model"]
|
|
m["line_items"][0]["rate_id"] = "RC-NOPE"
|
|
m["target_annual"] = 1
|
|
problems = arithmetic.tco(inputs["dispositions"], inputs["rate_card_document"])["APP-001"]
|
|
reasons = {p["reason"] for p in problems}
|
|
assert "unknown rate id" in reasons and any("sum of the mapped rates" in r for r in reasons)
|
|
assert arithmetic.tco(INPUTS["dispositions"], "not json")["APP-001"][0]["reason"].startswith("rate_card_document")
|
|
|
|
|
|
def test_a_check_the_model_owns_still_fails_the_run():
|
|
reply = copy.deepcopy(REPLY)
|
|
for c in reply["checks"]:
|
|
if c["check_id"] == "K-08":
|
|
c["result"] = "fail"
|
|
out = run(reply=reply)
|
|
assert out["verdict"] == "fail" and result(out, "K-08")["result"] == "fail"
|
|
|
|
|
|
def test_a_refused_plan_is_left_to_the_model():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
inputs["preconditions"]["failed"] = ["critic_verdict_pass"]
|
|
business_logic.pre_process(inputs)
|
|
assert business_logic.post_process(copy.deepcopy(REPLY)) is None
|
|
|
|
|
|
def test_the_model_sees_the_code_results():
|
|
inputs = copy.deepcopy(INPUTS)
|
|
business_logic.pre_process(inputs)
|
|
assert inputs["checked_in_code"]["K-04"] == "every risk score recalculates"
|
|
business_logic.post_process(copy.deepcopy(REPLY))
|