Files
Jonathan Boniface 3b3ca64de0
Some checks failed
check agents / check (push) Failing after 14s
F02: K-02 lookup in code (ADR 0013), represents rule in the profiler; first passing run
- Critic 0.3.0: the hooks look up every disposition citation in the verified
  Discovery package (app/evidence.py). An unresolved citation fails K-02 and
  rejects the disposition; whether the records support the basis stays the
  model's judgement.
- Critic 0.3.1: K-02 judges a disposition's citations together (evidence_ref
  is the primary record), ADR 0013 addendum.
- Profiler 0.1.1: represents edges are not dependencies (ADR 0012, now covering
  ALZ-ASM-02).
- Scaffold: builds every prompt before deleting anything.
- Tests on Discovery run exec_58ba6a71d00b's package and runs 007 and 009 (15).
- Run exec_49e355141382 (RUN-ASM-AZURE-012) passed on crucible-kab-dev.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 15:59:01 +00:00

167 lines
7.5 KiB
Python

"""The critic's code checks (F02 ADR 0011), on the real records of Assessment run exec_4d88027a4d26 (no runtime).
In that run the critic model failed K-04 (it expected a score of 0.70, not 1.00), K-05 (it summed 0.90 - 0.10 - 0.20
- 0.10 to 0.60) and K-11 (a citation's form). The producers' figures were right.
"""
import copy
import json
import sys
from pathlib import Path
import jsonschema
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import arithmetic, business_logic # noqa: E402
OUTPUT = json.loads((ROOT / "schemas" / "output.json").read_text())
RUN = json.loads((ROOT / "tests" / "fixtures" / "run-exec_4d88027a4d26.json").read_text())
INPUTS, REPLY = RUN["inputs"], RUN["reply"]
def run(inputs=None, reply=None):
inputs, reply = copy.deepcopy(inputs or INPUTS), copy.deepcopy(reply or REPLY)
business_logic.pre_process(inputs)
out = business_logic.post_process(reply)
return out if out is not None else reply
def result(reply, cid):
return next(c for c in reply["checks"] if c["check_id"] == cid)
def test_the_live_records_recalculate_and_the_verdict_is_set_in_code():
found = arithmetic.check(INPUTS)
assert found["K-04"] == [] and found["K-05"] == {} and found["K-06"] == {} and found["K-11"] == {}
out = run()
jsonschema.validate(out, OUTPUT)
assert [result(out, c)["result"] for c in ("K-04", "K-05", "K-06", "K-07", "K-11")] == ["pass"] * 5
assert result(out, "K-10")["result"] == "unverified" # the model's, kept: it never fails the run alone
assert out["verdict"] == "pass" and out["rejected_dispositions"] == [] and out["signoff_queue"] == ["APP-001"]
assert any(n.startswith("K-05: the critic model's reading was fail") for n in out["unverified_claims"])
def test_a_score_that_does_not_recalculate_fails_the_run():
inputs = copy.deepcopy(INPUTS)
inputs["scores"][0]["score"] = 0.7
out = run(inputs)
assert result(out, "K-04")["result"] == "fail" and out["verdict"] == "fail"
assert "score 1.0" in result(out, "K-04")["details"][0]["expected"]
def test_a_wrong_confidence_sum_rejects_the_disposition():
inputs = copy.deepcopy(INPUTS)
d = inputs["dispositions"][0]
d["confidence_workings"]["result"] = d["confidence"] = 0.6
out = run(inputs)
assert result(out, "K-05")["result"] == "fail" and out["verdict"] == "fail"
assert [r["check_id"] for r in out["rejected_dispositions"]] == ["K-05"]
assert out["signoff_queue"] == [] # a rejected disposition is not routed
def test_a_disposition_the_model_rejects_leaves_routing_correct():
"""Run exec_2cd7a953335b: the model rejected APP-001 under K-02, and K-07 was failed for the queue it then left."""
reply = copy.deepcopy(REPLY)
reply["rejected_dispositions"] = [{"application_id": "APP-001", "check_id": "K-02", "reason": "x", "named_dependent": None}]
for c in reply["checks"]:
if c["check_id"] == "K-02":
c["result"] = "fail"
out = run(reply=reply)
assert result(out, "K-07")["result"] == "pass" and out["signoff_queue"] == []
assert out["verdict"] == "fail" and [r["check_id"] for r in out["rejected_dispositions"]] == ["K-02"]
def test_a_producer_queue_missing_an_application_fails_k07():
inputs = copy.deepcopy(INPUTS)
inputs["signoff_queue"] = []
out = run(inputs)
assert result(out, "K-07")["result"] == "fail" and out["verdict"] == "fail"
def test_the_interim_cap_holds_repurchase_and_relocate_at_079():
d = {"application_id": "A", "disposition": "repurchase", "confidence": 0.79, "signoff_required": True,
"confidence_workings": {"base": 0.9, "adjustments": [], "cap": 0.79, "result": 0.79}}
assert arithmetic.confidence([d]) == {}
d["confidence_workings"]["result"] = d["confidence"] = 0.9
assert "result 0.79" in arithmetic.confidence([d])["A"][0]["expected"]
def test_tco_against_the_rate_card():
inputs = copy.deepcopy(INPUTS)
m = inputs["dispositions"][0]["tco_model"]
m["line_items"][0]["rate_id"] = "RC-NOPE"
m["target_annual"] = 1
problems = arithmetic.tco(inputs["dispositions"], inputs["rate_card_document"])["APP-001"]
reasons = {p["reason"] for p in problems}
assert "unknown rate id" in reasons and any("sum of the mapped rates" in r for r in reasons)
assert arithmetic.tco(INPUTS["dispositions"], "not json")["APP-001"][0]["reason"].startswith("rate_card_document")
def test_a_check_the_model_owns_still_fails_the_run():
reply = copy.deepcopy(REPLY)
for c in reply["checks"]:
if c["check_id"] == "K-08":
c["result"] = "fail"
out = run(reply=reply)
assert out["verdict"] == "fail" and result(out, "K-08")["result"] == "fail"
def test_a_refused_plan_is_left_to_the_model():
inputs = copy.deepcopy(INPUTS)
inputs["preconditions"]["failed"] = ["critic_verdict_pass"]
business_logic.pre_process(inputs)
assert business_logic.post_process(copy.deepcopy(REPLY)) is None
def test_the_model_sees_the_code_results():
inputs = copy.deepcopy(INPUTS)
business_logic.pre_process(inputs)
assert inputs["checked_in_code"]["K-04"] == "every risk score recalculates"
business_logic.post_process(copy.deepcopy(REPLY))
# --- K-02's lookup (ADR 0013), on Discovery run exec_58ba6a71d00b's real package --------------------------------
from app import evidence # noqa: E402
RUN9 = json.loads((ROOT / "tests" / "fixtures" / "run-exec_1f518b04d6e4.json").read_text())
def test_run_009s_citations_resolve_including_the_classification_records():
"""The critic model failed K-02 in run exec_1f518b04d6e4, saying ALZ-DSC-05/CI-002 and CI-010 don't resolve."""
payload = {**INPUTS, "dispositions": RUN9["dispositions"]}
assert evidence.resolve(payload) == {}
refs = {r for d in RUN9["dispositions"] for r in d["evidence_refs"]}
assert "discovery:exec_58ba6a71d00b/ALZ-DSC-05/CI-002" in refs
def test_with_every_citation_resolved_the_models_support_judgement_stands():
inputs = {**copy.deepcopy(INPUTS), "dispositions": copy.deepcopy(RUN9["dispositions"])}
out = run(inputs, RUN9["reply"])
assert result(out, "K-02") == next(c for c in RUN9["reply"]["checks"] if c["check_id"] == "K-02")
assert out["verdict"] == "fail" # the model's K-02 is a judgement, kept
def test_a_citation_that_does_not_resolve_fails_k02_in_code():
inputs = copy.deepcopy(INPUTS)
d = inputs["dispositions"][0]
d["evidence_refs"] = d["evidence_refs"] + ["discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-999",
"discovery:exec_000000000000/ALZ-DSC-03/CI-001",
"discovery:exec_58ba6a71d00b/ALZ-DSC-99/CI-001"]
out = run(inputs)
reasons = [x["reason"] for x in result(out, "K-02")["details"]]
assert result(out, "K-02")["result"] == "fail" and out["verdict"] == "fail" and len(reasons) == 3
assert any("no record CI-999" in r for r in reasons) and any("not this Discovery run" in r for r in reasons)
assert [r["check_id"] for r in out["rejected_dispositions"]] == ["K-02"] and out["signoff_queue"] == []
def test_a_part_cited_as_a_whole_resolves():
keys = evidence.index(evidence.package(INPUTS["discovery_documents"]))
assert evidence.unresolved("discovery:exec_58ba6a71d00b/ALZ-DSC-04/itsm_signals", "exec_58ba6a71d00b", keys) is None
def test_no_package_fails_k02():
inputs = copy.deepcopy(INPUTS)
inputs["discovery_documents"] = "no package here"
assert result(run(inputs), "K-02")["result"] == "fail"