F02: K-02 lookup in code (ADR 0013), represents rule in the profiler; first passing run
Some checks failed
check agents / check (push) Failing after 14s
Some checks failed
check agents / check (push) Failing after 14s
- Critic 0.3.0: the hooks look up every disposition citation in the verified Discovery package (app/evidence.py). An unresolved citation fails K-02 and rejects the disposition; whether the records support the basis stays the model's judgement. - Critic 0.3.1: K-02 judges a disposition's citations together (evidence_ref is the primary record), ADR 0013 addendum. - Profiler 0.1.1: represents edges are not dependencies (ADR 0012, now covering ALZ-ASM-02). - Scaffold: builds every prompt before deleting anything. - Tests on Discovery run exec_58ba6a71d00b's package and runs 007 and 009 (15). - Run exec_49e355141382 (RUN-ASM-AZURE-012) passed on crucible-kab-dev. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ This module checks only what follows from the producers' own records by arithmet
|
||||
- K-07: the disposition agent's sign-off queue is exactly its dispositions with confidence below 0.80. The critic's
|
||||
own queue is those, less the dispositions it rejects.
|
||||
- K-11: the records passed the input contract on arrival (schema and 7R enum), so they validate.
|
||||
- K-02, its lookup only (ADR 0013, `app/evidence.py`): a citation that doesn't resolve to a package record fails
|
||||
K-02 and rejects the disposition. Whether the records support the basis stays the model's.
|
||||
|
||||
The rules are ALZ-ASM-03's and ALZ-ASM-04's (their prompts' Scoring and Tail, Confidence, and TCO rules). Standard
|
||||
library only.
|
||||
@@ -238,6 +240,14 @@ def apply(reply: Dict[str, Any], found: Dict[str, Any]) -> Dict[str, Any]:
|
||||
rejected.append({"application_id": app, "check_id": cid, "named_dependent": None,
|
||||
"reason": "; ".join(f"{x['record']}: expected {x['expected']}, recorded {x['recorded']}"
|
||||
for x in found[cid][app])})
|
||||
for app, problems in (found.get("K-02") or {}).items():
|
||||
if not any(r.get("application_id") == app and r.get("check_id") == "K-02" for r in rejected):
|
||||
rejected.append({"application_id": app, "check_id": "K-02", "named_dependent": None,
|
||||
"reason": "; ".join(f"{x['recorded']}: {x['reason']}" for x in problems)})
|
||||
if found.get("K-02"):
|
||||
model = checks.get("K-02") or {}
|
||||
checks["K-02"] = {"check_id": "K-02", "result": "fail",
|
||||
"details": [x for v in found["K-02"].values() for x in v] + list(model.get("details") or [])}
|
||||
rejected_apps = {r.get("application_id") for r in rejected}
|
||||
queue = sorted(a for a, c in found["confidence"].items() if c is not None and float(c) < SIGNOFF_BELOW
|
||||
and a not in rejected_apps)
|
||||
@@ -262,6 +272,6 @@ def apply(reply: Dict[str, Any], found: Dict[str, Any]) -> Dict[str, Any]:
|
||||
summary = (f"Verdict {verdict}, set in code from the checks (ADR 0011): "
|
||||
+ (f"{', '.join(failed)} failed" if failed else "no check failed")
|
||||
+ (f"; {', '.join(unverified)} unverified" if unverified else "")
|
||||
+ ". K-04 to K-07 and K-11 are checked in code; the other checks are the critic model's.")
|
||||
+ ". K-04 to K-07, K-11 and K-02's lookup are checked in code; the other checks are the critic model's.")
|
||||
return {**reply, "verdict": verdict, "checks": ordered, "rejected_dispositions": rejected,
|
||||
"signoff_queue": queue, "unverified_claims": notes, "summary": summary}
|
||||
|
||||
@@ -14,7 +14,7 @@ import contextvars
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from app import arithmetic
|
||||
from app import arithmetic, evidence
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_found: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar("critic_checked_in_code", default=None)
|
||||
@@ -25,6 +25,7 @@ def pre_process(payload: dict) -> Optional[Any]:
|
||||
_found.set(None) # the planner refused: the critic returns fail on K-01 alone, and nothing is checked here
|
||||
return None
|
||||
found = arithmetic.check(payload)
|
||||
found["K-02"] = evidence.resolve(payload) # the lookup only (ADR 0013)
|
||||
_found.set(found)
|
||||
payload["checked_in_code"] = {
|
||||
"note": "K-04, K-05, K-06, K-07 and K-11 are checked in code from the producers' records (ADR 0011), and "
|
||||
@@ -35,6 +36,8 @@ def pre_process(payload: dict) -> Optional[Any]:
|
||||
"K-06": found["K-06"] or "every TCO model adds up against the rate card",
|
||||
"K-07": found["K-07"] or "the disposition agent routed exactly its dispositions below 0.80",
|
||||
"K-11": found["K-11"] or "every record passed the input contract; every category is in the 7R enum",
|
||||
"K-02 lookup": found["K-02"] or ("every disposition's evidence_ref resolves to a record in the Discovery "
|
||||
"package; judge only whether the cited records support the rule-trace basis"),
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
"""K-02's lookup, in code (F02 ADR 0013): does every disposition's evidence_ref resolve to a record in the package?
|
||||
|
||||
A citation is `discovery:<discovery_execution_id>/<agent_id>/<record_key>`. It resolves when the execution id is the
|
||||
run's, the agent is one whose records Assessment cites, and that agent's part of the Discovery package holds a record
|
||||
with that key in one of its id fields. A part's own field name (`itsm_signals`, say) also resolves: it cites the list,
|
||||
for example to show that it is empty.
|
||||
|
||||
Whether the cited records *support* the rule-trace basis is the critic model's judgement, not this module's.
|
||||
|
||||
The package is found the way the intake finds it (F02 ADR 0008, `cp1-assessment-intake-agent/app/intake.py`): the
|
||||
intake has already verified it, so it is read here, not checked again. Standard library only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Iterator, List, Optional, Set
|
||||
|
||||
FILE_SEPARATOR = "\n\n---\n\n" # the platform's join between parsed files, as in the intake
|
||||
# The Discovery agents Assessment cites, and their part of the package (F01 ADR 0020).
|
||||
PART_OF = {"ALZ-DSC-03": "inventory", "ALZ-DSC-04": "cmdb", "ALZ-DSC-05": "classification",
|
||||
"ALZ-DSC-06": "dependencies", "ALZ-DSC-07": "network"}
|
||||
ID_FIELDS = ("canonical_id", "drift_id", "ci_ref", "edge_id", "id", "finding_id", "rule_id", "range_id", "flow_id")
|
||||
|
||||
|
||||
def _decode(value: Any, depth: int = 0) -> Any:
|
||||
while isinstance(value, str) and depth < 3:
|
||||
try:
|
||||
value, depth = json.loads(value), depth + 1
|
||||
except ValueError:
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
def _packages(value: Any, depth: int = 0) -> Iterator[Dict[str, Any]]:
|
||||
value = _decode(value)
|
||||
if depth > 6:
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
if "package_version" in value and isinstance(value.get("manifest"), dict):
|
||||
yield value
|
||||
return
|
||||
for item in value.values():
|
||||
yield from _packages(item, depth + 1)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from _packages(item, depth + 1)
|
||||
|
||||
|
||||
def package(documents: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
found = [p for doc in (documents or "").split(FILE_SEPARATOR) for p in _packages(doc.strip())]
|
||||
return found[0] if len(found) == 1 else None
|
||||
|
||||
|
||||
def index(pkg: Dict[str, Any]) -> Dict[str, Set[str]]:
|
||||
"""Each cited agent's record keys: the id fields of every record in its part, and the part's own field names."""
|
||||
keys: Dict[str, Set[str]] = {}
|
||||
for agent, part in PART_OF.items():
|
||||
found: Set[str] = set()
|
||||
body = pkg.get(part) or {}
|
||||
for field, value in (body.items() if isinstance(body, dict) else []):
|
||||
found.add(field)
|
||||
for record in value if isinstance(value, list) else []:
|
||||
if isinstance(record, dict):
|
||||
found.update(str(record[f]) for f in ID_FIELDS if record.get(f) not in (None, ""))
|
||||
keys[agent] = found
|
||||
return keys
|
||||
|
||||
|
||||
def unresolved(ref: Any, execution_id: str, keys: Dict[str, Set[str]]) -> Optional[str]:
|
||||
"""Why `ref` doesn't resolve, or None when it does."""
|
||||
text = str(ref or "")
|
||||
if not text.startswith("discovery:"):
|
||||
return "not a discovery: citation"
|
||||
parts = text[len("discovery:"):].split("/", 2)
|
||||
if len(parts) != 3:
|
||||
return "not discovery:<execution_id>/<agent_id>/<record_key>"
|
||||
run, agent, key = parts
|
||||
if run != execution_id:
|
||||
return f"from {run}, not this Discovery run {execution_id}"
|
||||
if agent not in keys:
|
||||
return f"{agent} is not a Discovery agent whose records Assessment cites ({', '.join(PART_OF)})"
|
||||
if key not in keys[agent]:
|
||||
return f"no record {key} in {agent}'s part ({PART_OF[agent]}) of the package"
|
||||
return None
|
||||
|
||||
|
||||
def resolve(payload: Dict[str, Any]) -> Dict[str, List[dict]]:
|
||||
"""K-02 lookup details per application: empty when every disposition's citations resolve."""
|
||||
execution_id = payload.get("discovery_execution_id") or ""
|
||||
pkg = package(payload.get("discovery_documents"))
|
||||
out: Dict[str, List[dict]] = {}
|
||||
for d in payload.get("dispositions") or []:
|
||||
app = d.get("application_id", "?")
|
||||
refs = [d.get("evidence_ref")] + list(d.get("evidence_refs") or [])
|
||||
if pkg is None:
|
||||
out[app] = [{"record": app, "expected": "one Discovery package among discovery_documents",
|
||||
"recorded": "none, or more than one", "reason": "the citations can't be looked up"}]
|
||||
continue
|
||||
keys = index(pkg)
|
||||
problems = []
|
||||
for ref in dict.fromkeys(r for r in refs if r):
|
||||
why = unresolved(ref, execution_id, keys)
|
||||
if why:
|
||||
problems.append({"record": app, "expected": "a citation that resolves to a record in the package",
|
||||
"recorded": str(ref), "reason": why})
|
||||
if problems:
|
||||
out[app] = problems
|
||||
return out
|
||||
Reference in New Issue
Block a user