ALZ-F02 Assessment: critic code checks (ADR 0011), represents edges (ADR 0012)
Some checks failed
check agents / check (push) Failing after 12s

- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11
  (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py,
  app/business_logic.py); post_process sets those checks, the rejected dispositions, the
  sign-off queue and the verdict. The model keeps the judgement checks; a different reading of
  a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue.
  Tests on the records of run exec_4d88027a4d26, whose model sums were wrong.
- A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2
  leave it out of dependents and dependencies; the disposition and critic prompts carry the
  risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail.
- The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its
  app/ and tests/ when regenerating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-06 13:20:40 +00:00
parent bc84ee4df6
commit 041e199248
24 changed files with 1112 additions and 15 deletions

View File

@@ -135,6 +135,20 @@ RATECARD = {"type": ["string", "null"], "default": None,
"see the Feature README."}
DISCOVERY_IN = {"discovery_execution_id": DEXEC, "discovery_documents": DDOCS, "discovery_files": DFILES}
# A `represents` edge (F01 ADR 0019) is a CMDB record matched to the asset it stands for: the same asset, not a
# dependency. Run exec_2cd7a953335b counted eight of them as dependents (fan-in 1.0, high risk; retire ruled out).
REPRESENTS_REWRITES = [
("dependents(app) = { owner(e.source) FOR e IN edges IF e.target IN app.assets AND owner(e.source) != app }\n"
"depends_on(app) = { owner(e.target) FOR e IN edges IF e.source IN app.assets AND owner(e.target) != app }",
"dependency(e) = e.type != 'represents' # a CMDB record matched to its asset is that asset (F02 ADR 0012)\n"
"dependents(app) = { owner(e.source) FOR e IN edges IF dependency(e) AND e.target IN app.assets AND owner(e.source) != app }\n"
"depends_on(app) = { owner(e.target) FOR e IN edges IF dependency(e) AND e.source IN app.assets AND owner(e.target) != app }"),
("counts, so a dependency is never lost because its source has no application.",
"counts, so a dependency is never lost because its source has no application. A `represents` edge (`type: "
"represents`) is not a dependency: it joins a CMDB record to the inventory asset it stands for, so it is the same "
"asset, and it is left out of both sets."),
]
AGENTS = [
dict(n="01", key="cp1-assessment-planner-agent", ado=3214459, display="CP1 - Assessment Planner Agent",
desc="Checks that Assessment may start on an approved, passing Discovery run and a rate card, then plans "
@@ -189,7 +203,7 @@ AGENTS = [
required=["engagement_id", "discovery_execution_id", "tasks", "applications"],
output=obj({"discovery_execution_id": S, "scores": arr(SCORE), "high_risk_tail": arr(S), "findings_note": S},
["discovery_execution_id", "scores", "high_risk_tail", "findings_note"], title="Risk scores"),
rewrites=[]),
rewrites=REPRESENTS_REWRITES),
dict(n="05", key="cp1-security-posture-assessment-agent", ado=3214463,
display="CP1 - Security Posture Assessment Agent",
desc="For in-cloud estates only, evaluates five fixed controls, ranks findings by exploitability and impact "
@@ -230,6 +244,7 @@ AGENTS = [
title="Disposition records"),
rewrites=[]),
dict(n="06", key="cp1-assessment-critic-agent", ado=3214464, display="CP1 - Assessment Critic & Sign-off Router Agent",
hooks="app.business_logic", # K-04 to K-07, K-11 and the verdict in code (ADR 0011); app/ and tests/ are kept
desc="Independently validates the Assessment: coverage, evidence, dependents, recalculated scores, confidence "
"and TCO, the interim and advisory rules, and routes confidence below 0.80 to disposition sign-off.",
mcp={},
@@ -339,6 +354,11 @@ ALZ-DSC-06 documents with at least one record; `rate_card_present` needs a non-e
"""
CRITIC_EXTRA = """
`checked_in_code` holds what code found for K-04, K-05, K-06, K-07 and K-11 from the producers' records, before you
were called (ADR 0011). Code sets those checks, the sign-off queue and the verdict in your reply after you answer, so
spend your effort on the checks only you can make: coverage, evidence, dependents, advisory security, telemetry,
tool calls and model independence. Where you read a code check differently, say why in its details.
If `preconditions.failed` is not empty, the planner refused to start: the later steps ran on empty inputs. Return
`verdict: fail`, with a K-01 detail naming each failed precondition, and run no other check.
"""
@@ -403,6 +423,15 @@ def recalc_rules():
return "\n\n".join(parts) + "\n"
def dependents_rule():
"""ALZ-ASM-03's Application Dependents section, for ALZ-ASM-04: retire and the scoring note use the same lift."""
body = body_of(next(a for a in AGENTS if a["n"] == "03"))
text = section(body, "#### Application Dependents", lambda l: l.startswith("#### ") or l.startswith("### ") or l.startswith("## "))
assert text and "represents" in text
return ("## Application dependents\n\nThe dependents and dependencies of an application are lifted from ALZ-DSC-06 "
"by the same rule ALZ-ASM-03 uses (F02 ADR 0012):\n\n" + text + "\n")
def prompt_md(a):
body = body_of(a)
bad = [l for l in body.splitlines() if FORBIDDEN.search(l)]
@@ -415,7 +444,7 @@ def prompt_md(a):
tools, rules = "None. This agent calls no tool; any tool call is refused by the platform.", ""
steps = "\n".join(f"{i}. {s}" for i, s in enumerate(a["steps"], 1))
fields = ", ".join(f"`{k}`" for k in a["inputs"])
extra = {"01": PLANNER_EXTRA, "06": CRITIC_EXTRA + "\n" + recalc_rules()}.get(a["n"], "")
extra = {"01": PLANNER_EXTRA, "04": dependents_rule(), "06": CRITIC_EXTRA + "\n" + recalc_rules()}.get(a["n"], "")
return f"""# {a['display']}
ALZ-ASM-{a['n']} (ADO {a['ado']}), a step of the workflow "CP1 - Agentic Landing Zone - Assessment". The workflow
@@ -672,9 +701,9 @@ cases (it needs `kyndryl-agent-runtime[eval]`). Documentation for people is in [
"""
ADRS_FOR = {"01": ["0001", "0002", "0003", "0008", "0009"], "02": ["0001", "0002", "0007"], "03": ["0001", "0002"],
"05": ["0001", "0002", "0007"], "04": ["0001", "0002", "0003", "0009"],
"06": ["0001", "0002", "0004", "0005", "0006", "0009", "0010"]}
ADRS_FOR = {"01": ["0001", "0002", "0003", "0008", "0009"], "02": ["0001", "0002", "0007"], "03": ["0001", "0002", "0012"],
"05": ["0001", "0002", "0007"], "04": ["0001", "0002", "0003", "0009", "0012"],
"06": ["0001", "0002", "0004", "0005", "0006", "0009", "0010", "0011", "0012"]}
USER_GUIDE = """# User guide: {display}
@@ -773,6 +802,16 @@ def main():
missing = [a["n"] for a in AGENTS if not list(STORIES.glob(f"ALZ-ASM-{a['n']}-*"))] if STORIES.is_dir() else ["all"]
if missing:
sys.exit(f"no story files for ALZ-ASM-{', '.join(missing)} in {STORIES}: nothing was changed")
# An agent with hooks keeps its hand-written app/ and tests/ (ADR 0011): everything else is regenerated.
import tempfile
kept = Path(tempfile.mkdtemp(prefix="scaffold-kept-"))
for a in AGENTS:
for sub_dir in ("app", "tests") if a.get("hooks") else ():
if (OUT / a["key"] / sub_dir).is_dir():
shutil.copytree(OUT / a["key"] / sub_dir, kept / a["key"] / sub_dir,
ignore=shutil.ignore_patterns("__pycache__"))
if a.get("hooks") and not (kept / a["key"] / "app").is_dir():
sys.exit(f"{a['key']} declares hooks but has no app/ to keep: nothing was changed")
for a in AGENTS:
shutil.rmtree(OUT / a["key"], ignore_errors=True)
all_cases = cases()
@@ -785,6 +824,8 @@ def main():
"framework": "adk", "tags": ["cp1", "public-cloud", "assessment"],
"capabilities": {"streaming": False, "push_notifications": False}, "prompt": "prompt.md",
"contract": {"input": "schemas/input.json", "output": "schemas/output.json"}}
if a.get("hooks"):
manifest["hooks"] = a["hooks"]
if a["mcp"]:
manifest["mcp"] = a["mcp"]
manifest["skills"] = [skill]
@@ -801,7 +842,11 @@ def main():
f"# Eval cases for {a['display']}: the Meridian Retail Group sample Discovery outputs.\n"
+ yaml_dump(cases_doc))
write(d / "Dockerfile", f"ARG RUNTIME_IMAGE={RUNTIME}\nFROM ${{RUNTIME_IMAGE}}\n"
"COPY agent.yaml prompt.md /agent/\nCOPY schemas/ /agent/schemas/\nCOPY eval/ /agent/eval/\n")
"COPY agent.yaml prompt.md /agent/\nCOPY schemas/ /agent/schemas/\nCOPY eval/ /agent/eval/\n"
+ ("COPY app/ /agent/app/\n" if a.get("hooks") else ""))
for sub_dir in ("app", "tests"):
if (kept / a["key"] / sub_dir).is_dir():
shutil.copytree(kept / a["key"] / sub_dir, d / sub_dir)
write(d / "RELEASE_NOTES.md", f"## [0.1.0] - {DATE}\n\n### Added\n\n- First release: {a['desc']}\n")
write(d / "README.md", readme(a, model))
shutil.copytree(REF_K8S, d / "kubernetes")
@@ -826,6 +871,7 @@ def main():
tools = ", ".join(f"`{s}.{t}`" for s, ts in a["mcp"].items() for t in ts) or "none"
write(d / ".agents" / "CONTEXT.md", AGENT_CONTEXT.format(display=a["display"], n=a["n"], ado=a["ado"],
step=step, froms=froms, tools=tools, model=model))
shutil.rmtree(kept, ignore_errors=True)
print("generated", len(AGENTS), "agents in", OUT)