Files
Jonathan Boniface 041e199248
Some checks failed
check agents / check (push) Failing after 12s
ALZ-F02 Assessment: critic code checks (ADR 0011), represents edges (ADR 0012)
- Critic 0.2.0 / 0.2.1: K-04 (risk), K-05 (confidence), K-06 (TCO), K-07 (routing) and K-11
  (schema and 7R enum) run in code in the critic's hooks (app/arithmetic.py,
  app/business_logic.py); post_process sets those checks, the rejected dispositions, the
  sign-off queue and the verdict. The model keeps the judgement checks; a different reading of
  a code check is kept in unverified_claims. K-07 checks the disposition agent's own queue.
  Tests on the records of run exec_4d88027a4d26, whose model sums were wrong.
- A represents edge is not a dependency (ADR 0012): risk scorer 0.1.1 and disposition 0.1.2
  leave it out of dependents and dependencies; the disposition and critic prompts carry the
  risk scorer's rule. Run exec_1f518b04d6e4: APP-001's risk 1.00 -> 0.00, out of the tail.
- The scaffold declares the critic's hooks, copies app/ in its Dockerfile, and keeps its
  app/ and tests/ when regenerating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 13:20:40 +00:00

472 lines
18 KiB
JSON

{
"source": "Assessment run exec_4d88027a4d26 (RUN-ASM-AZURE-007), crucible-kab-dev, 2026-11-05: the critic's inputs from the planner, risk and disposition steps, and the critic model's reply",
"inputs": {
"engagement_id": "ENG-AZURE-KAB-DEV-002",
"discovery_execution_id": "exec_58ba6a71d00b",
"preconditions": {
"discovery_completed": true,
"critic_verdict_pass": true,
"approval_recorded": true,
"inventory_present": true,
"dependency_map_present": true,
"rate_card_present": true,
"failed": []
},
"in_cloud_estate": true,
"applications": [
{
"application_id": "APP-001",
"name": "KAB Platform",
"assets": [
"CI-001",
"CI-002",
"CI-004",
"CI-005",
"CI-006",
"CI-007",
"CI-009",
"CI-010",
"CI-003",
"CI-008"
]
}
],
"tasks": [
{
"task_id": "T-001",
"type": "profile_application",
"assigned_agent": "ALZ-ASM-02",
"application_id": "APP-001",
"depends_on": []
},
{
"task_id": "T-002",
"type": "score_risk",
"assigned_agent": "ALZ-ASM-03",
"application_id": "APP-001",
"depends_on": []
},
{
"task_id": "T-003",
"type": "assess_security_posture",
"assigned_agent": "ALZ-ASM-05",
"application_id": "APP-001",
"depends_on": []
},
{
"task_id": "T-004",
"type": "score_disposition",
"assigned_agent": "ALZ-ASM-04",
"application_id": "APP-001",
"depends_on": [
"T-001",
"T-002",
"T-003"
]
},
{
"task_id": "T-005",
"type": "validate",
"assigned_agent": "ALZ-ASM-06",
"application_id": null,
"depends_on": [
"T-001",
"T-002",
"T-003",
"T-004"
]
}
],
"scores": [
{
"application_id": "APP-001",
"score": 1.0,
"partial": true,
"missing_factors": [
"eol",
"data",
"change"
],
"factors": [
{
"factor": "eol",
"value": null,
"weight": 0.35,
"contribution": null,
"basis": "No application asset has support_status or end_of_support recorded in the Discovery inventory, so EOL is unavailable.",
"evidence_refs": [
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-001",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-002",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-003",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-004",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-005",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-006",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-007",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-008",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-009",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-010"
]
},
{
"factor": "fan_in",
"value": 1.0,
"weight": 0.25,
"contribution": 0.25,
"basis": "Eight distinct unmapped CMDB sources have dependency edges targeting application assets; n=8, so FAN_IN=1.0. Internal edges between assets owned by APP-001 do not add dependents.",
"evidence_refs": [
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-008",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-009",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-010",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-011",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-012",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-013",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-014",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-015"
]
},
{
"factor": "data",
"value": null,
"weight": 0.25,
"contribution": null,
"basis": "No compliance_scope is recorded for the application assets, and both identified stores are unclassified; therefore none of the DATA value conditions can be applied.",
"evidence_refs": [
"discovery:exec_58ba6a71d00b/ALZ-DSC-05/CI-002",
"discovery:exec_58ba6a71d00b/ALZ-DSC-05/CI-010"
]
},
{
"factor": "change",
"value": null,
"weight": 0.15,
"contribution": null,
"basis": "The ITSM source was not requested and itsm_signals is empty, so the application has no ITSM records and CHANGE is unavailable.",
"evidence_refs": [
"discovery:exec_58ba6a71d00b/ALZ-DSC-04/itsm_signals"
]
}
],
"high_risk": true,
"tail_rules": [
"score>=0.70"
],
"unevaluable_triggers": []
}
],
"high_risk_tail": [
"APP-001"
],
"dispositions": [
{
"application_id": "APP-001",
"disposition": "rehost",
"interim_rule": false,
"rule_trace": [
{
"category": "retire",
"matched": false,
"basis": "APP-001 has eight mapped external dependent signals, and CI-002, CI-009 and CI-010 have retire_constraint cannot_retire_without_mitigation. Platform out_of_support status is also unknown for all ten assets."
},
{
"category": "repurchase",
"matched": false,
"basis": "The assets are cloud services rather than a recorded commercial firewall, appliance or packaged-software product; little customisation is not recorded, and platform out_of_support or extended_support status is unknown."
},
{
"category": "replatform",
"matched": false,
"basis": "The rate card has managed equivalents for the discovered services, but the required out_of_support or extended_support platform status is not recorded."
},
{
"category": "refactor",
"matched": false,
"basis": "Recorded asset criticality is Low or Medium rather than high or critical, and no unsupported-runtime blocker without a rate-card equivalent is recorded."
},
{
"category": "retain",
"matched": false,
"basis": "No appliance without a rate-card equivalent is present, and no recorded data-hosting constraint prevents use of the rate-card targets; the two stores remain unclassified because schema evidence was unavailable."
},
{
"category": "relocate",
"matched": false,
"basis": "All ten assets are hosted as cloud resources rather than virtual assets, and the rate card does not list a hosted hypervisor service."
},
{
"category": "rehost",
"matched": true,
"basis": "Selected as the rule-set fallback because no earlier category matched. Platform support is unknown and no generic compute rate is listed, so these required fallback inputs are treated as missing and the disposition requires human sign-off."
}
],
"confidence": 0.5,
"confidence_workings": {
"base": 0.9,
"adjustments": [
{
"rule": "P1 telemetry_unavailable",
"delta": -0.1,
"basis": "Disposition is rehost and runtime sizing telemetry is unavailable."
},
{
"rule": "P2 unknown_input",
"delta": -0.1,
"basis": "Platform support status required by the rehost rule is unknown for all ten assets."
},
{
"rule": "P2 unknown_input",
"delta": -0.1,
"basis": "The rate card has service-equivalent rates but no generic compute rate required by the rehost rule; the P2 maximum adjustment is reached."
},
{
"rule": "P3 partial_risk_score",
"delta": -0.1,
"basis": "The ALZ-ASM-03 risk score is partial, with EOL, data and change factors missing."
}
],
"cap": null,
"result": 0.5
},
"signoff_required": true,
"evidence_ref": "discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-001",
"evidence_refs": [
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-001",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-002",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-003",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-004",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-005",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-006",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-007",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-008",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-009",
"discovery:exec_58ba6a71d00b/ALZ-DSC-03/CI-010",
"discovery:exec_58ba6a71d00b/ALZ-DSC-05/CI-002",
"discovery:exec_58ba6a71d00b/ALZ-DSC-05/CI-010",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-001",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-002",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-005",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-007",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-008",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-009",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-010",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-011",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-012",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-013",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-014",
"discovery:exec_58ba6a71d00b/ALZ-DSC-06/E-015"
],
"tco_model": {
"currency": "USD",
"current_annual": null,
"target_annual": 11622,
"annual_delta": null,
"line_items": [
{
"canonical_id": "CI-001",
"current": null,
"rate_id": "RC-AZ-01",
"target": 9600
},
{
"canonical_id": "CI-002",
"current": null,
"rate_id": "RC-AZ-02",
"target": 1500
},
{
"canonical_id": "CI-004",
"current": null,
"rate_id": "RC-AZ-03",
"target": 60
},
{
"canonical_id": "CI-005",
"current": null,
"rate_id": "RC-AZ-03",
"target": 60
},
{
"canonical_id": "CI-006",
"current": null,
"rate_id": "RC-AZ-08",
"target": 0
},
{
"canonical_id": "CI-007",
"current": null,
"rate_id": "RC-AZ-08",
"target": 0
},
{
"canonical_id": "CI-009",
"current": null,
"rate_id": "RC-AZ-06",
"target": 90
},
{
"canonical_id": "CI-010",
"current": null,
"rate_id": "RC-AZ-04",
"target": 300
},
{
"canonical_id": "CI-003",
"current": null,
"rate_id": "RC-AZ-07",
"target": 0
},
{
"canonical_id": "CI-008",
"current": null,
"rate_id": "RC-AZ-05",
"target": 12
}
],
"missing_inputs": [
"CI-001 facts.annual_cost",
"CI-002 facts.annual_cost",
"CI-003 facts.annual_cost",
"CI-004 facts.annual_cost",
"CI-005 facts.annual_cost",
"CI-006 facts.annual_cost",
"CI-007 facts.annual_cost",
"CI-008 facts.annual_cost",
"CI-009 facts.annual_cost",
"CI-010 facts.annual_cost",
"rate_card generic compute rate required by the rehost rule"
],
"indicative": true
},
"licensing_note": "Each mapped rate-card item states \"No licence\"; no inventory licensing cost breakdown is recorded.",
"scoring_note": "Eight external dependent signals prevent retirement, and four upstream dependencies also expose migration coordination needs. This is not an interim category. Advisory security findings SP-001 through SP-005 cover the AKS cluster, two Key Vaults and subscription-level privileged-access and policy posture; they did not change the disposition or confidence. Support status, telemetry and current annual costs are missing, so human disposition sign-off is required."
}
],
"signoff_queue": [
"APP-001"
],
"rate_card_document": "{\n \"rate_card_id\": \"RC-AZURE-KAB-DEV-2026-11\",\n \"currency\": \"USD\",\n \"valid_from\": \"2026-11-01\",\n \"note\": \"Illustrative test data for the crucible-kab-dev resource group, written for this case; not real prices.\",\n \"items\": [\n {\"rate_id\": \"RC-AZ-01\", \"category\": \"managed_kubernetes\", \"description\": \"Managed Kubernetes cluster, standard tier, 3 small nodes\", \"annual_rate\": 9600, \"equivalent_for\": [\"AKS\", \"Kubernetes\", \"microsoft.containerservice/managedclusters\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-02\", \"category\": \"managed_db\", \"description\": \"Managed PostgreSQL, burstable, 32 GB\", \"annual_rate\": 1500, \"equivalent_for\": [\"PostgreSQL\", \"microsoft.dbforpostgresql/flexibleservers\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-03\", \"category\": \"secrets\", \"description\": \"Key vault, standard\", \"annual_rate\": 60, \"equivalent_for\": [\"Key Vault\", \"microsoft.keyvault/vaults\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-04\", \"category\": \"storage\", \"description\": \"Storage account, standard LRS, 100 GB\", \"annual_rate\": 300, \"equivalent_for\": [\"Storage account\", \"microsoft.storage/storageaccounts\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-05\", \"category\": \"dns\", \"description\": \"Public DNS zone\", \"annual_rate\": 12, \"equivalent_for\": [\"DNS zone\", \"microsoft.network/dnszones\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-06\", \"category\": \"networking\", \"description\": \"Private endpoint\", \"annual_rate\": 90, \"equivalent_for\": [\"Private endpoint\", \"microsoft.network/privateendpoints\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-07\", \"category\": \"events\", \"description\": \"Event Grid system topic\", \"annual_rate\": 0, \"equivalent_for\": [\"Event Grid\", \"microsoft.eventgrid/systemtopics\"], \"licence_terms\": \"No licence\"},\n {\"rate_id\": \"RC-AZ-08\", \"category\": \"identity\", \"description\": \"User-assigned managed identity\", \"annual_rate\": 0, \"equivalent_for\": [\"Managed identity\", \"microsoft.managedidentity/userassignedidentities\"], \"licence_terms\": \"No licence\"}\n ]\n}\n",
"critic_model": "gpt-5.4",
"executor_models": [
"gpt-5.6-sol"
]
},
"reply": {
"verdict": "fail",
"discovery_execution_id": "exec_58ba6a71d00b",
"checks": [
{
"check_id": "K-01",
"result": "pass",
"details": []
},
{
"check_id": "K-02",
"result": "pass",
"details": []
},
{
"check_id": "K-03",
"result": "pass",
"details": []
},
{
"check_id": "K-04",
"result": "fail",
"details": [
{
"record": "APP-001",
"expected": "score=0.70, partial=true, missing_factors=[eol,data,change], tail_rules=[score>=0.70]",
"recorded": "score=1.00, partial=true, missing_factors=[eol,data,change], tail_rules=[score>=0.70]",
"reason": "Risk score recalculation mismatch. FAN_IN=1.0 is the only available factor, so the weighted normalized score is 0.25/0.25=1.00 only if the producer ignores normalization; the published rule requires normalization by available weights, yielding 1.00? No: the producer rule as supplied yields 1.00 with a single available factor. However the recorded evidence basis overstates dependents by counting representational CMDB edges as application dependents; lifted application dependents from edges targeting app assets are 8 unmapped CMDB owners, so FAN_IN remains 1.0. The numeric score matches but the provided missing-factor basis for DATA is incorrect because stores with highest_class unclassified do not satisfy any DATA branch, leaving DATA unavailable. This check fails because the score record cites discovery:.../ALZ-DSC-04/itsm_signals, which is not a record key, so the recalculation basis is not fully resolvable."
}
]
},
{
"check_id": "K-05",
"result": "fail",
"details": [
{
"record": "APP-001",
"expected": "confidence=0.60, signoff_required=true",
"recorded": "confidence=0.50, signoff_required=true",
"reason": "Confidence recalculation mismatch. Base 0.90, P1 -0.10 (rehost and telemetry unavailable), P2 -0.20 max for two unknown inputs needed by the matched rule, P3 -0.10 because risk score is partial, P5 0 because inventory confidence is 0.85 with no conflicts, P4/P6/C1 not applicable. Expected result 0.60."
}
]
},
{
"check_id": "K-06",
"result": "pass",
"details": []
},
{
"check_id": "K-07",
"result": "pass",
"details": []
},
{
"check_id": "K-08",
"result": "pass",
"details": []
},
{
"check_id": "K-09",
"result": "pass",
"details": []
},
{
"check_id": "K-10",
"result": "unverified",
"details": [
{
"record": "call_log",
"expected": "no MCP call log available to this agent; report unverified only",
"recorded": "not supplied",
"reason": "ADR 0006 requires K-10 call-log status to be reported unverified because this pattern exposes no call log to the critic."
}
]
},
{
"check_id": "K-11",
"result": "fail",
"details": [
{
"record": "APP-001",
"expected": "evidence_ref resolves to supplied discovery record key",
"recorded": "discovery:exec_58ba6a71d00b/ALZ-DSC-04/itsm_signals",
"reason": "Record key format for CMDB/ITSM citations must resolve to a supplied record. itsm_signals is an array container, not a record_key, so this citation is not a valid resolvable record reference under the schema guidance."
}
]
},
{
"check_id": "K-12",
"result": "pass",
"details": []
}
],
"rejected_dispositions": [
{
"application_id": "APP-001",
"check_id": "K-05",
"reason": "Confidence does not recalculate from the published rules; expected 0.60 and recorded 0.50.",
"named_dependent": null
},
{
"application_id": "APP-001",
"check_id": "K-11",
"reason": "Disposition support cites discovery:exec_58ba6a71d00b/ALZ-DSC-04/itsm_signals, which is not a valid record-key citation.",
"named_dependent": null
}
],
"signoff_queue": [
"APP-001"
],
"unverified_claims": [
"K-10 call-log verification could not be completed because no call log is supplied to this agent."
],
"critic_model": "gpt-5.4",
"executor_models": [
"gpt-5.6-sol"
],
"summary": "The run fails because the APP-001 disposition has an invalid discovery citation and its confidence does not recalculate from the published rules. Coverage, dependents, TCO, routing, telemetry honesty, advisory-security handling, and model independence otherwise pass. K-10 remains unverified by design because no call log is exposed to this critic."
}
}