Files
Jonathan Boniface e22d037ab0 ALZ-F02 Assessment: individual agents, intake, rate card input, critic fixes
- Six declarative agents generated from the v1.1 stories (scaffold_from_stories.py, ADR 0001),
  and a 9-step workflow export (build_workflow.py).
- Code-only intake (ADR 0008) verifies the one Discovery package in code before any agent.
- The rate card is the run input rate_card_document; no upload step (ADR 0009).
- Critic 0.1.6 after live runs on crucible-kab-dev:
  - K-12 model independence checked at build time against each agent's deployed model
    (ADR 0010); the critic reports it from its two inputs;
  - receives in_cloud_estate for K-10; an unverified K-10 never fails the run alone;
  - K-08 allows the scoring note to list findings as advisory, as ALZ-ASM-04 must;
  - carries ALZ-ASM-03's and ALZ-ASM-04's rule sections, copied by the scaffold, so it can
    recalculate risk, confidence and TCO.
- The scaffold refuses to regenerate unless every story file is present.
- Case azure-crucible-kab-dev-assessment with an illustrative rate card.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

72 lines
7.8 KiB
YAML

# Eval cases for CP1 - 7R Disposition & TCO Agent: the Meridian Retail Group sample Discovery outputs.
schema_version: "1.0"
agent: "cp1-disposition-tco-agent"
defaults:
timeout: 330
cases:
- id: "low-confidence-retire-goes-to-signoff"
description: "The file-transfer host has no owner, no criticality and an open FTP rule, so any disposition for it is below 0.80 and queued for sign-off."
input:
json:
engagement_id: "ENG-MERIDIAN-001"
run_id: "RUN-ASM-EVAL-01"
discovery_execution_id: "exec_0d5c0e2a7f01"
discovery_documents: "{\"records\": [{\"canonical_id\": \"CI-SE-001\", \"scope_entry_id\": \"SE-001\", \"hostname\": \"web-01\", \"asset_class\": \"server\", \"environment\": \"production\", \"os\": {\"family\": \"RHEL\", \"version\": \"7.9\"}, \"services\": [\"nginx 1.20\", \"Storefront (Node.js 16)\"], \"facts\": {\"owner\": \"Digital Channels\", \"site\": \"LON-DC1\", \"business_service\": \"Online Storefront\", \"compliance_scope\": [], \"annual_cost\": 18400, \"support_status\": \"extended_support\", \"criticality\": \"high\"}, \"sources\": [{\"source_type\": \"scan\", \"source_ref\": \"SE-001\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 0.9}, {\"canonical_id\": \"CI-SE-002\", \"scope_entry_id\": \"SE-002\", \"hostname\": \"app-01\", \"asset_class\": \"server\", \"environment\": \"production\", \"os\": {\"family\": \"Windows Server\", \"version\": \"2012 R2\"}, \"services\": [\"IIS 8.5\", \"POS Backoffice (.NET Framework 4.8)\"], \"facts\": {\"owner\": \"Retail Systems\", \"site\": \"LON-DC1\", \"business_service\": \"POS Backoffice\", \"compliance_scope\": [\"PCI-DSS\"], \"annual_cost\": 42600, \"support_status\": \"out_of_support\", \"end_of_support\": \"2023-10-10\", \"criticality\": \"critical\"}, \"sources\": [{\"source_type\": \"cmdb\", \"source_ref\": \"CONN-CMDB-SNOW:CI000102\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 1.0}, {\"canonical_id\": \"CI-SE-003\", \"scope_entry_id\": \"SE-003\", \"hostname\": \"pos-db-01\", \"asset_class\": \"database\", \"environment\": \"production\", \"os\": {\"family\": \"Oracle Linux\", \"version\": \"7.9\"}, \"services\": [\"Oracle Database 19c\"], \"facts\": {\"owner\": \"Retail Systems\", \"site\": \"LON-DC1\", \"business_service\": \"POS Backoffice\", \"compliance_scope\": [\"PCI-DSS\"], \"annual_cost\": 96800, \"support_status\": \"supported\", \"criticality\": \"critical\"}, \"sources\": [{\"source_type\": \"cmdb\", \"source_ref\": \"CONN-CMDB-SNOW:CI000103\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 1.0}, {\"canonical_id\": \"CI-SE-032\", \"scope_entry_id\": \"SE-032\", \"hostname\": \"legacy-ftp-01\", \"asset_class\": \"server\", \"environment\": \"unknown\", \"os\": {\"family\": \"CentOS\", \"version\": \"6.10\"}, \"services\": [\"vsftpd 2.2\"], \"facts\": {\"compliance_scope\": [], \"support_status\": \"out_of_support\"}, \"sources\": [{\"source_type\": \"scan\", \"source_ref\": \"SE-032\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 0.85}], \"sme_review_queue\": [\"CI-SE-032\"]}\n\n---\n\n{\"report_id\": \"DR-1\", \"engagement_id\": \"ENG-MERIDIAN-001\", \"drift_items\": [], \"itsm_signals\": [{\"canonical_id\": \"CI-SE-001\", \"window_months\": 12, \"incidents_total\": 1, \"incidents_p1_p2\": 1, \"changes_total\": 1, \"changes_failed\": 1, \"active_freeze\": {\"active\": false, \"until\": null, \"change_ref\": null}}, {\"canonical_id\": \"CI-SE-002\", \"window_months\": 12, \"incidents_total\": 7, \"incidents_p1_p2\": 2, \"changes_total\": 4, \"changes_failed\": 1, \"active_freeze\": {\"active\": true, \"until\": \"2026-12-31\", \"change_ref\": \"CHG0040112\"}}, {\"canonical_id\": \"CI-SE-003\", \"window_months\": 12, \"incidents_total\": 2, \"incidents_p1_p2\": 0, \"changes_total\": 3, \"changes_failed\": 0, \"active_freeze\": {\"active\": true, \"until\": \"2026-12-31\", \"change_ref\": \"CHG0040112\"}}], \"summary\": {\"matched\": 3, \"orphans\": 0, \"ghosts\": 0, \"stale_attributes\": 0}}\n\n---\n\n{\"stores\": [{\"canonical_id\": \"CI-SE-003\", \"store_name\": \"POSPROD\", \"highest_class\": \"PCI\", \"class_counts\": {\"PII\": 4, \"PCI\": 2, \"PHI\": 0, \"review_required\": 0}, \"requires_disposition_controls\": true, \"reason\": \"Card tokens and customer contact details.\"}]}\n\n---\n\n{\"ranges\": [], \"flows\": [], \"findings\": [{\"finding_id\": \"FW-0117\", \"type\": \"unowned_rule\", \"description\": \"any to 198.51.100.0/24 tcp/21, no owner\"}]}\n\n---\n\n{\"graph_id\": \"G-1\", \"engagement_id\": \"ENG-MERIDIAN-001\", \"edges\": [{\"edge_id\": \"E-001\", \"source\": \"CI-SE-001\", \"target\": \"CI-SE-002\", \"type\": \"app_to_app\"}, {\"edge_id\": \"E-002\", \"source\": \"CI-SE-002\", \"target\": \"CI-SE-003\", \"type\": \"app_to_db\"}], \"nodes\": [{\"canonical_id\": \"CI-SE-002\", \"fan_in\": 1, \"fan_out\": 1, \"depended_on\": true, \"retire_constraint\": \"cannot_retire_without_mitigation\"}, {\"canonical_id\": \"CI-SE-032\", \"fan_in\": 0, \"fan_out\": 0, \"depended_on\": false, \"retire_constraint\": \"none\"}], \"orphans\": [], \"validation_queue\": []}\n\n---\n\n{\"coverage_complete\": true, \"verdict\": \"pass\", \"note\": \"Sample verdict.\"}\n\n---\n\n{\"final_html\": \"<p>Discovery report</p>\"}"
discovery_files:
- name: "exec_0d5c0e2a7f01_s3"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s4"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s5"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s6"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s7"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s8"
type: "application/json"
- name: "exec_0d5c0e2a7f01_s9"
type: "application/json"
tasks:
- task_id: "T-001"
type: "score_disposition"
assigned_agent: "ALZ-ASM-04"
application_id: "APP-STOREFRONT"
depends_on: []
- task_id: "T-002"
type: "score_disposition"
assigned_agent: "ALZ-ASM-04"
application_id: "APP-POS-BACKOFFICE"
depends_on: []
- task_id: "T-003"
type: "score_disposition"
assigned_agent: "ALZ-ASM-04"
application_id: "APP-FILE-TRANSFER"
depends_on: []
applications:
- application_id: "APP-STOREFRONT"
name: "Online Storefront"
assets:
- "CI-SE-001"
- application_id: "APP-POS-BACKOFFICE"
name: "POS Backoffice"
assets:
- "CI-SE-002"
- "CI-SE-003"
- application_id: "APP-FILE-TRANSFER"
name: "Legacy file transfer"
assets:
- "CI-SE-032"
rate_card_id: "RC-MERIDIAN-2026-11"
rate_card_document: "{\"rate_card_id\": \"RC-MERIDIAN-2026-11\", \"currency\": \"GBP\", \"items\": [{\"rate_id\": \"RC-02\", \"category\": \"managed_app\", \"description\": \"Managed app service, Windows / .NET\", \"annual_rate\": 4800, \"equivalent_for\": [\"IIS\", \".NET Framework\"], \"licence_terms\": \"Windows licence included\"}, {\"rate_id\": \"RC-03\", \"category\": \"managed_db\", \"description\": \"Managed Oracle-compatible database\", \"annual_rate\": 38000, \"equivalent_for\": [\"Oracle Database\"], \"licence_terms\": \"Bring your own Oracle licence\"}, {\"rate_id\": \"RC-08\", \"category\": \"managed_app\", \"description\": \"Managed container app\", \"annual_rate\": 3000, \"equivalent_for\": [\"Node.js\", \"nginx\"], \"licence_terms\": null}]}"
profiles: []
scores: []
high_risk_tail:
- "APP-POS-BACKOFFICE"
checks:
- schema: true
- field: "signoff_queue"
contains: "APP-FILE-TRANSFER"
- field: "dispositions"
contains: "discovery:exec_0d5c0e2a7f01/ALZ-DSC-03"