- Six declarative agents generated from the v1.1 stories (scaffold_from_stories.py, ADR 0001),
and a 9-step workflow export (build_workflow.py).
- Code-only intake (ADR 0008) verifies the one Discovery package in code before any agent.
- The rate card is the run input rate_card_document; no upload step (ADR 0009).
- Critic 0.1.6 after live runs on crucible-kab-dev:
- K-12 model independence checked at build time against each agent's deployed model
(ADR 0010); the critic reports it from its two inputs;
- receives in_cloud_estate for K-10; an unverified K-10 never fails the run alone;
- K-08 allows the scoring note to list findings as advisory, as ALZ-ASM-04 must;
- carries ALZ-ASM-03's and ALZ-ASM-04's rule sections, copied by the scaffold, so it can
recalculate risk, confidence and TCO.
- The scaffold refuses to regenerate unless every story file is present.
- Case azure-crucible-kab-dev-assessment with an illustrative rate card.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
52 lines
6.3 KiB
YAML
52 lines
6.3 KiB
YAML
# Eval cases for CP1 - Security Posture Assessment Agent: the Meridian Retail Group sample Discovery outputs.
|
|
schema_version: "1.0"
|
|
agent: "cp1-security-posture-assessment-agent"
|
|
defaults:
|
|
timeout: 330
|
|
cases:
|
|
- id: "not-applicable-on-premises"
|
|
description: "The Meridian estate is on-premises: not applicable, with no tool call."
|
|
input:
|
|
json:
|
|
engagement_id: "ENG-MERIDIAN-001"
|
|
run_id: "RUN-ASM-EVAL-01"
|
|
discovery_execution_id: "exec_0d5c0e2a7f01"
|
|
discovery_documents: "{\"records\": [{\"canonical_id\": \"CI-SE-001\", \"scope_entry_id\": \"SE-001\", \"hostname\": \"web-01\", \"asset_class\": \"server\", \"environment\": \"production\", \"os\": {\"family\": \"RHEL\", \"version\": \"7.9\"}, \"services\": [\"nginx 1.20\", \"Storefront (Node.js 16)\"], \"facts\": {\"owner\": \"Digital Channels\", \"site\": \"LON-DC1\", \"business_service\": \"Online Storefront\", \"compliance_scope\": [], \"annual_cost\": 18400, \"support_status\": \"extended_support\", \"criticality\": \"high\"}, \"sources\": [{\"source_type\": \"scan\", \"source_ref\": \"SE-001\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 0.9}, {\"canonical_id\": \"CI-SE-002\", \"scope_entry_id\": \"SE-002\", \"hostname\": \"app-01\", \"asset_class\": \"server\", \"environment\": \"production\", \"os\": {\"family\": \"Windows Server\", \"version\": \"2012 R2\"}, \"services\": [\"IIS 8.5\", \"POS Backoffice (.NET Framework 4.8)\"], \"facts\": {\"owner\": \"Retail Systems\", \"site\": \"LON-DC1\", \"business_service\": \"POS Backoffice\", \"compliance_scope\": [\"PCI-DSS\"], \"annual_cost\": 42600, \"support_status\": \"out_of_support\", \"end_of_support\": \"2023-10-10\", \"criticality\": \"critical\"}, \"sources\": [{\"source_type\": \"cmdb\", \"source_ref\": \"CONN-CMDB-SNOW:CI000102\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 1.0}, {\"canonical_id\": \"CI-SE-003\", \"scope_entry_id\": \"SE-003\", \"hostname\": \"pos-db-01\", \"asset_class\": \"database\", \"environment\": \"production\", \"os\": {\"family\": \"Oracle Linux\", \"version\": \"7.9\"}, \"services\": [\"Oracle Database 19c\"], \"facts\": {\"owner\": \"Retail Systems\", \"site\": \"LON-DC1\", \"business_service\": \"POS Backoffice\", \"compliance_scope\": [\"PCI-DSS\"], \"annual_cost\": 96800, \"support_status\": \"supported\", \"criticality\": \"critical\"}, \"sources\": [{\"source_type\": \"cmdb\", \"source_ref\": \"CONN-CMDB-SNOW:CI000103\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 1.0}, {\"canonical_id\": \"CI-SE-032\", \"scope_entry_id\": \"SE-032\", \"hostname\": \"legacy-ftp-01\", \"asset_class\": \"server\", \"environment\": \"unknown\", \"os\": {\"family\": \"CentOS\", \"version\": \"6.10\"}, \"services\": [\"vsftpd 2.2\"], \"facts\": {\"compliance_scope\": [], \"support_status\": \"out_of_support\"}, \"sources\": [{\"source_type\": \"scan\", \"source_ref\": \"SE-032\", \"evidence_ref\": null}], \"conflicts\": [], \"confidence\": 0.85}], \"sme_review_queue\": [\"CI-SE-032\"]}\n\n---\n\n{\"report_id\": \"DR-1\", \"engagement_id\": \"ENG-MERIDIAN-001\", \"drift_items\": [], \"itsm_signals\": [{\"canonical_id\": \"CI-SE-001\", \"window_months\": 12, \"incidents_total\": 1, \"incidents_p1_p2\": 1, \"changes_total\": 1, \"changes_failed\": 1, \"active_freeze\": {\"active\": false, \"until\": null, \"change_ref\": null}}, {\"canonical_id\": \"CI-SE-002\", \"window_months\": 12, \"incidents_total\": 7, \"incidents_p1_p2\": 2, \"changes_total\": 4, \"changes_failed\": 1, \"active_freeze\": {\"active\": true, \"until\": \"2026-12-31\", \"change_ref\": \"CHG0040112\"}}, {\"canonical_id\": \"CI-SE-003\", \"window_months\": 12, \"incidents_total\": 2, \"incidents_p1_p2\": 0, \"changes_total\": 3, \"changes_failed\": 0, \"active_freeze\": {\"active\": true, \"until\": \"2026-12-31\", \"change_ref\": \"CHG0040112\"}}], \"summary\": {\"matched\": 3, \"orphans\": 0, \"ghosts\": 0, \"stale_attributes\": 0}}\n\n---\n\n{\"stores\": [{\"canonical_id\": \"CI-SE-003\", \"store_name\": \"POSPROD\", \"highest_class\": \"PCI\", \"class_counts\": {\"PII\": 4, \"PCI\": 2, \"PHI\": 0, \"review_required\": 0}, \"requires_disposition_controls\": true, \"reason\": \"Card tokens and customer contact details.\"}]}\n\n---\n\n{\"ranges\": [], \"flows\": [], \"findings\": [{\"finding_id\": \"FW-0117\", \"type\": \"unowned_rule\", \"description\": \"any to 198.51.100.0/24 tcp/21, no owner\"}]}\n\n---\n\n{\"graph_id\": \"G-1\", \"engagement_id\": \"ENG-MERIDIAN-001\", \"edges\": [{\"edge_id\": \"E-001\", \"source\": \"CI-SE-001\", \"target\": \"CI-SE-002\", \"type\": \"app_to_app\"}, {\"edge_id\": \"E-002\", \"source\": \"CI-SE-002\", \"target\": \"CI-SE-003\", \"type\": \"app_to_db\"}], \"nodes\": [{\"canonical_id\": \"CI-SE-002\", \"fan_in\": 1, \"fan_out\": 1, \"depended_on\": true, \"retire_constraint\": \"cannot_retire_without_mitigation\"}, {\"canonical_id\": \"CI-SE-032\", \"fan_in\": 0, \"fan_out\": 0, \"depended_on\": false, \"retire_constraint\": \"none\"}], \"orphans\": [], \"validation_queue\": []}\n\n---\n\n{\"coverage_complete\": true, \"verdict\": \"pass\", \"note\": \"Sample verdict.\"}\n\n---\n\n{\"final_html\": \"<p>Discovery report</p>\"}"
|
|
discovery_files:
|
|
- name: "exec_0d5c0e2a7f01_s3"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s4"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s5"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s6"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s7"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s8"
|
|
type: "application/json"
|
|
- name: "exec_0d5c0e2a7f01_s9"
|
|
type: "application/json"
|
|
tasks: []
|
|
applications:
|
|
- application_id: "APP-STOREFRONT"
|
|
name: "Online Storefront"
|
|
assets:
|
|
- "CI-SE-001"
|
|
- application_id: "APP-POS-BACKOFFICE"
|
|
name: "POS Backoffice"
|
|
assets:
|
|
- "CI-SE-002"
|
|
- "CI-SE-003"
|
|
- application_id: "APP-FILE-TRANSFER"
|
|
name: "Legacy file transfer"
|
|
assets:
|
|
- "CI-SE-032"
|
|
in_cloud_estate: false
|
|
checks:
|
|
- schema: true
|
|
- field: "status"
|
|
equals: "not_applicable"
|
|
- field: "findings"
|
|
equals: []
|