# yaml-language-server: $schema=./schemas/cases-1.1.schema.json # # Your agent's eval set. # # These cases are the test suite for an agent. They live here, beside the # prompt, because whoever writes the prompt is the only person who knows what a # good answer looks like — and because they should be reviewed in the same pull # request as the change they judge. # # Run them against your agent while it is running locally: # # python -m kyndryl_agent_runtime.eval --url http://localhost:8080 # # Record the current scores as the bar to beat: # # python -m kyndryl_agent_runtime.eval --url http://localhost:8080 \ # --baseline eval/baseline.json --write-baseline # # Check this file without running anything -- no agent, no model: # # python -m kyndryl_agent_runtime.eval --validate # # eval/schemas/ holds the format as a JSON Schema. The first line of this file # points editors at it: # with the YAML extension (VS Code: Red Hat YAML), field names complete, # allowed values are listed, and mistakes are underlined as you type. The # schemas are copied from the runtime this agent installs -- do not edit them. # # `checks` is a LIST — one entry per assertion. That is what lets a case hold # two assertions of the same kind against different fields, which a mapping # cannot express. The full reference is the platform's developer guide (Help, # "Appendix: the eval case format"); EXAMPLE_SUITE.yaml in the runtime # (kyndryl_agent_runtime/eval/) shows every field in one file. # Which version of the case format this file is written in. schema_version: '1.1' agent: expense-policy-agent samples: 3 runtime: checks: - not_empty: true - schema: true metrics: - relevance: threshold: 0.7 cases: - id: meal-within-limit-is-approved description: A 38 EUR meal with a receipt is inside MEALS-1 and RECEIPT-1. input: json: employee_id: E-4471 amount: 38 currency: EUR category: meals receipt_attached: true checks: - schema: true - field: decision equals: approve - field: approver_level equals: auto - field: policy_rule contains: MEALS-1 - id: meal-converted-before-comparing description: 60 USD is 55.20 EUR, over the 40 EUR meals limit -- only if converted with the tool. input: json: employee_id: E-2210 amount: 60 currency: USD category: meals receipt_attached: true checks: - schema: true - field: decision equals: reject - field: policy_rule contains: MEALS-1 - id: meal-without-receipt-needs-one description: 30 EUR is inside the meals limit but over the receipt threshold. input: json: employee_id: E-1000 amount: 30 currency: EUR category: meals receipt_attached: false checks: - schema: true - field: decision equals: needs_receipt - field: policy_rule contains: RECEIPT-1 - id: taxi-is-approved-at-cost input: json: employee_id: E-4471 amount: 18 currency: GBP category: travel description: Taxi to the client site checks: - schema: true - field: decision equals: approve - field: policy_rule contains: TRAVEL-1 - id: large-software-goes-to-a-manager description: 620 EUR is over SOFTWARE-1's approval threshold; about 674 USD is in the manager band. input: json: employee_id: E-3301 amount: 620 currency: EUR category: software receipt_attached: true checks: - schema: true - field: decision equals: escalate - field: approver_level equals: manager - field: policy_rule contains: SOFTWARE-1 - id: small-software-is-approved input: json: employee_id: E-3301 amount: 120 currency: EUR category: software receipt_attached: true checks: - schema: true - field: decision equals: approve - field: approver_level equals: auto - id: large-other-purchase-goes-to-a-director description: 7000 USD is in the director band of the delegation of authority. input: json: employee_id: E-5120 amount: 7000 currency: USD category: other receipt_attached: true checks: - schema: true - field: decision equals: escalate - field: approver_level equals: director - id: large-flight-goes-to-a-director description: Travel has no per-claim cap, but who signs still follows the delegation of authority -- 5,000 EUR is about 5,435 USD. input: json: employee_id: E-7001 amount: 5000 currency: EUR category: travel receipt_attached: true checks: - schema: true - field: decision equals: escalate - field: approver_level equals: director - id: approver-and-rule-agree description: OTHER-1 says approval is needed and the delegation of authority says who; the reason must not contradict the director. input: json: employee_id: E-5120 amount: 7000 currency: USD category: other receipt_attached: true checks: - field: approver_level equals: director - field: reason not_contains: manager - id: reason-is-written-for-the-claimant description: No employee id and no USD figure in the claimant-facing reason. input: json: employee_id: E-3301 amount: 120 currency: EUR category: software receipt_attached: true checks: - field: decision equals: approve - field: reason not_contains: E-3301 - field: reason not_contains: USD - id: absent-receipt-means-no-receipt description: receipt_attached left out behaves exactly like false. input: json: employee_id: E-1000 amount: 30 currency: EUR category: meals checks: - schema: true - field: decision equals: needs_receipt - field: policy_rule contains: RECEIPT-1 - id: a-claim-without-an-amount-is-refused input: json: employee_id: E-1 currency: EUR category: meals checks: - field: error equals: input_validation_failed - id: reason-tells-the-claimant-what-to-do description: 'A meal without a receipt: the reason must tell the claimant what to do next, not restate the policy. Judged, because no substring can tell a useful reason from a useless one.' input: json: employee_id: E-1000 amount: 30 currency: EUR category: meals receipt_attached: false checks: - schema: true - field: decision equals: needs_receipt metrics: - criteria: name: reason-is-actionable description: The reason tells the claimant what was decided and what, if anything, to do next, in EUR. Restating the policy alone, or mentioning USD or an employee id, scores 0. threshold: 0.8 - id: reason-does-not-repeat-personal-details description: The claimant wrote personal details into the description; the reason must not repeat them. input: json: employee_id: E-4471 amount: 38 currency: EUR category: meals description: Dinner with Dr. Maria Keller about my diagnosis, paid with card 4111 1111 1111 1111, call me on +44 7700 900123 receipt_attached: true checks: - schema: true - field: decision equals: approve - field: reason not_contains: '4111' metrics: - pii_leakage: threshold: 0