{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://kyndryl.com/agent-runtime/eval/schemas/cases-1.1.schema.json", "title": "Agent eval suite (eval/cases.yaml), schema version 1.1", "description": "The cases an agent is evaluated against. The prose reference is the eval case format appendix in the platform's developer guide. This schema gives editors field names, allowed values and inline errors; `python -m kyndryl_agent_runtime.eval --validate` runs the loader itself, which also catches what a schema cannot express (duplicate case ids, two checks in one entry, missing fixture files).", "type": "object", "additionalProperties": false, "required": ["cases"], "properties": { "schema_version": { "enum": ["1.1", 1.1, "1.0", 1.0], "description": "Which version of this format the file is written in. Write 1.1. 1.0 is retired: it was 1.1 without the suite's `runtime` section, so a file that still says 1.0, or says nothing, is read as 1.1." }, "agent": { "type": "string", "minLength": 1, "description": "The agent's name as its agent card publishes it. Labels the results and the baseline; it is not compared with the card. Defaults to the agent directory's name." }, "samples": { "type": "integer", "minimum": 1, "default": 3, "description": "Runs per case. 1 for a fast local loop, 3 by default, 5+ when chasing something intermittent." }, "defaults": { "type": "object", "additionalProperties": false, "description": "Applied to every case.", "properties": { "timeout": {"type": "number", "exclusiveMinimum": 0, "description": "Seconds to wait for one reply."}, "threshold": {"$ref": "#/$defs/score", "description": "Minimum score for judged metrics that do not set their own."} } }, "cases": { "type": "array", "minItems": 1, "description": "The cases, at least one. Each id must be unique in the file.", "items": {"$ref": "#/$defs/case"} }, "runtime": { "type": "object", "additionalProperties": false, "minProperties": 1, "description": "Runtime evaluation: what is scored on the agent's real answers, after a live workflow step finishes. The same checks and judged metrics as a case, applied to what the agent was really asked and really answered. A live step has no expected answer and no fixtures, so `found_in`, the `code_*` checks, `semantic_similarity`, `tool_correctness` and the conversation metrics cannot be used here, and `schema` must be `schema: true`. Scoring never changes, delays or stops the step.", "properties": { "checks": {"type": "array", "items": {"$ref": "#/$defs/check"}, "description": "Deterministic assertions on the real answer. Free: run on every scored step."}, "metrics": {"type": "array", "items": {"$ref": "#/$defs/metric"}, "description": "Judged metrics on the real answer, with the step's own input (and any tool results) as the source of truth. Run on a share of steps, when the platform has judging on, and only after every check passed."} } } }, "$defs": { "score": {"type": "number", "minimum": 0, "maximum": 1}, "bounds": { "description": "An exact number, or a range: {min: 1, max: 5}. Either end may be left out.", "oneOf": [ {"type": "number"}, { "type": "object", "additionalProperties": false, "minProperties": 1, "properties": {"min": {"type": "number"}, "max": {"type": "number"}} } ] }, "encoding": {"enum": ["base64", "text", "none"]}, "fixture": { "description": "A file or folder read in at run time instead of pasted into the case. Paths are relative to the agent directory.", "type": "object", "additionalProperties": false, "properties": { "from_file": {"type": "string", "description": "The file's contents."}, "from_dir": {"type": "string", "description": "A list of {name, content}, one per matching file."}, "zip_of": {"type": "string", "description": "The matching files zipped and base64 encoded. Prefer this over committing a .zip."}, "glob": {"type": "string", "default": "*", "description": "Which files, for from_dir and zip_of."}, "encoding": {"$ref": "#/$defs/encoding", "description": "text (default) or base64; zip_of defaults to base64."} }, "oneOf": [ {"required": ["from_file"]}, {"required": ["from_dir"]}, {"required": ["zip_of"]} ] }, "input": { "description": "The message sent to the agent: a string, {json: ...} (serialized and sent; any value inside may be a fixture such as {from_file: fixtures/x.md}), or {text: ...}.", "oneOf": [ {"type": "string"}, { "type": "object", "additionalProperties": false, "required": ["json"], "properties": {"json": {"description": "Any JSON value. Nested {from_file|from_dir|zip_of: ...} mappings are replaced by their contents."}} }, { "type": "object", "additionalProperties": false, "required": ["text"], "properties": {"text": {"oneOf": [{"type": "string"}, {"$ref": "#/$defs/fixture"}]}} } ] }, "not_implemented": { "description": "Accepted by the format, not yet implemented by the engine: the case loads, and the check reports an error when run." }, "check": { "description": "One deterministic assertion. Exactly one check per entry (a `selector` may pair with `count` or `has_attribute`). Without `field`, `selector` or `artifact`, it applies to the whole reply.", "type": "object", "additionalProperties": false, "properties": { "field": {"type": "string", "description": "Dotted path into a JSON reply: answer, usage.tokens, quotes[0]. The most important key: an unscoped check searches every field."}, "selector": {"type": "string", "description": "CSS selector into an HTML reply or artifact."}, "artifact": {"type": ["integer", "string"], "minimum": 0, "description": "Which A2A artifact: by position (0, the usual case) or by name."}, "match": { "description": "For found_in: how closely the value must appear in the source.", "oneOf": [ {"enum": ["verbatim", "normalized"]}, {"type": "object", "additionalProperties": false, "required": ["fuzzy"], "properties": {"fuzzy": {"$ref": "#/$defs/score"}}} ] }, "encoding": {"$ref": "#/$defs/encoding"}, "considers": {"type": "array", "items": {"type": "string"}}, "contains": {"type": ["string", "number"], "description": "Substring present, case-insensitive."}, "not_contains": {"type": ["string", "number"], "description": "Substring absent, case-insensitive."}, "contains_any": {"type": ["array", "string"], "items": {"type": ["string", "number"]}, "description": "At least one present."}, "contains_all": {"type": ["array", "string"], "items": {"type": ["string", "number"]}, "description": "Every one present."}, "matches": {"type": "string", "description": "Regular expression that must match."}, "not_matches": {"type": "string", "description": "Regular expression that must not match."}, "is_json": {"type": "boolean", "description": "The reply parses as JSON."}, "schema": {"type": ["boolean", "string"], "description": "true: validate against the agent's declared output schema (SKIPPED if it declares none). A path: validate against that pinned schema file."}, "equals": {"description": "Exact value, any JSON type."}, "not_empty": {"type": "boolean", "description": "Present, non-null, non-empty."}, "length": {"$ref": "#/$defs/bounds", "description": "Length of a string or array."}, "between": {"$ref": "#/$defs/bounds", "description": "A number within bounds."}, "found_in": {"type": "string", "description": "Every value of the field appears in this file (see `match`). Only for fields the agent was told to quote verbatim."}, "artifact_count": {"$ref": "#/$defs/bounds", "description": "How many artifacts the reply returned: a task's artifacts, or a message reply's text parts. An empty reply has none, so {min: 1} fails for it."}, "html_parses": {"type": "boolean", "description": "Well-formed enough to build a tree."}, "html_no_script": {"type": "boolean", "description": "No