Files
cp1-agents/docs/source-material/agent-shapes/agent-pattern/eval/schemas/cases-1.1.schema.json
Jonathan Boniface 416c66ced4
All checks were successful
check agents / check (push) Successful in 10s
Add CP1 Discovery agents in both patterns
ALZ-F01 Discovery as a sub-agent orchestrator (sub-agents/) and as eight
individual declarative agents (individual-agents/), with their KAB workflow
exports. Shared check, render and import scripts, the alz-discovery-sample
MCP server, the crucible-kab-dev deploy profile, deployment docs, platform
issue reports and the source material. Agent steps time out at 20 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 18:25:57 +00:00

290 lines
20 KiB
JSON

{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://kyndryl.com/agent-runtime/eval/schemas/cases-1.1.schema.json",
"title": "Agent eval suite (eval/cases.yaml), schema version 1.1",
"description": "The cases an agent is evaluated against. The prose reference is the eval case format appendix in the platform's developer guide. This schema gives editors field names, allowed values and inline errors; `python -m kyndryl_agent_runtime.eval --validate` runs the loader itself, which also catches what a schema cannot express (duplicate case ids, two checks in one entry, missing fixture files).",
"type": "object",
"additionalProperties": false,
"required": ["cases"],
"properties": {
"schema_version": {
"enum": ["1.1", 1.1, "1.0", 1.0],
"description": "Which version of this format the file is written in. Write 1.1. 1.0 is retired: it was 1.1 without the suite's `runtime` section, so a file that still says 1.0, or says nothing, is read as 1.1."
},
"agent": {
"type": "string",
"minLength": 1,
"description": "The agent's name as its agent card publishes it. Labels the results and the baseline; it is not compared with the card. Defaults to the agent directory's name."
},
"samples": {
"type": "integer",
"minimum": 1,
"default": 3,
"description": "Runs per case. 1 for a fast local loop, 3 by default, 5+ when chasing something intermittent."
},
"defaults": {
"type": "object",
"additionalProperties": false,
"description": "Applied to every case.",
"properties": {
"timeout": {"type": "number", "exclusiveMinimum": 0, "description": "Seconds to wait for one reply."},
"threshold": {"$ref": "#/$defs/score", "description": "Minimum score for judged metrics that do not set their own."}
}
},
"cases": {
"type": "array",
"minItems": 1,
"description": "The cases, at least one. Each id must be unique in the file.",
"items": {"$ref": "#/$defs/case"}
},
"runtime": {
"type": "object",
"additionalProperties": false,
"minProperties": 1,
"description": "Runtime evaluation: what is scored on the agent's real answers, after a live workflow step finishes. The same checks and judged metrics as a case, applied to what the agent was really asked and really answered. A live step has no expected answer and no fixtures, so `found_in`, the `code_*` checks, `semantic_similarity`, `tool_correctness` and the conversation metrics cannot be used here, and `schema` must be `schema: true`. Scoring never changes, delays or stops the step.",
"properties": {
"checks": {"type": "array", "items": {"$ref": "#/$defs/check"}, "description": "Deterministic assertions on the real answer. Free: run on every scored step."},
"metrics": {"type": "array", "items": {"$ref": "#/$defs/metric"}, "description": "Judged metrics on the real answer, with the step's own input (and any tool results) as the source of truth. Run on a share of steps, when the platform has judging on, and only after every check passed."}
}
}
},
"$defs": {
"score": {"type": "number", "minimum": 0, "maximum": 1},
"bounds": {
"description": "An exact number, or a range: {min: 1, max: 5}. Either end may be left out.",
"oneOf": [
{"type": "number"},
{
"type": "object",
"additionalProperties": false,
"minProperties": 1,
"properties": {"min": {"type": "number"}, "max": {"type": "number"}}
}
]
},
"encoding": {"enum": ["base64", "text", "none"]},
"fixture": {
"description": "A file or folder read in at run time instead of pasted into the case. Paths are relative to the agent directory.",
"type": "object",
"additionalProperties": false,
"properties": {
"from_file": {"type": "string", "description": "The file's contents."},
"from_dir": {"type": "string", "description": "A list of {name, content}, one per matching file."},
"zip_of": {"type": "string", "description": "The matching files zipped and base64 encoded. Prefer this over committing a .zip."},
"glob": {"type": "string", "default": "*", "description": "Which files, for from_dir and zip_of."},
"encoding": {"$ref": "#/$defs/encoding", "description": "text (default) or base64; zip_of defaults to base64."}
},
"oneOf": [
{"required": ["from_file"]},
{"required": ["from_dir"]},
{"required": ["zip_of"]}
]
},
"input": {
"description": "The message sent to the agent: a string, {json: ...} (serialized and sent; any value inside may be a fixture such as {from_file: fixtures/x.md}), or {text: ...}.",
"oneOf": [
{"type": "string"},
{
"type": "object",
"additionalProperties": false,
"required": ["json"],
"properties": {"json": {"description": "Any JSON value. Nested {from_file|from_dir|zip_of: ...} mappings are replaced by their contents."}}
},
{
"type": "object",
"additionalProperties": false,
"required": ["text"],
"properties": {"text": {"oneOf": [{"type": "string"}, {"$ref": "#/$defs/fixture"}]}}
}
]
},
"not_implemented": {
"description": "Accepted by the format, not yet implemented by the engine: the case loads, and the check reports an error when run."
},
"check": {
"description": "One deterministic assertion. Exactly one check per entry (a `selector` may pair with `count` or `has_attribute`). Without `field`, `selector` or `artifact`, it applies to the whole reply.",
"type": "object",
"additionalProperties": false,
"properties": {
"field": {"type": "string", "description": "Dotted path into a JSON reply: answer, usage.tokens, quotes[0]. The most important key: an unscoped check searches every field."},
"selector": {"type": "string", "description": "CSS selector into an HTML reply or artifact."},
"artifact": {"type": ["integer", "string"], "minimum": 0, "description": "Which A2A artifact: by position (0, the usual case) or by name."},
"match": {
"description": "For found_in: how closely the value must appear in the source.",
"oneOf": [
{"enum": ["verbatim", "normalized"]},
{"type": "object", "additionalProperties": false, "required": ["fuzzy"], "properties": {"fuzzy": {"$ref": "#/$defs/score"}}}
]
},
"encoding": {"$ref": "#/$defs/encoding"},
"considers": {"type": "array", "items": {"type": "string"}},
"contains": {"type": ["string", "number"], "description": "Substring present, case-insensitive."},
"not_contains": {"type": ["string", "number"], "description": "Substring absent, case-insensitive."},
"contains_any": {"type": ["array", "string"], "items": {"type": ["string", "number"]}, "description": "At least one present."},
"contains_all": {"type": ["array", "string"], "items": {"type": ["string", "number"]}, "description": "Every one present."},
"matches": {"type": "string", "description": "Regular expression that must match."},
"not_matches": {"type": "string", "description": "Regular expression that must not match."},
"is_json": {"type": "boolean", "description": "The reply parses as JSON."},
"schema": {"type": ["boolean", "string"], "description": "true: validate against the agent's declared output schema (SKIPPED if it declares none). A path: validate against that pinned schema file."},
"equals": {"description": "Exact value, any JSON type."},
"not_empty": {"type": "boolean", "description": "Present, non-null, non-empty."},
"length": {"$ref": "#/$defs/bounds", "description": "Length of a string or array."},
"between": {"$ref": "#/$defs/bounds", "description": "A number within bounds."},
"found_in": {"type": "string", "description": "Every value of the field appears in this file (see `match`). Only for fields the agent was told to quote verbatim."},
"artifact_count": {"$ref": "#/$defs/bounds", "description": "How many artifacts the reply returned: a task's artifacts, or a message reply's text parts. An empty reply has none, so {min: 1} fails for it."},
"html_parses": {"type": "boolean", "description": "Well-formed enough to build a tree."},
"html_no_script": {"type": "boolean", "description": "No <script>, javascript:, onerror= or onload= in the text (case-insensitive)."},
"no_placeholders": {"type": "boolean", "description": "No \u007b\u007b \u007d\u007d, TODO, FIXME or lorem ipsum left in."},
"count": {"$ref": "#/$defs/bounds", "description": "How many elements match the selector."},
"markdown_parses": {"type": "boolean", "description": "NOT YET IMPLEMENTED: reports an error when run. Parses as Markdown."},
"heading_structure": {"type": "boolean", "description": "NOT YET IMPLEMENTED: reports an error when run. Headings descend without skipping levels."},
"csv_columns": {"type": "array", "items": {"type": "string"}, "description": "NOT YET IMPLEMENTED: reports an error when run. Exactly these columns, in order."},
"has_attribute": {"type": "string", "description": "NOT YET IMPLEMENTED: reports an error when run. Every selector match carries this attribute."},
"code_parses": {"type": "string", "description": "NOT YET IMPLEMENTED: reports an error when run. Needs requires: [code_execution]. Syntactically valid source in this language."},
"code_defines": {"type": ["array", "string"], "items": {"type": "string"}, "description": "NOT YET IMPLEMENTED: reports an error when run. Needs requires: [code_execution]."},
"code_passes": {"type": "string", "description": "NOT YET IMPLEMENTED: reports an error when run. Needs requires: [code_execution]. Test code run against the generated code."},
"latency_ms": {"$ref": "#/$defs/bounds", "description": "The reply arrived in time, e.g. {max: 10000}."},
"tokens": {"$ref": "#/$defs/bounds", "description": "The request used no more than this many tokens, sub-agents included, e.g. {max: 4000}. Read from the reply's metadata.usage, which agents on runtime 0.25.0 or later report; an older agent reports none and reads as 0."}
},
"anyOf": [
{"required": ["contains"]}, {"required": ["not_contains"]},
{"required": ["contains_any"]}, {"required": ["contains_all"]},
{"required": ["matches"]}, {"required": ["not_matches"]},
{"required": ["is_json"]}, {"required": ["schema"]},
{"required": ["equals"]}, {"required": ["not_empty"]},
{"required": ["length"]}, {"required": ["between"]},
{"required": ["found_in"]}, {"required": ["artifact_count"]},
{"required": ["html_parses"]}, {"required": ["html_no_script"]},
{"required": ["markdown_parses"]}, {"required": ["heading_structure"]},
{"required": ["no_placeholders"]}, {"required": ["csv_columns"]},
{"required": ["count"]}, {"required": ["has_attribute"]},
{"required": ["code_parses"]}, {"required": ["code_defines"]}, {"required": ["code_passes"]},
{"required": ["latency_ms"]}, {"required": ["tokens"]}
]
},
"metric_config": {
"description": "threshold, plus any argument the metric itself needs (e.g. misuse needs its domain).",
"type": ["object", "null"],
"properties": {"threshold": {"$ref": "#/$defs/score"}}
},
"metric": {
"description": "One judged metric, as a single-key mapping: {groundedness: {threshold: 0.8}}. Runs only with judging enabled, and only after every check passed. Costs tokens.",
"type": "object",
"minProperties": 1,
"maxProperties": 1,
"properties": {
"groundedness": {"$ref": "#/$defs/metric_config", "description": "Every claim is supported by the case's context."},
"relevance": {"$ref": "#/$defs/metric_config", "description": "Answers what was asked."},
"completeness": {"$ref": "#/$defs/metric_config", "description": "Covers what the context offers."},
"hallucination": {"$ref": "#/$defs/metric_config", "description": "Contradicts the case's context. Needs context. Lower is better: passes at or below the threshold."},
"semantic_similarity": {"$ref": "#/$defs/metric_config", "description": "Close to expected_output. Usually the wrong tool: assert properties instead."},
"summarization": {"$ref": "#/$defs/metric_config", "description": "The reply is a faithful summary of the input."},
"toxicity": {"$ref": "#/$defs/metric_config", "description": "Safety: threshold 0.0 means none tolerated."},
"bias": {"$ref": "#/$defs/metric_config", "description": "Safety: threshold 0.0 means none tolerated."},
"pii_leakage": {"$ref": "#/$defs/metric_config", "description": "Safety: threshold 0.0 means none tolerated."},
"misuse": {"$ref": "#/$defs/metric_config", "description": "Safety. Needs the domain it judges misuse against."},
"role_violation": {"$ref": "#/$defs/metric_config", "description": "Safety. Needs the role the agent was given."},
"knowledge_retention": {"$ref": "#/$defs/metric_config", "description": "Multi-turn (not run yet)."},
"conversation_completeness": {"$ref": "#/$defs/metric_config", "description": "Multi-turn (not run yet)."},
"conversation_relevancy": {"$ref": "#/$defs/metric_config", "description": "Multi-turn (not run yet)."},
"role_adherence": {"$ref": "#/$defs/metric_config", "description": "Multi-turn (not run yet)."},
"tool_correctness": {"$ref": "#/$defs/metric_config", "description": "Judged, on the tools the agent called: not connected to the agent's trail yet, reported SKIPPED. Check the path with expected_tools."},
"task_completion": {"$ref": "#/$defs/metric_config", "description": "Judged, on the tools the agent called: not connected to the agent's trail yet. Check the path with expected_tools."},
"criteria": {
"description": "An open-ended rubric. The description IS the rubric: write it as an instruction to a careful reader.",
"type": "object",
"additionalProperties": false,
"required": ["description"],
"properties": {
"name": {"type": "string"},
"description": {"type": "string", "minLength": 1},
"threshold": {"$ref": "#/$defs/score"},
"considers": {
"type": "array",
"items": {"enum": ["input", "output", "actual_output", "expected_output", "context", "retrieval_context"]},
"description": "Which parts of the case the judge is shown. Default: input, output, context."
}
}
}
},
"additionalProperties": false
},
"consistency": {
"description": "An assertion about the spread across a case's samples, not one run.",
"type": "object",
"additionalProperties": false,
"properties": {
"field": {"type": "string"},
"max_spread": {"type": "number", "minimum": 0, "description": "Highest minus lowest across runs."},
"identical": {"type": "boolean", "description": "Every run returned the same value. Right for classifiers."}
},
"anyOf": [
{"required": ["max_spread"]},
{"required": ["identical"], "properties": {"identical": {"const": true}}}
]
},
"case": {
"type": "object",
"additionalProperties": false,
"required": ["id"],
"properties": {
"id": {"type": "string", "minLength": 1, "description": "Short, stable and unique in the file. It appears in baselines, so renaming one discards its history."},
"description": {"type": "string", "description": "Why the case exists. Printed when it fails: write it for the person reading a red build."},
"input": {"$ref": "#/$defs/input"},
"turns": {
"type": "array",
"minItems": 1,
"description": "Multi-turn, instead of input. Accepted, but not run yet: reported SKIPPED.",
"items": {
"type": "object",
"required": ["user"],
"properties": {
"user": {"type": "string"},
"checks": {"type": "array", "items": {"$ref": "#/$defs/check"}}
}
}
},
"checks": {"type": "array", "items": {"$ref": "#/$defs/check"}, "description": "Deterministic assertions. Free and instant: exhaust these before metrics."},
"metrics": {"type": "array", "items": {"$ref": "#/$defs/metric"}, "description": "Judged metrics (LLM as judge). Run only with judging on, and only after every check passed."},
"consistency": {"type": "array", "items": {"$ref": "#/$defs/consistency"}, "description": "Assertions about the spread across the case's samples. Need samples of 2 or more."},
"context": {
"type": "array",
"description": "Ground truth the judge is shown. Without it, groundedness has nothing to be grounded against.",
"items": {
"oneOf": [
{"type": "object", "additionalProperties": false, "required": ["file"], "properties": {"file": {"type": "string"}}},
{"type": "object", "additionalProperties": false, "required": ["text"], "properties": {"text": {"type": "string"}}}
]
}
},
"expected_output": {"type": "string", "description": "A reference answer, for semantic_similarity."},
"expected_tools": {
"type": "array",
"description": "The tools the agent must call on its way to the answer, by name (a sub-agent the model calls counts as a tool). Checked against the trail the agent returns with its answer (platform agent SDK 0.31 or later); against an agent that returns none, reported SKIPPED. `arguments` cannot be checked and is refused.",
"items": {"type": "object", "required": ["name"], "additionalProperties": false, "properties": {"name": {"type": "string", "description": "The tool's name, as the agent declares it."}}}
},
"tool_order": {"enum": ["exact", "in_order", "any_order"], "default": "any_order", "description": "How expected_tools is compared with the tools called: any_order (each was called), in_order (in that order, other calls allowed between), exact (those tools and no others, in that order; an empty expected_tools means no tools at all)."},
"expected_skills": {"type": "array", "minItems": 1, "description": "The skills the agent must load on its way to the answer, in any order. A skill pinned from the platform's catalog (a shared skill) is loaded at a version: give `version` to require that exact version. Checked against the trail the agent returns with its answer; against an agent that returns none, reported SKIPPED.", "items": {"type": "object", "required": ["name"], "additionalProperties": false, "properties": {"name": {"type": "string", "minLength": 1, "description": "The skill's name."}, "version": {"type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$", "description": "A shared skill's exact version, e.g. \"1.0.0\" (quoted)."}}}},
"samples": {"type": "integer", "minimum": 1, "description": "Overrides the suite's samples for this case."},
"timeout": {"type": "number", "exclusiveMinimum": 0, "description": "Seconds, this case only."},
"tags": {"type": "array", "items": {"type": "string"}, "description": "Select with --tags."},
"enabled": {"type": "boolean", "default": true, "description": "false: the case is not run, and is reported SKIPPED with its skip_reason."},
"skip_reason": {"type": "string", "description": "Required when enabled is false."},
"requires": {"type": "array", "items": {"type": "string", "examples": ["code_execution"]}, "description": "code_execution runs only with --allow-code-execution."}
},
"anyOf": [{"required": ["input"]}, {"required": ["turns"]}],
"if": {"required": ["enabled"], "properties": {"enabled": {"const": false}}},
"then": {"required": ["skip_reason"]}
}
}
}