Files
cp1-agents/scripts/check_agents.py
Jonathan Boniface 416c66ced4
All checks were successful
check agents / check (push) Successful in 10s
Add CP1 Discovery agents in both patterns
ALZ-F01 Discovery as a sub-agent orchestrator (sub-agents/) and as eight
individual declarative agents (individual-agents/), with their KAB workflow
exports. Shared check, render and import scripts, the alz-discovery-sample
MCP server, the crucible-kab-dev deploy profile, deployment docs, platform
issue reports and the source material. Agent steps time out at 20 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 18:25:57 +00:00

213 lines
11 KiB
Python

#!/usr/bin/env python3
"""Check agent folders before they ship: the rules a pull request to modernization-agents is held to, plus ours.
Standard library only, so it runs on any CI runner with Python 3.10 or later.
Adapted from cp4-agents for CP1: an agent may carry app/ when agent.yaml declares the code that lives there
(functions, hooks or sub_agents), and the Dockerfile must then copy it; each sub-agent's prompt must exist and be
copied. A purely declarative agent is still refused an app/ it does not use.
Checked: the declarative files are present and there is no unused app/; agent.yaml's name is the folder's, it has a
display_name, its version is #.#.# or DEV with a matching RELEASE_NOTES.md entry; the prompt and contract files
it names exist; every skill it lists has its SKILL.md, and the Dockerfile copies them; the Dockerfile builds on
the pinned runtime image; Kyndryl's CI placeholders are kept in kubernetes/; an agent on a reasoning model has eval
cases that wait long enough for its replies; no file carries a secret. (JSON Schema
validity and eval-case inputs need jsonschema, and are checked where the folders are built.)
Usage: check_agents.py --runtime-image <image> <folder>... prints "<folder>: <problem>", exits 1 on any
"""
import argparse
import re
import sys
from pathlib import Path
K8S = ("configmap", "deployment", "kustomization", "namespace", "secretnames", "service")
REQUIRED = ("agent.yaml", "prompt.md", "schemas/input.json", "schemas/output.json", "eval/cases.yaml", "Dockerfile",
"RELEASE_NOTES.md", "README.md", *(f"kubernetes/{n}.yaml" for n in K8S))
SEMVER_OR_DEV = re.compile(r"^(\d+\.\d+\.\d+|DEV)$")
SECRET = re.compile(r"\bsk-[A-Za-z0-9_-]{20,}|abk1_|-----BEGIN")
RUNTIME_ARG = re.compile(r"^ARG RUNTIME_IMAGE=(\S*)$", re.M)
REASONING_MODELS = ("gpt-5.6-sol", "gpt-5.6-luna") # their replies take longer than the eval runner's 120 s wait
CASE_TIMEOUT_FLOOR = 300
SECTIONS = ("### Added", "### Changed", "### Fixed", "### Removed")
def _scalar(v: str) -> str:
v = v.strip()
return v[1:-1] if len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"" else v
def read_agent_yaml(text: str) -> dict:
"""The few agent.yaml keys checked here, from the plain block style these folders are written in."""
agent: dict = {"contract": {}, "skills": [], "keys": set(), "sub_agent_prompts": []}
key = None
skills_indent = None # the indent of the skills list's own items, so nested lists (an entry's tags) are not read
for line in text.splitlines():
if not line.strip() or line.lstrip().startswith("#"):
continue
if not line.startswith((" ", "-")):
k, _, v = line.partition(":")
key = k.strip()
agent["keys"].add(key)
if v.strip():
agent[key] = _scalar(v.split(" #")[0])
elif key == "contract" and (m := re.match(r"^\s+(\w+):\s*(.*)$", line)):
agent["contract"][m.group(1)] = _scalar(m.group(2))
elif key == "skills" and line.lstrip().startswith("- "):
indent = len(line) - len(line.lstrip())
skills_indent = indent if skills_indent is None else skills_indent
if indent != skills_indent:
continue
item = line.lstrip()[2:].strip()
agent["skills"].append({"id": item} if re.match(r"^\w+:\s", item) else _scalar(item))
elif key == "sub_agents" and (m := re.match(r"^\s*(?:- )?\s*prompt:\s*(.+)$", line)):
agent["sub_agent_prompts"].append(_scalar(m.group(1)))
return agent
CODE_KEYS = ("functions", "hooks", "sub_agents")
def check_folder(path: Path, runtime_image: str) -> list[str]:
problems = [f"missing: {rel}" for rel in REQUIRED if not (path / rel).is_file()]
agent = read_agent_yaml((path / "agent.yaml").read_text()) if (path / "agent.yaml").is_file() else None
uses_code = bool(agent and agent["keys"] & set(CODE_KEYS))
if (path / "app").exists() and not uses_code:
problems.append("app-present: app/ (agent.yaml declares no functions, hooks or sub_agents that need it)")
if uses_code and not (path / "app").is_dir():
problems.append("missing: app/ (agent.yaml declares functions, hooks or sub_agents)")
if agent is not None:
problems += [f"sub-agent-prompt-missing: {rel}" for rel in agent["sub_agent_prompts"]
if not (path / rel).is_file()]
if agent.get("name") != path.name:
problems.append(f"name: agent.yaml name {agent.get('name')!r} is not the folder name {path.name!r}")
if not agent.get("display_name"):
problems.append("name: agent.yaml has no display_name")
version = str(agent.get("version", ""))
if not SEMVER_OR_DEV.match(version):
problems.append(f"version: {version!r} is neither #.#.# nor DEV")
elif version != "DEV":
problems += _release_notes(path, version)
pointed = [agent.get("prompt") or "prompt.md"]
for key in ("input", "output"):
if not agent["contract"].get(key):
problems.append(f"contract-file-missing: contract.{key} is not set")
else:
pointed.append(agent["contract"][key])
problems += [f"contract-file-missing: {rel}" for rel in pointed if not (path / rel).is_file()]
problems += [f"skill-missing: {s}" for s in agent["skills"]
if isinstance(s, str) and not (path / "skills" / s / "SKILL.md").is_file()]
problems += _dockerfile(path, runtime_image, agent)
problems += _kubernetes(path)
problems += _case_timeout(path)
for f in sorted(p for p in path.rglob("*") if p.is_file()):
rel = f.relative_to(path).as_posix()
text = f.read_text(errors="replace")
if SECRET.search(text):
problems.append(f"secret: {rel}")
return problems
def _release_notes(path: Path, version: str) -> list[str]:
notes = path / "RELEASE_NOTES.md"
if not notes.is_file():
return []
heads = [line.rstrip() for line in notes.read_text().splitlines() if line.startswith("#")]
if not heads or not re.match(rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}$", heads[0]):
return [f"release-notes: the first heading must be '## [{version}] - YYYY-MM-DD'"]
if len(heads) < 2 or heads[1] not in SECTIONS:
return [f"release-notes: '## [{version}]' must be followed by one of {', '.join(SECTIONS)}"]
return []
def _dockerfile(path: Path, runtime_image: str, agent) -> list[str]:
df = path / "Dockerfile"
if not df.is_file():
return []
text = df.read_text()
problems = []
m = RUNTIME_ARG.search(text)
if not m or m.group(1) != runtime_image or "FROM ${RUNTIME_IMAGE}" not in text:
problems.append(f"runtime-unpinned: the Dockerfile must build FROM {runtime_image} "
f"(found {m.group(1) if m else 'no RUNTIME_IMAGE'})")
copies = "COPY skills/ /agent/skills/" in text
if [s for s in (agent or {}).get("skills") or [] if isinstance(s, str)] and not copies:
problems.append("skill-missing: the Dockerfile does not copy skills/")
if copies and not (path / "skills").is_dir():
problems.append("missing: skills/ (the Dockerfile copies it)")
if (path / "app").is_dir() and "COPY app/ /agent/app/" not in text:
problems.append("app-not-copied: the Dockerfile does not copy app/")
if (agent or {}).get("sub_agent_prompts") and "COPY prompts/ /agent/prompts/" not in text:
problems.append("sub-agent-prompt-missing: the Dockerfile does not copy prompts/")
return problems
def _kubernetes(path: Path) -> list[str]:
problems = []
for name in K8S:
f = path / "kubernetes" / f"{name}.yaml"
if f.is_file() and ("{{" in f.read_text() or "{%" in f.read_text()):
problems.append(f"kubernetes: {name}.yaml has unfilled template fields")
dep, cm = path / "kubernetes" / "deployment.yaml", path / "kubernetes" / "configmap.yaml"
if dep.is_file():
for marker in ("<placeholder_for_agent_name>", "<placeholder_for_image>"):
if marker not in dep.read_text():
problems.append(f"kubernetes: deployment.yaml lost Kyndryl CI's {marker}")
if cm.is_file():
text = cm.read_text()
if 'PORT: "8080"' not in text:
problems.append('kubernetes: configmap.yaml must set PORT: "8080"')
if not re.search(r'^ LITELLM_MODEL: "[^"{]+"$', text, re.M):
problems.append("kubernetes: configmap.yaml must set LITELLM_MODEL")
return problems
def _number(text: str):
return int(text) if text.isdigit() else float(text)
def _case_timeout(path: Path) -> list[str]:
"""An agent whose configmap.yaml names a reasoning model has eval cases that wait at least CASE_TIMEOUT_FLOOR
seconds for a reply: the eval runner waits 120 unless the cases say otherwise (a case's own `timeout`, else the
file's `defaults.timeout`)."""
cm, cases = path / "kubernetes" / "configmap.yaml", path / "eval" / "cases.yaml"
model = re.search(r'^ LITELLM_MODEL: "([^"]+)"$', cm.read_text(), re.M) if cm.is_file() else None
if not model or model.group(1) not in REASONING_MODELS or not cases.is_file():
return []
parts = re.split(r"^cases:[ \t]*$", cases.read_text(), maxsplit=1, flags=re.M)
head, body = parts[0], parts[1] if len(parts) > 1 else ""
block = re.search(r"^defaults:[ \t]*\n((?:[ \t]+.*(?:\n|$))*)", head, re.M)
found = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", block.group(1), re.M) if block else None
default = _number(found.group(1)) if found else None
short = []
for case in re.split(r"^- ", body, flags=re.M)[1:]:
own = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", case, re.M)
wait = _number(own.group(1)) if own else (default or 120)
if wait < CASE_TIMEOUT_FLOOR:
name = re.match(r"id: ['\"]?([^'\"\n]+)", case)
short.append(name.group(1).strip() if name else "?")
if not short:
return []
return [f"case-timeout: {model.group(1)} is a reasoning model, and eval/cases.yaml waits "
f"{default if default is not None else 'the runner default of 120'} s for a reply"
f" (cases {', '.join(short[:3])}{'...' if len(short) > 3 else ''}): set defaults.timeout to at least "
f"{CASE_TIMEOUT_FLOOR}"]
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--runtime-image", required=True)
parser.add_argument("folders", nargs="+")
args = parser.parse_args(argv)
bad = 0
for folder in args.folders:
path = Path(folder).resolve()
for p in check_folder(path, args.runtime_image):
print(f"{path.name}: {p}")
bad += 1
print(f"{len(args.folders)} folders, {bad} problems")
return 1 if bad else 0
if __name__ == "__main__":
sys.exit(main())