#!/usr/bin/env python3 """Check agent folders before they ship: the rules a pull request to modernization-agents is held to, plus ours. Standard library only, so it runs on any CI runner with Python 3.10 or later. Adapted from cp4-agents for CP1: an agent may carry app/ when agent.yaml declares the code that lives there (functions, hooks or sub_agents), and the Dockerfile must then copy it; each sub-agent's prompt must exist and be copied. A purely declarative agent is still refused an app/ it does not use. Checked: the declarative files are present and there is no unused app/; agent.yaml's name is the folder's, it has a display_name, its version is #.#.# or DEV with a matching RELEASE_NOTES.md entry; the prompt and contract files it names exist; every skill it lists has its SKILL.md, and the Dockerfile copies them; the Dockerfile builds on the pinned runtime image; Kyndryl's CI placeholders are kept in kubernetes/; an agent on a reasoning model has eval cases that wait long enough for its replies; no file carries a secret. (JSON Schema validity and eval-case inputs need jsonschema, and are checked where the folders are built.) Usage: check_agents.py --runtime-image ... prints ": ", exits 1 on any """ import argparse import re import sys from pathlib import Path K8S = ("configmap", "deployment", "kustomization", "namespace", "secretnames", "service") REQUIRED = ("agent.yaml", "prompt.md", "schemas/input.json", "schemas/output.json", "eval/cases.yaml", "Dockerfile", "RELEASE_NOTES.md", "README.md", *(f"kubernetes/{n}.yaml" for n in K8S)) SEMVER_OR_DEV = re.compile(r"^(\d+\.\d+\.\d+|DEV)$") SECRET = re.compile(r"\bsk-[A-Za-z0-9_-]{20,}|abk1_|-----BEGIN") RUNTIME_ARG = re.compile(r"^ARG RUNTIME_IMAGE=(\S*)$", re.M) REASONING_MODELS = ("gpt-5.6-sol", "gpt-5.6-luna") # their replies take longer than the eval runner's 120 s wait CASE_TIMEOUT_FLOOR = 300 SECTIONS = ("### Added", "### Changed", "### Fixed", "### Removed") def _scalar(v: str) -> str: v = v.strip() return v[1:-1] if len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"" else v def read_agent_yaml(text: str) -> dict: """The few agent.yaml keys checked here, from the plain block style these folders are written in.""" agent: dict = {"contract": {}, "skills": [], "keys": set(), "sub_agent_prompts": []} key = None skills_indent = None # the indent of the skills list's own items, so nested lists (an entry's tags) are not read for line in text.splitlines(): if not line.strip() or line.lstrip().startswith("#"): continue if not line.startswith((" ", "-")): k, _, v = line.partition(":") key = k.strip() agent["keys"].add(key) if v.strip(): agent[key] = _scalar(v.split(" #")[0]) elif key == "contract" and (m := re.match(r"^\s+(\w+):\s*(.*)$", line)): agent["contract"][m.group(1)] = _scalar(m.group(2)) elif key == "skills" and line.lstrip().startswith("- "): indent = len(line) - len(line.lstrip()) skills_indent = indent if skills_indent is None else skills_indent if indent != skills_indent: continue item = line.lstrip()[2:].strip() agent["skills"].append({"id": item} if re.match(r"^\w+:\s", item) else _scalar(item)) elif key == "sub_agents" and (m := re.match(r"^\s*(?:- )?\s*prompt:\s*(.+)$", line)): agent["sub_agent_prompts"].append(_scalar(m.group(1))) return agent CODE_KEYS = ("functions", "hooks", "sub_agents") def check_folder(path: Path, runtime_image: str) -> list[str]: problems = [f"missing: {rel}" for rel in REQUIRED if not (path / rel).is_file()] agent = read_agent_yaml((path / "agent.yaml").read_text()) if (path / "agent.yaml").is_file() else None uses_code = bool(agent and agent["keys"] & set(CODE_KEYS)) if (path / "app").exists() and not uses_code: problems.append("app-present: app/ (agent.yaml declares no functions, hooks or sub_agents that need it)") if uses_code and not (path / "app").is_dir(): problems.append("missing: app/ (agent.yaml declares functions, hooks or sub_agents)") if agent is not None: problems += [f"sub-agent-prompt-missing: {rel}" for rel in agent["sub_agent_prompts"] if not (path / rel).is_file()] if agent.get("name") != path.name: problems.append(f"name: agent.yaml name {agent.get('name')!r} is not the folder name {path.name!r}") if not agent.get("display_name"): problems.append("name: agent.yaml has no display_name") version = str(agent.get("version", "")) if not SEMVER_OR_DEV.match(version): problems.append(f"version: {version!r} is neither #.#.# nor DEV") elif version != "DEV": problems += _release_notes(path, version) pointed = [agent.get("prompt") or "prompt.md"] for key in ("input", "output"): if not agent["contract"].get(key): problems.append(f"contract-file-missing: contract.{key} is not set") else: pointed.append(agent["contract"][key]) problems += [f"contract-file-missing: {rel}" for rel in pointed if not (path / rel).is_file()] problems += [f"skill-missing: {s}" for s in agent["skills"] if isinstance(s, str) and not (path / "skills" / s / "SKILL.md").is_file()] problems += _dockerfile(path, runtime_image, agent) problems += _kubernetes(path) problems += _case_timeout(path) for f in sorted(p for p in path.rglob("*") if p.is_file()): rel = f.relative_to(path).as_posix() text = f.read_text(errors="replace") if SECRET.search(text): problems.append(f"secret: {rel}") return problems def _release_notes(path: Path, version: str) -> list[str]: notes = path / "RELEASE_NOTES.md" if not notes.is_file(): return [] heads = [line.rstrip() for line in notes.read_text().splitlines() if line.startswith("#")] if not heads or not re.match(rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}$", heads[0]): return [f"release-notes: the first heading must be '## [{version}] - YYYY-MM-DD'"] if len(heads) < 2 or heads[1] not in SECTIONS: return [f"release-notes: '## [{version}]' must be followed by one of {', '.join(SECTIONS)}"] return [] def _dockerfile(path: Path, runtime_image: str, agent) -> list[str]: df = path / "Dockerfile" if not df.is_file(): return [] text = df.read_text() problems = [] m = RUNTIME_ARG.search(text) if not m or m.group(1) != runtime_image or "FROM ${RUNTIME_IMAGE}" not in text: problems.append(f"runtime-unpinned: the Dockerfile must build FROM {runtime_image} " f"(found {m.group(1) if m else 'no RUNTIME_IMAGE'})") copies = "COPY skills/ /agent/skills/" in text if [s for s in (agent or {}).get("skills") or [] if isinstance(s, str)] and not copies: problems.append("skill-missing: the Dockerfile does not copy skills/") if copies and not (path / "skills").is_dir(): problems.append("missing: skills/ (the Dockerfile copies it)") if (path / "app").is_dir() and "COPY app/ /agent/app/" not in text: problems.append("app-not-copied: the Dockerfile does not copy app/") if (agent or {}).get("sub_agent_prompts") and "COPY prompts/ /agent/prompts/" not in text: problems.append("sub-agent-prompt-missing: the Dockerfile does not copy prompts/") return problems def _kubernetes(path: Path) -> list[str]: problems = [] for name in K8S: f = path / "kubernetes" / f"{name}.yaml" if f.is_file() and ("{{" in f.read_text() or "{%" in f.read_text()): problems.append(f"kubernetes: {name}.yaml has unfilled template fields") dep, cm = path / "kubernetes" / "deployment.yaml", path / "kubernetes" / "configmap.yaml" if dep.is_file(): for marker in ("", ""): if marker not in dep.read_text(): problems.append(f"kubernetes: deployment.yaml lost Kyndryl CI's {marker}") if cm.is_file(): text = cm.read_text() if 'PORT: "8080"' not in text: problems.append('kubernetes: configmap.yaml must set PORT: "8080"') if not re.search(r'^ LITELLM_MODEL: "[^"{]+"$', text, re.M): problems.append("kubernetes: configmap.yaml must set LITELLM_MODEL") return problems def _number(text: str): return int(text) if text.isdigit() else float(text) def _case_timeout(path: Path) -> list[str]: """An agent whose configmap.yaml names a reasoning model has eval cases that wait at least CASE_TIMEOUT_FLOOR seconds for a reply: the eval runner waits 120 unless the cases say otherwise (a case's own `timeout`, else the file's `defaults.timeout`).""" cm, cases = path / "kubernetes" / "configmap.yaml", path / "eval" / "cases.yaml" model = re.search(r'^ LITELLM_MODEL: "([^"]+)"$', cm.read_text(), re.M) if cm.is_file() else None if not model or model.group(1) not in REASONING_MODELS or not cases.is_file(): return [] parts = re.split(r"^cases:[ \t]*$", cases.read_text(), maxsplit=1, flags=re.M) head, body = parts[0], parts[1] if len(parts) > 1 else "" block = re.search(r"^defaults:[ \t]*\n((?:[ \t]+.*(?:\n|$))*)", head, re.M) found = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", block.group(1), re.M) if block else None default = _number(found.group(1)) if found else None short = [] for case in re.split(r"^- ", body, flags=re.M)[1:]: own = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", case, re.M) wait = _number(own.group(1)) if own else (default or 120) if wait < CASE_TIMEOUT_FLOOR: name = re.match(r"id: ['\"]?([^'\"\n]+)", case) short.append(name.group(1).strip() if name else "?") if not short: return [] return [f"case-timeout: {model.group(1)} is a reasoning model, and eval/cases.yaml waits " f"{default if default is not None else 'the runner default of 120'} s for a reply" f" (cases {', '.join(short[:3])}{'...' if len(short) > 3 else ''}): set defaults.timeout to at least " f"{CASE_TIMEOUT_FLOOR}"] def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--runtime-image", required=True) parser.add_argument("folders", nargs="+") args = parser.parse_args(argv) bad = 0 for folder in args.folders: path = Path(folder).resolve() for p in check_folder(path, args.runtime_image): print(f"{path.name}: {p}") bad += 1 print(f"{len(args.folders)} folders, {bad} problems") return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())