All checks were successful
check agents / check (push) Successful in 10s
ALZ-F01 Discovery as a sub-agent orchestrator (sub-agents/) and as eight individual declarative agents (individual-agents/), with their KAB workflow exports. Shared check, render and import scripts, the alz-discovery-sample MCP server, the crucible-kab-dev deploy profile, deployment docs, platform issue reports and the source material. Agent steps time out at 20 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
213 lines
11 KiB
Python
213 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Check agent folders before they ship: the rules a pull request to modernization-agents is held to, plus ours.
|
|
Standard library only, so it runs on any CI runner with Python 3.10 or later.
|
|
|
|
Adapted from cp4-agents for CP1: an agent may carry app/ when agent.yaml declares the code that lives there
|
|
(functions, hooks or sub_agents), and the Dockerfile must then copy it; each sub-agent's prompt must exist and be
|
|
copied. A purely declarative agent is still refused an app/ it does not use.
|
|
|
|
Checked: the declarative files are present and there is no unused app/; agent.yaml's name is the folder's, it has a
|
|
display_name, its version is #.#.# or DEV with a matching RELEASE_NOTES.md entry; the prompt and contract files
|
|
it names exist; every skill it lists has its SKILL.md, and the Dockerfile copies them; the Dockerfile builds on
|
|
the pinned runtime image; Kyndryl's CI placeholders are kept in kubernetes/; an agent on a reasoning model has eval
|
|
cases that wait long enough for its replies; no file carries a secret. (JSON Schema
|
|
validity and eval-case inputs need jsonschema, and are checked where the folders are built.)
|
|
|
|
Usage: check_agents.py --runtime-image <image> <folder>... prints "<folder>: <problem>", exits 1 on any
|
|
"""
|
|
import argparse
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
K8S = ("configmap", "deployment", "kustomization", "namespace", "secretnames", "service")
|
|
REQUIRED = ("agent.yaml", "prompt.md", "schemas/input.json", "schemas/output.json", "eval/cases.yaml", "Dockerfile",
|
|
"RELEASE_NOTES.md", "README.md", *(f"kubernetes/{n}.yaml" for n in K8S))
|
|
SEMVER_OR_DEV = re.compile(r"^(\d+\.\d+\.\d+|DEV)$")
|
|
SECRET = re.compile(r"\bsk-[A-Za-z0-9_-]{20,}|abk1_|-----BEGIN")
|
|
RUNTIME_ARG = re.compile(r"^ARG RUNTIME_IMAGE=(\S*)$", re.M)
|
|
REASONING_MODELS = ("gpt-5.6-sol", "gpt-5.6-luna") # their replies take longer than the eval runner's 120 s wait
|
|
CASE_TIMEOUT_FLOOR = 300
|
|
SECTIONS = ("### Added", "### Changed", "### Fixed", "### Removed")
|
|
|
|
|
|
def _scalar(v: str) -> str:
|
|
v = v.strip()
|
|
return v[1:-1] if len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"" else v
|
|
|
|
|
|
def read_agent_yaml(text: str) -> dict:
|
|
"""The few agent.yaml keys checked here, from the plain block style these folders are written in."""
|
|
agent: dict = {"contract": {}, "skills": [], "keys": set(), "sub_agent_prompts": []}
|
|
key = None
|
|
skills_indent = None # the indent of the skills list's own items, so nested lists (an entry's tags) are not read
|
|
for line in text.splitlines():
|
|
if not line.strip() or line.lstrip().startswith("#"):
|
|
continue
|
|
if not line.startswith((" ", "-")):
|
|
k, _, v = line.partition(":")
|
|
key = k.strip()
|
|
agent["keys"].add(key)
|
|
if v.strip():
|
|
agent[key] = _scalar(v.split(" #")[0])
|
|
elif key == "contract" and (m := re.match(r"^\s+(\w+):\s*(.*)$", line)):
|
|
agent["contract"][m.group(1)] = _scalar(m.group(2))
|
|
elif key == "skills" and line.lstrip().startswith("- "):
|
|
indent = len(line) - len(line.lstrip())
|
|
skills_indent = indent if skills_indent is None else skills_indent
|
|
if indent != skills_indent:
|
|
continue
|
|
item = line.lstrip()[2:].strip()
|
|
agent["skills"].append({"id": item} if re.match(r"^\w+:\s", item) else _scalar(item))
|
|
elif key == "sub_agents" and (m := re.match(r"^\s*(?:- )?\s*prompt:\s*(.+)$", line)):
|
|
agent["sub_agent_prompts"].append(_scalar(m.group(1)))
|
|
return agent
|
|
|
|
|
|
CODE_KEYS = ("functions", "hooks", "sub_agents")
|
|
|
|
|
|
def check_folder(path: Path, runtime_image: str) -> list[str]:
|
|
problems = [f"missing: {rel}" for rel in REQUIRED if not (path / rel).is_file()]
|
|
agent = read_agent_yaml((path / "agent.yaml").read_text()) if (path / "agent.yaml").is_file() else None
|
|
uses_code = bool(agent and agent["keys"] & set(CODE_KEYS))
|
|
if (path / "app").exists() and not uses_code:
|
|
problems.append("app-present: app/ (agent.yaml declares no functions, hooks or sub_agents that need it)")
|
|
if uses_code and not (path / "app").is_dir():
|
|
problems.append("missing: app/ (agent.yaml declares functions, hooks or sub_agents)")
|
|
if agent is not None:
|
|
problems += [f"sub-agent-prompt-missing: {rel}" for rel in agent["sub_agent_prompts"]
|
|
if not (path / rel).is_file()]
|
|
if agent.get("name") != path.name:
|
|
problems.append(f"name: agent.yaml name {agent.get('name')!r} is not the folder name {path.name!r}")
|
|
if not agent.get("display_name"):
|
|
problems.append("name: agent.yaml has no display_name")
|
|
version = str(agent.get("version", ""))
|
|
if not SEMVER_OR_DEV.match(version):
|
|
problems.append(f"version: {version!r} is neither #.#.# nor DEV")
|
|
elif version != "DEV":
|
|
problems += _release_notes(path, version)
|
|
pointed = [agent.get("prompt") or "prompt.md"]
|
|
for key in ("input", "output"):
|
|
if not agent["contract"].get(key):
|
|
problems.append(f"contract-file-missing: contract.{key} is not set")
|
|
else:
|
|
pointed.append(agent["contract"][key])
|
|
problems += [f"contract-file-missing: {rel}" for rel in pointed if not (path / rel).is_file()]
|
|
problems += [f"skill-missing: {s}" for s in agent["skills"]
|
|
if isinstance(s, str) and not (path / "skills" / s / "SKILL.md").is_file()]
|
|
problems += _dockerfile(path, runtime_image, agent)
|
|
problems += _kubernetes(path)
|
|
problems += _case_timeout(path)
|
|
for f in sorted(p for p in path.rglob("*") if p.is_file()):
|
|
rel = f.relative_to(path).as_posix()
|
|
text = f.read_text(errors="replace")
|
|
if SECRET.search(text):
|
|
problems.append(f"secret: {rel}")
|
|
return problems
|
|
|
|
|
|
def _release_notes(path: Path, version: str) -> list[str]:
|
|
notes = path / "RELEASE_NOTES.md"
|
|
if not notes.is_file():
|
|
return []
|
|
heads = [line.rstrip() for line in notes.read_text().splitlines() if line.startswith("#")]
|
|
if not heads or not re.match(rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}$", heads[0]):
|
|
return [f"release-notes: the first heading must be '## [{version}] - YYYY-MM-DD'"]
|
|
if len(heads) < 2 or heads[1] not in SECTIONS:
|
|
return [f"release-notes: '## [{version}]' must be followed by one of {', '.join(SECTIONS)}"]
|
|
return []
|
|
|
|
|
|
def _dockerfile(path: Path, runtime_image: str, agent) -> list[str]:
|
|
df = path / "Dockerfile"
|
|
if not df.is_file():
|
|
return []
|
|
text = df.read_text()
|
|
problems = []
|
|
m = RUNTIME_ARG.search(text)
|
|
if not m or m.group(1) != runtime_image or "FROM ${RUNTIME_IMAGE}" not in text:
|
|
problems.append(f"runtime-unpinned: the Dockerfile must build FROM {runtime_image} "
|
|
f"(found {m.group(1) if m else 'no RUNTIME_IMAGE'})")
|
|
copies = "COPY skills/ /agent/skills/" in text
|
|
if [s for s in (agent or {}).get("skills") or [] if isinstance(s, str)] and not copies:
|
|
problems.append("skill-missing: the Dockerfile does not copy skills/")
|
|
if copies and not (path / "skills").is_dir():
|
|
problems.append("missing: skills/ (the Dockerfile copies it)")
|
|
if (path / "app").is_dir() and "COPY app/ /agent/app/" not in text:
|
|
problems.append("app-not-copied: the Dockerfile does not copy app/")
|
|
if (agent or {}).get("sub_agent_prompts") and "COPY prompts/ /agent/prompts/" not in text:
|
|
problems.append("sub-agent-prompt-missing: the Dockerfile does not copy prompts/")
|
|
return problems
|
|
|
|
|
|
def _kubernetes(path: Path) -> list[str]:
|
|
problems = []
|
|
for name in K8S:
|
|
f = path / "kubernetes" / f"{name}.yaml"
|
|
if f.is_file() and ("{{" in f.read_text() or "{%" in f.read_text()):
|
|
problems.append(f"kubernetes: {name}.yaml has unfilled template fields")
|
|
dep, cm = path / "kubernetes" / "deployment.yaml", path / "kubernetes" / "configmap.yaml"
|
|
if dep.is_file():
|
|
for marker in ("<placeholder_for_agent_name>", "<placeholder_for_image>"):
|
|
if marker not in dep.read_text():
|
|
problems.append(f"kubernetes: deployment.yaml lost Kyndryl CI's {marker}")
|
|
if cm.is_file():
|
|
text = cm.read_text()
|
|
if 'PORT: "8080"' not in text:
|
|
problems.append('kubernetes: configmap.yaml must set PORT: "8080"')
|
|
if not re.search(r'^ LITELLM_MODEL: "[^"{]+"$', text, re.M):
|
|
problems.append("kubernetes: configmap.yaml must set LITELLM_MODEL")
|
|
return problems
|
|
|
|
|
|
def _number(text: str):
|
|
return int(text) if text.isdigit() else float(text)
|
|
|
|
|
|
def _case_timeout(path: Path) -> list[str]:
|
|
"""An agent whose configmap.yaml names a reasoning model has eval cases that wait at least CASE_TIMEOUT_FLOOR
|
|
seconds for a reply: the eval runner waits 120 unless the cases say otherwise (a case's own `timeout`, else the
|
|
file's `defaults.timeout`)."""
|
|
cm, cases = path / "kubernetes" / "configmap.yaml", path / "eval" / "cases.yaml"
|
|
model = re.search(r'^ LITELLM_MODEL: "([^"]+)"$', cm.read_text(), re.M) if cm.is_file() else None
|
|
if not model or model.group(1) not in REASONING_MODELS or not cases.is_file():
|
|
return []
|
|
parts = re.split(r"^cases:[ \t]*$", cases.read_text(), maxsplit=1, flags=re.M)
|
|
head, body = parts[0], parts[1] if len(parts) > 1 else ""
|
|
block = re.search(r"^defaults:[ \t]*\n((?:[ \t]+.*(?:\n|$))*)", head, re.M)
|
|
found = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", block.group(1), re.M) if block else None
|
|
default = _number(found.group(1)) if found else None
|
|
short = []
|
|
for case in re.split(r"^- ", body, flags=re.M)[1:]:
|
|
own = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", case, re.M)
|
|
wait = _number(own.group(1)) if own else (default or 120)
|
|
if wait < CASE_TIMEOUT_FLOOR:
|
|
name = re.match(r"id: ['\"]?([^'\"\n]+)", case)
|
|
short.append(name.group(1).strip() if name else "?")
|
|
if not short:
|
|
return []
|
|
return [f"case-timeout: {model.group(1)} is a reasoning model, and eval/cases.yaml waits "
|
|
f"{default if default is not None else 'the runner default of 120'} s for a reply"
|
|
f" (cases {', '.join(short[:3])}{'...' if len(short) > 3 else ''}): set defaults.timeout to at least "
|
|
f"{CASE_TIMEOUT_FLOOR}"]
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--runtime-image", required=True)
|
|
parser.add_argument("folders", nargs="+")
|
|
args = parser.parse_args(argv)
|
|
bad = 0
|
|
for folder in args.folders:
|
|
path = Path(folder).resolve()
|
|
for p in check_folder(path, args.runtime_image):
|
|
print(f"{path.name}: {p}")
|
|
bad += 1
|
|
print(f"{len(args.folders)} folders, {bad} problems")
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|