Add CP1 Discovery agents in both patterns
All checks were successful
check agents / check (push) Successful in 10s
All checks were successful
check agents / check (push) Successful in 10s
ALZ-F01 Discovery as a sub-agent orchestrator (sub-agents/) and as eight individual declarative agents (individual-agents/), with their KAB workflow exports. Shared check, render and import scripts, the alz-discovery-sample MCP server, the crucible-kab-dev deploy profile, deployment docs, platform issue reports and the source material. Agent steps time out at 20 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
212
scripts/check_agents.py
Normal file
212
scripts/check_agents.py
Normal file
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check agent folders before they ship: the rules a pull request to modernization-agents is held to, plus ours.
|
||||
Standard library only, so it runs on any CI runner with Python 3.10 or later.
|
||||
|
||||
Adapted from cp4-agents for CP1: an agent may carry app/ when agent.yaml declares the code that lives there
|
||||
(functions, hooks or sub_agents), and the Dockerfile must then copy it; each sub-agent's prompt must exist and be
|
||||
copied. A purely declarative agent is still refused an app/ it does not use.
|
||||
|
||||
Checked: the declarative files are present and there is no unused app/; agent.yaml's name is the folder's, it has a
|
||||
display_name, its version is #.#.# or DEV with a matching RELEASE_NOTES.md entry; the prompt and contract files
|
||||
it names exist; every skill it lists has its SKILL.md, and the Dockerfile copies them; the Dockerfile builds on
|
||||
the pinned runtime image; Kyndryl's CI placeholders are kept in kubernetes/; an agent on a reasoning model has eval
|
||||
cases that wait long enough for its replies; no file carries a secret. (JSON Schema
|
||||
validity and eval-case inputs need jsonschema, and are checked where the folders are built.)
|
||||
|
||||
Usage: check_agents.py --runtime-image <image> <folder>... prints "<folder>: <problem>", exits 1 on any
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
K8S = ("configmap", "deployment", "kustomization", "namespace", "secretnames", "service")
|
||||
REQUIRED = ("agent.yaml", "prompt.md", "schemas/input.json", "schemas/output.json", "eval/cases.yaml", "Dockerfile",
|
||||
"RELEASE_NOTES.md", "README.md", *(f"kubernetes/{n}.yaml" for n in K8S))
|
||||
SEMVER_OR_DEV = re.compile(r"^(\d+\.\d+\.\d+|DEV)$")
|
||||
SECRET = re.compile(r"\bsk-[A-Za-z0-9_-]{20,}|abk1_|-----BEGIN")
|
||||
RUNTIME_ARG = re.compile(r"^ARG RUNTIME_IMAGE=(\S*)$", re.M)
|
||||
REASONING_MODELS = ("gpt-5.6-sol", "gpt-5.6-luna") # their replies take longer than the eval runner's 120 s wait
|
||||
CASE_TIMEOUT_FLOOR = 300
|
||||
SECTIONS = ("### Added", "### Changed", "### Fixed", "### Removed")
|
||||
|
||||
|
||||
def _scalar(v: str) -> str:
|
||||
v = v.strip()
|
||||
return v[1:-1] if len(v) >= 2 and v[0] == v[-1] and v[0] in "'\"" else v
|
||||
|
||||
|
||||
def read_agent_yaml(text: str) -> dict:
|
||||
"""The few agent.yaml keys checked here, from the plain block style these folders are written in."""
|
||||
agent: dict = {"contract": {}, "skills": [], "keys": set(), "sub_agent_prompts": []}
|
||||
key = None
|
||||
skills_indent = None # the indent of the skills list's own items, so nested lists (an entry's tags) are not read
|
||||
for line in text.splitlines():
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
if not line.startswith((" ", "-")):
|
||||
k, _, v = line.partition(":")
|
||||
key = k.strip()
|
||||
agent["keys"].add(key)
|
||||
if v.strip():
|
||||
agent[key] = _scalar(v.split(" #")[0])
|
||||
elif key == "contract" and (m := re.match(r"^\s+(\w+):\s*(.*)$", line)):
|
||||
agent["contract"][m.group(1)] = _scalar(m.group(2))
|
||||
elif key == "skills" and line.lstrip().startswith("- "):
|
||||
indent = len(line) - len(line.lstrip())
|
||||
skills_indent = indent if skills_indent is None else skills_indent
|
||||
if indent != skills_indent:
|
||||
continue
|
||||
item = line.lstrip()[2:].strip()
|
||||
agent["skills"].append({"id": item} if re.match(r"^\w+:\s", item) else _scalar(item))
|
||||
elif key == "sub_agents" and (m := re.match(r"^\s*(?:- )?\s*prompt:\s*(.+)$", line)):
|
||||
agent["sub_agent_prompts"].append(_scalar(m.group(1)))
|
||||
return agent
|
||||
|
||||
|
||||
CODE_KEYS = ("functions", "hooks", "sub_agents")
|
||||
|
||||
|
||||
def check_folder(path: Path, runtime_image: str) -> list[str]:
|
||||
problems = [f"missing: {rel}" for rel in REQUIRED if not (path / rel).is_file()]
|
||||
agent = read_agent_yaml((path / "agent.yaml").read_text()) if (path / "agent.yaml").is_file() else None
|
||||
uses_code = bool(agent and agent["keys"] & set(CODE_KEYS))
|
||||
if (path / "app").exists() and not uses_code:
|
||||
problems.append("app-present: app/ (agent.yaml declares no functions, hooks or sub_agents that need it)")
|
||||
if uses_code and not (path / "app").is_dir():
|
||||
problems.append("missing: app/ (agent.yaml declares functions, hooks or sub_agents)")
|
||||
if agent is not None:
|
||||
problems += [f"sub-agent-prompt-missing: {rel}" for rel in agent["sub_agent_prompts"]
|
||||
if not (path / rel).is_file()]
|
||||
if agent.get("name") != path.name:
|
||||
problems.append(f"name: agent.yaml name {agent.get('name')!r} is not the folder name {path.name!r}")
|
||||
if not agent.get("display_name"):
|
||||
problems.append("name: agent.yaml has no display_name")
|
||||
version = str(agent.get("version", ""))
|
||||
if not SEMVER_OR_DEV.match(version):
|
||||
problems.append(f"version: {version!r} is neither #.#.# nor DEV")
|
||||
elif version != "DEV":
|
||||
problems += _release_notes(path, version)
|
||||
pointed = [agent.get("prompt") or "prompt.md"]
|
||||
for key in ("input", "output"):
|
||||
if not agent["contract"].get(key):
|
||||
problems.append(f"contract-file-missing: contract.{key} is not set")
|
||||
else:
|
||||
pointed.append(agent["contract"][key])
|
||||
problems += [f"contract-file-missing: {rel}" for rel in pointed if not (path / rel).is_file()]
|
||||
problems += [f"skill-missing: {s}" for s in agent["skills"]
|
||||
if isinstance(s, str) and not (path / "skills" / s / "SKILL.md").is_file()]
|
||||
problems += _dockerfile(path, runtime_image, agent)
|
||||
problems += _kubernetes(path)
|
||||
problems += _case_timeout(path)
|
||||
for f in sorted(p for p in path.rglob("*") if p.is_file()):
|
||||
rel = f.relative_to(path).as_posix()
|
||||
text = f.read_text(errors="replace")
|
||||
if SECRET.search(text):
|
||||
problems.append(f"secret: {rel}")
|
||||
return problems
|
||||
|
||||
|
||||
def _release_notes(path: Path, version: str) -> list[str]:
|
||||
notes = path / "RELEASE_NOTES.md"
|
||||
if not notes.is_file():
|
||||
return []
|
||||
heads = [line.rstrip() for line in notes.read_text().splitlines() if line.startswith("#")]
|
||||
if not heads or not re.match(rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}$", heads[0]):
|
||||
return [f"release-notes: the first heading must be '## [{version}] - YYYY-MM-DD'"]
|
||||
if len(heads) < 2 or heads[1] not in SECTIONS:
|
||||
return [f"release-notes: '## [{version}]' must be followed by one of {', '.join(SECTIONS)}"]
|
||||
return []
|
||||
|
||||
|
||||
def _dockerfile(path: Path, runtime_image: str, agent) -> list[str]:
|
||||
df = path / "Dockerfile"
|
||||
if not df.is_file():
|
||||
return []
|
||||
text = df.read_text()
|
||||
problems = []
|
||||
m = RUNTIME_ARG.search(text)
|
||||
if not m or m.group(1) != runtime_image or "FROM ${RUNTIME_IMAGE}" not in text:
|
||||
problems.append(f"runtime-unpinned: the Dockerfile must build FROM {runtime_image} "
|
||||
f"(found {m.group(1) if m else 'no RUNTIME_IMAGE'})")
|
||||
copies = "COPY skills/ /agent/skills/" in text
|
||||
if [s for s in (agent or {}).get("skills") or [] if isinstance(s, str)] and not copies:
|
||||
problems.append("skill-missing: the Dockerfile does not copy skills/")
|
||||
if copies and not (path / "skills").is_dir():
|
||||
problems.append("missing: skills/ (the Dockerfile copies it)")
|
||||
if (path / "app").is_dir() and "COPY app/ /agent/app/" not in text:
|
||||
problems.append("app-not-copied: the Dockerfile does not copy app/")
|
||||
if (agent or {}).get("sub_agent_prompts") and "COPY prompts/ /agent/prompts/" not in text:
|
||||
problems.append("sub-agent-prompt-missing: the Dockerfile does not copy prompts/")
|
||||
return problems
|
||||
|
||||
|
||||
def _kubernetes(path: Path) -> list[str]:
|
||||
problems = []
|
||||
for name in K8S:
|
||||
f = path / "kubernetes" / f"{name}.yaml"
|
||||
if f.is_file() and ("{{" in f.read_text() or "{%" in f.read_text()):
|
||||
problems.append(f"kubernetes: {name}.yaml has unfilled template fields")
|
||||
dep, cm = path / "kubernetes" / "deployment.yaml", path / "kubernetes" / "configmap.yaml"
|
||||
if dep.is_file():
|
||||
for marker in ("<placeholder_for_agent_name>", "<placeholder_for_image>"):
|
||||
if marker not in dep.read_text():
|
||||
problems.append(f"kubernetes: deployment.yaml lost Kyndryl CI's {marker}")
|
||||
if cm.is_file():
|
||||
text = cm.read_text()
|
||||
if 'PORT: "8080"' not in text:
|
||||
problems.append('kubernetes: configmap.yaml must set PORT: "8080"')
|
||||
if not re.search(r'^ LITELLM_MODEL: "[^"{]+"$', text, re.M):
|
||||
problems.append("kubernetes: configmap.yaml must set LITELLM_MODEL")
|
||||
return problems
|
||||
|
||||
|
||||
def _number(text: str):
|
||||
return int(text) if text.isdigit() else float(text)
|
||||
|
||||
|
||||
def _case_timeout(path: Path) -> list[str]:
|
||||
"""An agent whose configmap.yaml names a reasoning model has eval cases that wait at least CASE_TIMEOUT_FLOOR
|
||||
seconds for a reply: the eval runner waits 120 unless the cases say otherwise (a case's own `timeout`, else the
|
||||
file's `defaults.timeout`)."""
|
||||
cm, cases = path / "kubernetes" / "configmap.yaml", path / "eval" / "cases.yaml"
|
||||
model = re.search(r'^ LITELLM_MODEL: "([^"]+)"$', cm.read_text(), re.M) if cm.is_file() else None
|
||||
if not model or model.group(1) not in REASONING_MODELS or not cases.is_file():
|
||||
return []
|
||||
parts = re.split(r"^cases:[ \t]*$", cases.read_text(), maxsplit=1, flags=re.M)
|
||||
head, body = parts[0], parts[1] if len(parts) > 1 else ""
|
||||
block = re.search(r"^defaults:[ \t]*\n((?:[ \t]+.*(?:\n|$))*)", head, re.M)
|
||||
found = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", block.group(1), re.M) if block else None
|
||||
default = _number(found.group(1)) if found else None
|
||||
short = []
|
||||
for case in re.split(r"^- ", body, flags=re.M)[1:]:
|
||||
own = re.search(r"^ timeout: ([0-9.]+)[ \t]*$", case, re.M)
|
||||
wait = _number(own.group(1)) if own else (default or 120)
|
||||
if wait < CASE_TIMEOUT_FLOOR:
|
||||
name = re.match(r"id: ['\"]?([^'\"\n]+)", case)
|
||||
short.append(name.group(1).strip() if name else "?")
|
||||
if not short:
|
||||
return []
|
||||
return [f"case-timeout: {model.group(1)} is a reasoning model, and eval/cases.yaml waits "
|
||||
f"{default if default is not None else 'the runner default of 120'} s for a reply"
|
||||
f" (cases {', '.join(short[:3])}{'...' if len(short) > 3 else ''}): set defaults.timeout to at least "
|
||||
f"{CASE_TIMEOUT_FLOOR}"]
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--runtime-image", required=True)
|
||||
parser.add_argument("folders", nargs="+")
|
||||
args = parser.parse_args(argv)
|
||||
bad = 0
|
||||
for folder in args.folders:
|
||||
path = Path(folder).resolve()
|
||||
for p in check_folder(path, args.runtime_image):
|
||||
print(f"{path.name}: {p}")
|
||||
bad += 1
|
||||
print(f"{len(args.folders)} folders, {bad} problems")
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user