ALZ-F01 Discovery sub-agent pattern: report phase and fetch in code

Orchestrator report phase (ADR 0013) and fetching in code (ADR 0014), with the report renderer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent e22d037ab0
commit 8bb62a4a42
33 changed files with 1449 additions and 128 deletions

View File

@@ -11,6 +11,8 @@ import json
import os
import sys
import types
import yaml
from pathlib import Path
agent_dir, storage = Path(sys.argv[1]), sys.argv[2]
@@ -45,6 +47,24 @@ async def ask_sub_agent(name, request):
rt.SubAgentFailed, rt.Reply, rt.ask_sub_agent = SubAgentFailed, Reply, ask_sub_agent
rt_tools = types.ModuleType("kyndryl_agent_runtime.tools")
class ToolError(RuntimeError):
pass
async def acall_tool(name, arguments=None):
if arguments and arguments.get("scope_entry_id") == "SE-REFUSED":
raise ToolError("refused by the server")
if name == "get_call_log":
return {"success": True, "data": {"calls": [{"seq": i} for i in range(1, 501)], "evidence_ref": "ev-log"}}
return {"success": True, "data": {"tool": name, "arguments": arguments, "evidence_ref": f"ev-{name}"}}
rt_tools.ToolError, rt_tools.acall_tool = ToolError, acall_tool
rt.tools = rt_tools
sys.modules["kyndryl_agent_runtime.tools"] = rt_tools
rt.agent_manifest = lambda: _Manifest()
sys.modules["kyndryl_agent_runtime"] = rt
@@ -123,11 +143,66 @@ check("collect before plan -> failed via pre_process Reply", out["status"] == "f
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
check("no phase on a new run -> plan", workspace.current_run().phase == "plan")
# --- fetch: code stores what the tool returned ---------------------------------
stored = asyncio.run(tools.fetch("call-log-test", [{"tool": "get_call_log", "arguments": {}}]))
log = workspace.get(workspace.current_run(), stored["ref"])
check("fetch stores a long result whole", len(log["content"]["data"]["calls"]) == 500)
check("fetch records the tool's evidence_ref", log["evidence_refs"] == ["ev-log"])
check("fetch gives a receipt, not the content", "content" not in stored and stored["calls"] == 1)
many = asyncio.run(tools.fetch("scan-test", [{"tool": "scan_target", "arguments": {"scope_entry_id": s}} for s in ("SE-001", "SE-REFUSED")],
key="scope_entry_id"))
record = workspace.get(workspace.current_run(), many["ref"])["content"]
check("fetch maps results by key", set(record) == {"SE-001", "SE-REFUSED"})
check("a refused call is stored and reported", "refused" in record["SE-REFUSED"] and many["refused"][0]["tool"] == "scan_target")
check("fetch refuses a tool the agent doesn't have",
"error" in asyncio.run(tools.fetch("x", [{"tool": "delete_everything", "arguments": {}}])))
check("fetch is write-once", "error" in asyncio.run(tools.fetch("scan-test", [{"tool": "get_scope", "arguments": {}}])))
class _Declared:
def __init__(self, name):
self.name = name
class _Tool:
function_declarations = [_Declared("fetch"), _Declared("scan_target")]
class _Config:
tools = [_Tool()]
class _Request:
tools_dict = {"fetch": 1, "scan_target": 2}
config = _Config()
class _Call:
framework, native = "adk", (None, _Request())
def add_instruction(self, _text):
pass
bl.before_model(_Call())
check("raw MCP tools are withheld from the model", list(_Request.tools_dict) == ["fetch"]
and [d.name for d in _Config.tools[0].function_declarations] == ["fetch"])
check("MCP_TOOLS matches agent.yaml", set(tools.MCP_TOOLS) == {t for ts in yaml.safe_load((agent_dir / "agent.yaml").read_text())["mcp"].values() for t in ts})
SERVER = yaml.safe_load((agent_dir.resolve().parents[2] / "mcp-servers" / "alz-discovery-sample.server.yaml").read_text())
check("every tool's arguments match the server's", all(tuple(p["name"] for p in s.get("parameters", [])) == tools.MCP_TOOLS[s["name"]]
for s in SERVER["tools"]))
wrong = asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site_id": "LON-DC1"}}]))
check("a wrong argument is refused before any call, and names the right one", "site" in wrong.get("error", "") and "Nothing was called" in wrong["error"])
check("a refused fetch stores nothing, so the name stays free", "error" not in asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site": "LON-DC1"}}])))
# --- plan --------------------------------------------------------------------
assert phase("plan") is None
tools.workspace_put("scope", SCOPE, ["ev-1"])
options = workspace.get(workspace.current_run(), "plan/run-options")
check("run options stored from the run form, as the platform's", options["content"] == {"include_cmdb": "yes", "include_itsm": "yes"}
and options["produced_by"] == "platform")
tools.workspace_put("scope", {"success": True, "data": SCOPE}, ["ev-1"]) # MCP results are stored in this wrapper
tools.workspace_put("sources", {"sources": []}, ["ev-2"])
tools.workspace_put("budgets", {"scan_rps": 5}, ["ev-3"])
tools.workspace_put("budgets", {"success": True, "data": {"scan_rps": 5}}, ["ev-3"])
check("write-once refuses a second 'scope'", "error" in tools.workspace_put("scope", {}, []))
ANSWERS["dsc_01_planner"] = "```json\n" + json.dumps(PLAN) + "\n```"
res = asyncio.run(tools.delegate("dsc_01_planner", "Plan it.", ["plan/scope", "plan/sources", "plan/budgets"]))
@@ -140,7 +215,7 @@ check(f"check_plan ok on a sound plan ({cp})", cp.get("ok") is True)
out = bl.post_process({"status": "completed", "artifacts": [{"name": "forged"}], "summary": "s", "issues": []})
check("plan post_process: completed, next collect", out["status"] == "completed" and out["next_phase"] == "collect")
check("artifacts come from the manifest, not the model",
{a["name"] for a in out["artifacts"]} == {"scope", "sources", "budgets", "task-graph"})
{a["name"] for a in out["artifacts"]} == {"run-options", "scope", "sources", "budgets", "task-graph", "call-log-test", "scan-test", "wrong-arg"})
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
check("no phase after plan -> collect", workspace.current_run().phase == "collect")
@@ -170,6 +245,16 @@ out = bl.post_process({"status": "completed", "summary": "s", "issues": []})
check("analyse: failed dsc_06 reported and status not completed",
out["status"] == "completed_with_gaps" and any("dependency-graph" in i for i in out["issues"]))
# --- analyse: tool data alone is not an inventory --------------------------------
sandbox = workspace.Run(engagement_id=ENG, run_id="RUN-DRY-NOINV", phase="analyse")
workspace.put(sandbox, "inventory-sources", {"apps": []}, "orchestrator")
check("orchestrator tool data never counts as the inventory", not bl._latest(sandbox, "analyse", "inventory"))
check("so analyse with only tool data fails", bl._check_analyse(sandbox, []) == "failed")
workspace.bind_run(workspace.Run(engagement_id=ENG, run_id=RUN, phase="analyse"))
request = tools._request(workspace.current_run(), "dsc_03_inventory", "Do it.", ["collect/itsm-records", "collect/findings-b1"])
check("a hand-off naming an unstored record goes ahead and says so",
"Not stored in this run, so not available: collect/itsm-records" in request and "collect/findings-b1" in request)
# --- critique: the critic says pass, but SE-002 has no finding -------------------
assert phase("critique") is None
tools.workspace_put("scope-authoritative", SCOPE, ["ev-6"])
@@ -188,8 +273,19 @@ check("critique: the unparseable network reply is a reason",
# --- the chain of phases when the request names none ----------------------------
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
check("no phase after critique -> refused, the run is finished",
reply is not None and "already been critiqued" in json.loads(reply.text)["issues"][0])
check("no phase after critique -> the report, answered without a model", reply is not None)
done = json.loads(reply.text)
check("report: the enforced fail is the verdict, though the critic said pass",
done["report_data"]["verdict"] == "fail" and done["verdict"]["verdict"] == "fail")
check("report: Markdown and HTML both say the run didn't pass",
"didn't pass the critic" in done["report_markdown"] and "didn&#x27;t pass the critic" in done["report_html"])
check("report: the HTML is a draft, the Markdown is the record",
"Not for release" in done["report_html"] and "Not for release" not in done["report_markdown"])
check("report: stored as chained records", {"report/report-data", "report/report-markdown", "report/report-html"} <=
{a["ref"] for a in done["artifacts"]} and not workspace.verify_chain(workspace.Run(engagement_id=ENG, run_id=RUN, phase="report")))
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
check("no phase after the report -> refused, the run is finished",
reply is not None and "already finished" in json.loads(reply.text)["issues"][0])
RUN_BAD = "RUN-DRY-FAILED-PLAN"
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN_BAD}) is None # plan, inferred