ALZ-F01 Discovery sub-agent pattern: report phase and fetch in code
Orchestrator report phase (ADR 0013) and fetching in code (ADR 0014), with the report renderer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,8 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
agent_dir, storage = Path(sys.argv[1]), sys.argv[2]
|
||||
@@ -45,6 +47,24 @@ async def ask_sub_agent(name, request):
|
||||
|
||||
|
||||
rt.SubAgentFailed, rt.Reply, rt.ask_sub_agent = SubAgentFailed, Reply, ask_sub_agent
|
||||
rt_tools = types.ModuleType("kyndryl_agent_runtime.tools")
|
||||
|
||||
|
||||
class ToolError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
async def acall_tool(name, arguments=None):
|
||||
if arguments and arguments.get("scope_entry_id") == "SE-REFUSED":
|
||||
raise ToolError("refused by the server")
|
||||
if name == "get_call_log":
|
||||
return {"success": True, "data": {"calls": [{"seq": i} for i in range(1, 501)], "evidence_ref": "ev-log"}}
|
||||
return {"success": True, "data": {"tool": name, "arguments": arguments, "evidence_ref": f"ev-{name}"}}
|
||||
|
||||
|
||||
rt_tools.ToolError, rt_tools.acall_tool = ToolError, acall_tool
|
||||
rt.tools = rt_tools
|
||||
sys.modules["kyndryl_agent_runtime.tools"] = rt_tools
|
||||
rt.agent_manifest = lambda: _Manifest()
|
||||
sys.modules["kyndryl_agent_runtime"] = rt
|
||||
|
||||
@@ -123,11 +143,66 @@ check("collect before plan -> failed via pre_process Reply", out["status"] == "f
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
|
||||
check("no phase on a new run -> plan", workspace.current_run().phase == "plan")
|
||||
|
||||
# --- fetch: code stores what the tool returned ---------------------------------
|
||||
stored = asyncio.run(tools.fetch("call-log-test", [{"tool": "get_call_log", "arguments": {}}]))
|
||||
log = workspace.get(workspace.current_run(), stored["ref"])
|
||||
check("fetch stores a long result whole", len(log["content"]["data"]["calls"]) == 500)
|
||||
check("fetch records the tool's evidence_ref", log["evidence_refs"] == ["ev-log"])
|
||||
check("fetch gives a receipt, not the content", "content" not in stored and stored["calls"] == 1)
|
||||
many = asyncio.run(tools.fetch("scan-test", [{"tool": "scan_target", "arguments": {"scope_entry_id": s}} for s in ("SE-001", "SE-REFUSED")],
|
||||
key="scope_entry_id"))
|
||||
record = workspace.get(workspace.current_run(), many["ref"])["content"]
|
||||
check("fetch maps results by key", set(record) == {"SE-001", "SE-REFUSED"})
|
||||
check("a refused call is stored and reported", "refused" in record["SE-REFUSED"] and many["refused"][0]["tool"] == "scan_target")
|
||||
check("fetch refuses a tool the agent doesn't have",
|
||||
"error" in asyncio.run(tools.fetch("x", [{"tool": "delete_everything", "arguments": {}}])))
|
||||
check("fetch is write-once", "error" in asyncio.run(tools.fetch("scan-test", [{"tool": "get_scope", "arguments": {}}])))
|
||||
|
||||
|
||||
class _Declared:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
|
||||
class _Tool:
|
||||
function_declarations = [_Declared("fetch"), _Declared("scan_target")]
|
||||
|
||||
|
||||
class _Config:
|
||||
tools = [_Tool()]
|
||||
|
||||
|
||||
class _Request:
|
||||
tools_dict = {"fetch": 1, "scan_target": 2}
|
||||
config = _Config()
|
||||
|
||||
|
||||
class _Call:
|
||||
framework, native = "adk", (None, _Request())
|
||||
|
||||
def add_instruction(self, _text):
|
||||
pass
|
||||
|
||||
|
||||
bl.before_model(_Call())
|
||||
check("raw MCP tools are withheld from the model", list(_Request.tools_dict) == ["fetch"]
|
||||
and [d.name for d in _Config.tools[0].function_declarations] == ["fetch"])
|
||||
check("MCP_TOOLS matches agent.yaml", set(tools.MCP_TOOLS) == {t for ts in yaml.safe_load((agent_dir / "agent.yaml").read_text())["mcp"].values() for t in ts})
|
||||
SERVER = yaml.safe_load((agent_dir.resolve().parents[2] / "mcp-servers" / "alz-discovery-sample.server.yaml").read_text())
|
||||
check("every tool's arguments match the server's", all(tuple(p["name"] for p in s.get("parameters", [])) == tools.MCP_TOOLS[s["name"]]
|
||||
for s in SERVER["tools"]))
|
||||
wrong = asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site_id": "LON-DC1"}}]))
|
||||
check("a wrong argument is refused before any call, and names the right one", "site" in wrong.get("error", "") and "Nothing was called" in wrong["error"])
|
||||
check("a refused fetch stores nothing, so the name stays free", "error" not in asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site": "LON-DC1"}}])))
|
||||
|
||||
# --- plan --------------------------------------------------------------------
|
||||
assert phase("plan") is None
|
||||
tools.workspace_put("scope", SCOPE, ["ev-1"])
|
||||
options = workspace.get(workspace.current_run(), "plan/run-options")
|
||||
check("run options stored from the run form, as the platform's", options["content"] == {"include_cmdb": "yes", "include_itsm": "yes"}
|
||||
and options["produced_by"] == "platform")
|
||||
tools.workspace_put("scope", {"success": True, "data": SCOPE}, ["ev-1"]) # MCP results are stored in this wrapper
|
||||
tools.workspace_put("sources", {"sources": []}, ["ev-2"])
|
||||
tools.workspace_put("budgets", {"scan_rps": 5}, ["ev-3"])
|
||||
tools.workspace_put("budgets", {"success": True, "data": {"scan_rps": 5}}, ["ev-3"])
|
||||
check("write-once refuses a second 'scope'", "error" in tools.workspace_put("scope", {}, []))
|
||||
ANSWERS["dsc_01_planner"] = "```json\n" + json.dumps(PLAN) + "\n```"
|
||||
res = asyncio.run(tools.delegate("dsc_01_planner", "Plan it.", ["plan/scope", "plan/sources", "plan/budgets"]))
|
||||
@@ -140,7 +215,7 @@ check(f"check_plan ok on a sound plan ({cp})", cp.get("ok") is True)
|
||||
out = bl.post_process({"status": "completed", "artifacts": [{"name": "forged"}], "summary": "s", "issues": []})
|
||||
check("plan post_process: completed, next collect", out["status"] == "completed" and out["next_phase"] == "collect")
|
||||
check("artifacts come from the manifest, not the model",
|
||||
{a["name"] for a in out["artifacts"]} == {"scope", "sources", "budgets", "task-graph"})
|
||||
{a["name"] for a in out["artifacts"]} == {"run-options", "scope", "sources", "budgets", "task-graph", "call-log-test", "scan-test", "wrong-arg"})
|
||||
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
|
||||
check("no phase after plan -> collect", workspace.current_run().phase == "collect")
|
||||
@@ -170,6 +245,16 @@ out = bl.post_process({"status": "completed", "summary": "s", "issues": []})
|
||||
check("analyse: failed dsc_06 reported and status not completed",
|
||||
out["status"] == "completed_with_gaps" and any("dependency-graph" in i for i in out["issues"]))
|
||||
|
||||
# --- analyse: tool data alone is not an inventory --------------------------------
|
||||
sandbox = workspace.Run(engagement_id=ENG, run_id="RUN-DRY-NOINV", phase="analyse")
|
||||
workspace.put(sandbox, "inventory-sources", {"apps": []}, "orchestrator")
|
||||
check("orchestrator tool data never counts as the inventory", not bl._latest(sandbox, "analyse", "inventory"))
|
||||
check("so analyse with only tool data fails", bl._check_analyse(sandbox, []) == "failed")
|
||||
workspace.bind_run(workspace.Run(engagement_id=ENG, run_id=RUN, phase="analyse"))
|
||||
request = tools._request(workspace.current_run(), "dsc_03_inventory", "Do it.", ["collect/itsm-records", "collect/findings-b1"])
|
||||
check("a hand-off naming an unstored record goes ahead and says so",
|
||||
"Not stored in this run, so not available: collect/itsm-records" in request and "collect/findings-b1" in request)
|
||||
|
||||
# --- critique: the critic says pass, but SE-002 has no finding -------------------
|
||||
assert phase("critique") is None
|
||||
tools.workspace_put("scope-authoritative", SCOPE, ["ev-6"])
|
||||
@@ -188,8 +273,19 @@ check("critique: the unparseable network reply is a reason",
|
||||
|
||||
# --- the chain of phases when the request names none ----------------------------
|
||||
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
|
||||
check("no phase after critique -> refused, the run is finished",
|
||||
reply is not None and "already been critiqued" in json.loads(reply.text)["issues"][0])
|
||||
check("no phase after critique -> the report, answered without a model", reply is not None)
|
||||
done = json.loads(reply.text)
|
||||
check("report: the enforced fail is the verdict, though the critic said pass",
|
||||
done["report_data"]["verdict"] == "fail" and done["verdict"]["verdict"] == "fail")
|
||||
check("report: Markdown and HTML both say the run didn't pass",
|
||||
"didn't pass the critic" in done["report_markdown"] and "didn't pass the critic" in done["report_html"])
|
||||
check("report: the HTML is a draft, the Markdown is the record",
|
||||
"Not for release" in done["report_html"] and "Not for release" not in done["report_markdown"])
|
||||
check("report: stored as chained records", {"report/report-data", "report/report-markdown", "report/report-html"} <=
|
||||
{a["ref"] for a in done["artifacts"]} and not workspace.verify_chain(workspace.Run(engagement_id=ENG, run_id=RUN, phase="report")))
|
||||
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
|
||||
check("no phase after the report -> refused, the run is finished",
|
||||
reply is not None and "already finished" in json.loads(reply.text)["issues"][0])
|
||||
|
||||
RUN_BAD = "RUN-DRY-FAILED-PLAN"
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN_BAD}) is None # plan, inferred
|
||||
|
||||
Reference in New Issue
Block a user