ALZ-F01 Discovery sub-agent pattern: report phase and fetch in code
Orchestrator report phase (ADR 0013) and fetching in code (ADR 0014), with the report renderer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
"""Write workflows/cp1-alz-discovery.json: the KAB workflow export for ALZ-F01 Discovery.
|
||||
|
||||
The same export format as cp4-agents' workflow (export_version 2.0.0). Four steps call the one orchestrator agent,
|
||||
one per phase. A step can pass only workflow inputs and earlier steps' outputs, so step 1 sends no phase (the agent
|
||||
one per phase. The report step stops for approval. A step can pass only workflow inputs and earlier steps' outputs, so step 1 sends no phase (the agent
|
||||
plans a new run) and each later step sends the previous step's next_phase. The critique step stops for the
|
||||
Engagement Lead's approval. The step schemas are the agent's own contract, read from its schemas/ folder, so run
|
||||
this again after changing them. Standard library only.
|
||||
@@ -39,6 +39,7 @@ PHASES = [
|
||||
("collect", "Ingest CMDB / ITSM records and scan the planned targets into one finding per scope entry."),
|
||||
("analyse", "Build the canonical inventory, CMDB drift, data classification, network map and dependency graph."),
|
||||
("critique", "Validate coverage, injection handling, evidence and tool-call integrity; the verdict is enforced in code."),
|
||||
("report", "Write the customer's report as Markdown and HTML, in code, from the run's records. No model is called."),
|
||||
]
|
||||
|
||||
|
||||
@@ -57,7 +58,7 @@ def build(with_upload: bool = True, artifact_ids=()) -> dict:
|
||||
steps, connections = list(front), []
|
||||
for index, (phase, description) in enumerate(PHASES):
|
||||
step_id = f"s{index + 1}"
|
||||
inputs = {"from_workflow": ["engagement_id", "run_id"]}
|
||||
inputs = {"from_workflow": ["engagement_id", "run_id"] + ([] if index else ["include_cmdb", "include_itsm"])}
|
||||
depends = [f"s{index}"] if index else []
|
||||
if index:
|
||||
inputs["from_steps"] = {"phase": f"s{index}.next_phase"}
|
||||
@@ -75,11 +76,11 @@ def build(with_upload: bool = True, artifact_ids=()) -> dict:
|
||||
depends.append(f["id"])
|
||||
connections.append({"source": f["id"], "target": step_id})
|
||||
hitl = {"enabled": False}
|
||||
if phase == "critique":
|
||||
if phase == "report":
|
||||
hitl = {
|
||||
"enabled": True,
|
||||
"checkpoint_type": "approval",
|
||||
"review_focus": "The Engagement Lead reviews the critic's verdict, declared gaps and SME queues before "
|
||||
"review_focus": "The Engagement Lead reads the report, with the verdict, declared gaps and SME queues, before "
|
||||
"Assessment starts. A fail verdict is never auto-approved.",
|
||||
"timeout_hours": 24,
|
||||
"allowed_actions": ["continue", "rerun", "abort"],
|
||||
|
||||
@@ -11,6 +11,8 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
agent_dir, storage = Path(sys.argv[1]), sys.argv[2]
|
||||
@@ -45,6 +47,24 @@ async def ask_sub_agent(name, request):
|
||||
|
||||
|
||||
rt.SubAgentFailed, rt.Reply, rt.ask_sub_agent = SubAgentFailed, Reply, ask_sub_agent
|
||||
rt_tools = types.ModuleType("kyndryl_agent_runtime.tools")
|
||||
|
||||
|
||||
class ToolError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
async def acall_tool(name, arguments=None):
|
||||
if arguments and arguments.get("scope_entry_id") == "SE-REFUSED":
|
||||
raise ToolError("refused by the server")
|
||||
if name == "get_call_log":
|
||||
return {"success": True, "data": {"calls": [{"seq": i} for i in range(1, 501)], "evidence_ref": "ev-log"}}
|
||||
return {"success": True, "data": {"tool": name, "arguments": arguments, "evidence_ref": f"ev-{name}"}}
|
||||
|
||||
|
||||
rt_tools.ToolError, rt_tools.acall_tool = ToolError, acall_tool
|
||||
rt.tools = rt_tools
|
||||
sys.modules["kyndryl_agent_runtime.tools"] = rt_tools
|
||||
rt.agent_manifest = lambda: _Manifest()
|
||||
sys.modules["kyndryl_agent_runtime"] = rt
|
||||
|
||||
@@ -123,11 +143,66 @@ check("collect before plan -> failed via pre_process Reply", out["status"] == "f
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
|
||||
check("no phase on a new run -> plan", workspace.current_run().phase == "plan")
|
||||
|
||||
# --- fetch: code stores what the tool returned ---------------------------------
|
||||
stored = asyncio.run(tools.fetch("call-log-test", [{"tool": "get_call_log", "arguments": {}}]))
|
||||
log = workspace.get(workspace.current_run(), stored["ref"])
|
||||
check("fetch stores a long result whole", len(log["content"]["data"]["calls"]) == 500)
|
||||
check("fetch records the tool's evidence_ref", log["evidence_refs"] == ["ev-log"])
|
||||
check("fetch gives a receipt, not the content", "content" not in stored and stored["calls"] == 1)
|
||||
many = asyncio.run(tools.fetch("scan-test", [{"tool": "scan_target", "arguments": {"scope_entry_id": s}} for s in ("SE-001", "SE-REFUSED")],
|
||||
key="scope_entry_id"))
|
||||
record = workspace.get(workspace.current_run(), many["ref"])["content"]
|
||||
check("fetch maps results by key", set(record) == {"SE-001", "SE-REFUSED"})
|
||||
check("a refused call is stored and reported", "refused" in record["SE-REFUSED"] and many["refused"][0]["tool"] == "scan_target")
|
||||
check("fetch refuses a tool the agent doesn't have",
|
||||
"error" in asyncio.run(tools.fetch("x", [{"tool": "delete_everything", "arguments": {}}])))
|
||||
check("fetch is write-once", "error" in asyncio.run(tools.fetch("scan-test", [{"tool": "get_scope", "arguments": {}}])))
|
||||
|
||||
|
||||
class _Declared:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
|
||||
class _Tool:
|
||||
function_declarations = [_Declared("fetch"), _Declared("scan_target")]
|
||||
|
||||
|
||||
class _Config:
|
||||
tools = [_Tool()]
|
||||
|
||||
|
||||
class _Request:
|
||||
tools_dict = {"fetch": 1, "scan_target": 2}
|
||||
config = _Config()
|
||||
|
||||
|
||||
class _Call:
|
||||
framework, native = "adk", (None, _Request())
|
||||
|
||||
def add_instruction(self, _text):
|
||||
pass
|
||||
|
||||
|
||||
bl.before_model(_Call())
|
||||
check("raw MCP tools are withheld from the model", list(_Request.tools_dict) == ["fetch"]
|
||||
and [d.name for d in _Config.tools[0].function_declarations] == ["fetch"])
|
||||
check("MCP_TOOLS matches agent.yaml", set(tools.MCP_TOOLS) == {t for ts in yaml.safe_load((agent_dir / "agent.yaml").read_text())["mcp"].values() for t in ts})
|
||||
SERVER = yaml.safe_load((agent_dir.resolve().parents[2] / "mcp-servers" / "alz-discovery-sample.server.yaml").read_text())
|
||||
check("every tool's arguments match the server's", all(tuple(p["name"] for p in s.get("parameters", [])) == tools.MCP_TOOLS[s["name"]]
|
||||
for s in SERVER["tools"]))
|
||||
wrong = asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site_id": "LON-DC1"}}]))
|
||||
check("a wrong argument is refused before any call, and names the right one", "site" in wrong.get("error", "") and "Nothing was called" in wrong["error"])
|
||||
check("a refused fetch stores nothing, so the name stays free", "error" not in asyncio.run(tools.fetch("wrong-arg", [{"tool": "collect_topology", "arguments": {"site": "LON-DC1"}}])))
|
||||
|
||||
# --- plan --------------------------------------------------------------------
|
||||
assert phase("plan") is None
|
||||
tools.workspace_put("scope", SCOPE, ["ev-1"])
|
||||
options = workspace.get(workspace.current_run(), "plan/run-options")
|
||||
check("run options stored from the run form, as the platform's", options["content"] == {"include_cmdb": "yes", "include_itsm": "yes"}
|
||||
and options["produced_by"] == "platform")
|
||||
tools.workspace_put("scope", {"success": True, "data": SCOPE}, ["ev-1"]) # MCP results are stored in this wrapper
|
||||
tools.workspace_put("sources", {"sources": []}, ["ev-2"])
|
||||
tools.workspace_put("budgets", {"scan_rps": 5}, ["ev-3"])
|
||||
tools.workspace_put("budgets", {"success": True, "data": {"scan_rps": 5}}, ["ev-3"])
|
||||
check("write-once refuses a second 'scope'", "error" in tools.workspace_put("scope", {}, []))
|
||||
ANSWERS["dsc_01_planner"] = "```json\n" + json.dumps(PLAN) + "\n```"
|
||||
res = asyncio.run(tools.delegate("dsc_01_planner", "Plan it.", ["plan/scope", "plan/sources", "plan/budgets"]))
|
||||
@@ -140,7 +215,7 @@ check(f"check_plan ok on a sound plan ({cp})", cp.get("ok") is True)
|
||||
out = bl.post_process({"status": "completed", "artifacts": [{"name": "forged"}], "summary": "s", "issues": []})
|
||||
check("plan post_process: completed, next collect", out["status"] == "completed" and out["next_phase"] == "collect")
|
||||
check("artifacts come from the manifest, not the model",
|
||||
{a["name"] for a in out["artifacts"]} == {"scope", "sources", "budgets", "task-graph"})
|
||||
{a["name"] for a in out["artifacts"]} == {"run-options", "scope", "sources", "budgets", "task-graph", "call-log-test", "scan-test", "wrong-arg"})
|
||||
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN}) is None
|
||||
check("no phase after plan -> collect", workspace.current_run().phase == "collect")
|
||||
@@ -170,6 +245,16 @@ out = bl.post_process({"status": "completed", "summary": "s", "issues": []})
|
||||
check("analyse: failed dsc_06 reported and status not completed",
|
||||
out["status"] == "completed_with_gaps" and any("dependency-graph" in i for i in out["issues"]))
|
||||
|
||||
# --- analyse: tool data alone is not an inventory --------------------------------
|
||||
sandbox = workspace.Run(engagement_id=ENG, run_id="RUN-DRY-NOINV", phase="analyse")
|
||||
workspace.put(sandbox, "inventory-sources", {"apps": []}, "orchestrator")
|
||||
check("orchestrator tool data never counts as the inventory", not bl._latest(sandbox, "analyse", "inventory"))
|
||||
check("so analyse with only tool data fails", bl._check_analyse(sandbox, []) == "failed")
|
||||
workspace.bind_run(workspace.Run(engagement_id=ENG, run_id=RUN, phase="analyse"))
|
||||
request = tools._request(workspace.current_run(), "dsc_03_inventory", "Do it.", ["collect/itsm-records", "collect/findings-b1"])
|
||||
check("a hand-off naming an unstored record goes ahead and says so",
|
||||
"Not stored in this run, so not available: collect/itsm-records" in request and "collect/findings-b1" in request)
|
||||
|
||||
# --- critique: the critic says pass, but SE-002 has no finding -------------------
|
||||
assert phase("critique") is None
|
||||
tools.workspace_put("scope-authoritative", SCOPE, ["ev-6"])
|
||||
@@ -188,8 +273,19 @@ check("critique: the unparseable network reply is a reason",
|
||||
|
||||
# --- the chain of phases when the request names none ----------------------------
|
||||
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
|
||||
check("no phase after critique -> refused, the run is finished",
|
||||
reply is not None and "already been critiqued" in json.loads(reply.text)["issues"][0])
|
||||
check("no phase after critique -> the report, answered without a model", reply is not None)
|
||||
done = json.loads(reply.text)
|
||||
check("report: the enforced fail is the verdict, though the critic said pass",
|
||||
done["report_data"]["verdict"] == "fail" and done["verdict"]["verdict"] == "fail")
|
||||
check("report: Markdown and HTML both say the run didn't pass",
|
||||
"didn't pass the critic" in done["report_markdown"] and "didn't pass the critic" in done["report_html"])
|
||||
check("report: the HTML is a draft, the Markdown is the record",
|
||||
"Not for release" in done["report_html"] and "Not for release" not in done["report_markdown"])
|
||||
check("report: stored as chained records", {"report/report-data", "report/report-markdown", "report/report-html"} <=
|
||||
{a["ref"] for a in done["artifacts"]} and not workspace.verify_chain(workspace.Run(engagement_id=ENG, run_id=RUN, phase="report")))
|
||||
reply = bl.pre_process({"engagement_id": ENG, "run_id": RUN})
|
||||
check("no phase after the report -> refused, the run is finished",
|
||||
reply is not None and "already finished" in json.loads(reply.text)["issues"][0])
|
||||
|
||||
RUN_BAD = "RUN-DRY-FAILED-PLAN"
|
||||
assert bl.pre_process({"engagement_id": ENG, "run_id": RUN_BAD}) is None # plan, inferred
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write the clean Discovery report for an approved run, from the report step's output.
|
||||
|
||||
Usage: render_discovery_report.py <step output.json> <report.md> [notice] also writes <report>.html; refuses to overwrite
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("report", ROOT / "cp1-alz-discovery-orchestrator-agent/app/report.py")
|
||||
report = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(report)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
source, target = Path(sys.argv[1]), Path(sys.argv[2])
|
||||
page = target.with_suffix(".html")
|
||||
if target.exists() or page.exists():
|
||||
raise SystemExit(f"{target} or {page} exists; reports are never overwritten")
|
||||
notice = sys.argv[3] if len(sys.argv) > 3 else None
|
||||
data = json.loads(source.read_text())["report_data"]
|
||||
target.write_text(report.render(data, notice))
|
||||
page.write_text(report.render_html(data, notice))
|
||||
print(f"wrote {target} and {page}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user