Files
cp1-agents/outcome-pattern/solution_architecture/cp1-solution-architecture-agent/tests/test_tools.py
Jonathan Boniface bc84ee4df6
Some checks failed
check agents / check (push) Failing after 13s
Solution architecture outcome pattern: agent and its MCP servers
The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate
stream from the individual and sub-agent patterns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

90 lines
3.4 KiB
Python

"""What a role gets on KAB: its own Skills, and the MCP tools of its bundle."""
from app.core.providers import load_provider
from app.engine.skill_tools import skill_tools
from app.mcp_tools import McpCatalog
BUNDLE = load_provider("gcp")
SKILLS = {name: meta.path for name, meta in BUNDLE.skills.items()}
def tools(allowed):
return {tool.name: tool for tool in skill_tools(SKILLS, allowed)}
async def test_a_role_reads_its_own_skill_and_no_other():
own = tools(("requirements-chain",))
read = await own["load_skill"].handler(skill_name="requirements-chain")
other = await own["load_skill"].handler(skill_name="scoping")
assert "Requirements" in read.message or "requirements" in read.message
assert other.message.startswith("Refused")
async def test_a_skills_files_can_be_read_and_nothing_outside_them():
own = tools(("google-cloud-iac-patterns",))
listing = (await own["load_skill"].handler(skill_name="google-cloud-iac-patterns")).message
name = listing.rsplit(": ", 1)[1].split(", ")[0]
read = await own["load_skill_resource"].handler(skill_name="google-cloud-iac-patterns", path=name)
escape = await own["load_skill_resource"].handler(
skill_name="google-cloud-iac-patterns", path="../../agent.yaml"
)
assert not read.message.startswith("Refused")
assert escape.message.startswith("Refused")
def test_a_role_gets_its_bundles_tools_with_the_servers_names():
catalog = McpCatalog(BUNDLE, docs_search=True, call=None)
names = {tool.name for tool in catalog.tools_for(BUNDLE.role_mcp_servers["product-selector"])}
assert "gcp_developer_knowledge_search_documents" in names and "gcp_describe_tool" in names
def test_free_text_search_is_off_until_a_deployment_turns_it_on():
catalog = McpCatalog(BUNDLE, call=None)
assert catalog.tools_for(BUNDLE.role_mcp_servers["product-selector"]) == []
async def test_a_tools_arguments_are_passed_through_and_a_failure_is_a_gap():
calls = []
async def call(name, arguments):
calls.append((name, arguments))
if arguments.get("tool_arguments", {}).get("project") == "bad":
raise RuntimeError("secret detail")
return {"success": True, "data": {"items": []}}
catalog = McpCatalog(BUNDLE, call=call)
tool = next(t for t in catalog.tools_for(("compute",)) if t.name == "gcp_compute_list_instances")
good = await tool.handler(project="p1")
bad = await tool.handler(project="bad")
assert calls[0] == ("gcp_compute_list_instances", {"tool_arguments": {"project": "p1"}})
assert "items" in good.message
assert bad.message.startswith("Discovery gap") and "secret" not in bad.message
async def test_a_servers_refusal_is_a_gap():
async def call(name, arguments):
return {"success": False, "error": {"code": "not_read_only", "message": "refused"}}
tool = McpCatalog(BUNDLE, call=call).tools_for(("compute",))[0]
assert (await tool.handler()).message.startswith("Discovery gap")
def test_every_allowed_tool_is_one_the_server_defines():
import yaml
from pathlib import Path
definition = Path(__file__).resolve().parents[2] / "mcp-servers" / "sa-gcp-cloud.server.yaml"
offered = {tool["name"] for tool in yaml.safe_load(definition.read_text())["tools"]}
wanted = {
BUNDLE.mcp_tool_name(server_id, tool)
for server_id, server in BUNDLE.mcp_servers.items()
for tool in server.allow
}
assert wanted <= offered and BUNDLE.mcp_describe_tool in offered