Files
gcp_solution_architecture_a…/tests/test_providers.py
Jonathan Boniface 5bb22d984e
Some checks failed
validation / verify (push) Failing after 8s
fixed multile issues
2026-10-07 17:33:36 +01:00

273 lines
8.8 KiB
Python

"""Cloud provider bundle loading and validation."""
from pathlib import Path
import pytest
import yaml
from app.providers import ProviderConfigError, load_provider
REPO_ROOT = Path(__file__).parents[1]
def test_gcp_bundle_loads_vendored_skills_and_template():
bundle = load_provider("gcp")
assert bundle.name == "gcp"
assert "google-cloud-solution-architecture" in bundle.skills
assert {name for name in bundle.skills if name.startswith("google-cloud-waf-")} == {
"google-cloud-waf-security",
"google-cloud-waf-reliability",
"google-cloud-waf-cost-optimization",
"google-cloud-waf-operational-excellence",
"google-cloud-waf-performance-optimization",
"google-cloud-waf-sustainability",
}
assert bundle.output_template == (
REPO_ROOT
/ "skills/vendor/google/google-cloud-solution-architecture/assets/output-template.md"
)
assert bundle.models.default
def test_gcp_bundle_mcp_servers_are_tiered_and_referenced_consistently():
bundle = load_provider("gcp")
general = {
server.id for server in bundle.mcp_servers.values() if server.tier == "general"
}
assert general == {"developer_knowledge"}
assert all(
server.url.startswith("https://") for server in bundle.mcp_servers.values()
)
for role, server_ids in bundle.role_mcp_servers.items():
assert set(server_ids) <= set(bundle.mcp_servers), role
def test_role_mcp_servers_can_exclude_preview_servers():
bundle = load_provider("gcp")
with_preview = {s.id for s in bundle.mcp_servers_for_role("requirements-analyst")}
without_preview = {
s.id
for s in bundle.mcp_servers_for_role(
"requirements-analyst", include_preview=False
)
}
assert "asset_inventory" in with_preview
assert "asset_inventory" not in without_preview
assert bundle.mcp_servers_for_role("no-such-role") == ()
def test_gcp_bundle_maps_roles_and_pillars_to_vendored_skills():
bundle = load_provider("gcp")
assert bundle.role_skills["orchestrator"] == ("google-cloud-solution-architecture",)
assert bundle.pillar_skills == {
"security": "google-cloud-waf-security",
"reliability": "google-cloud-waf-reliability",
"operational-excellence": "google-cloud-waf-operational-excellence",
"cost-optimization": "google-cloud-waf-cost-optimization",
"performance-optimization": "google-cloud-waf-performance-optimization",
"sustainability": "google-cloud-waf-sustainability",
}
def test_model_for_role_falls_back_to_default():
bundle = load_provider("gcp")
assert bundle.models.for_role("no-such-role") == bundle.models.default
def test_unknown_provider_is_rejected(tmp_path):
(tmp_path / "providers").mkdir()
with pytest.raises(ProviderConfigError, match="aws"):
load_provider("aws", root=tmp_path)
def minimal_bundle(tmp_path: Path, **overrides) -> dict:
skill_dir = tmp_path / "skills" / "alpha"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: alpha\ndescription: d\n---\n", encoding="utf-8"
)
(tmp_path / "template.md").write_text("# Template\n", encoding="utf-8")
config = {
"name": "test",
"skills_dirs": ["skills"],
"output_template": "template.md",
"models": {"default": "gemini/test-model"},
"tools": {"general": ["mermaid_check"], "cloud_read": []},
"mcp_auth": {"kind": "google_adc"},
"mcp_servers": {
"docs": {"url": "https://example.com/mcp", "tier": "general", "allow": []}
},
"role_mcp_servers": {"describer": ["docs"]},
}
config.update(overrides)
return config
def write_bundle(tmp_path: Path, config: dict) -> None:
(tmp_path / "providers").mkdir(exist_ok=True)
(tmp_path / "providers" / "test.yaml").write_text(
yaml.safe_dump(config), encoding="utf-8"
)
def test_minimal_bundle_loads(tmp_path):
write_bundle(tmp_path, minimal_bundle(tmp_path))
bundle = load_provider("test", root=tmp_path)
assert list(bundle.skills) == ["alpha"]
assert bundle.mcp_servers["docs"].allow == ()
assert bundle.mcp_servers["docs"].preview is False
@pytest.mark.parametrize(
("override", "message"),
[
({"name": "other"}, "name"),
({"skills_dirs": ["missing"]}, "skills_dirs"),
({"output_template": "missing.md"}, "output_template"),
({"models": {}}, "models.default"),
({"tools": {"mutating": ["terraform_apply"]}}, "tier"),
(
{
"mcp_servers": {
"docs": {
"url": "http://example.com/mcp",
"tier": "general",
"allow": [],
}
}
},
"https",
),
(
{
"mcp_servers": {
"docs": {
"url": "https://example.com/mcp",
"tier": "write",
"allow": [],
}
}
},
"tier",
),
(
{
"mcp_servers": {
"docs": {"url": "https://example.com/mcp", "tier": "general"}
}
},
"allow",
),
({"role_mcp_servers": {"describer": ["missing"]}}, "missing"),
({"role_skills": {"describer": ["no-such-skill"]}}, "no-such-skill"),
({"pillar_skills": {"security": "no-such-skill"}}, "no-such-skill"),
],
)
def test_invalid_bundles_are_rejected(tmp_path, override, message):
write_bundle(tmp_path, minimal_bundle(tmp_path, **override))
with pytest.raises(ProviderConfigError, match=message):
load_provider("test", root=tmp_path)
REVIEW = REPO_ROOT / "providers" / "reviews" / "gcp-mcp-tools.json"
def test_every_allowlisted_mcp_tool_was_reviewed_as_read_only():
import json
review = json.loads(REVIEW.read_text(encoding="utf-8"))
bundle = load_provider("gcp")
for server in bundle.mcp_servers.values():
assert server.allow, f"{server.id} exposes no tools"
reviewed = {tool["name"]: tool for tool in review[server.id]["tools"]}
for name in server.allow:
assert name in reviewed, (
f"{server.id}.{name} was not in the reviewed tools/list"
)
assert reviewed[name]["read_only"], f"{server.id}.{name} is not read-only"
def test_data_reading_and_secret_capable_tools_stay_excluded():
bundle = load_provider("gcp")
excluded = {
"cloud_sql": {"execute_sql", "execute_sql_readonly"},
"storage": {"read_object", "read_text", "list_objects"},
"gke": {"get_k8s_resource", "describe_k8s_resource"},
}
for server_id, tools in excluded.items():
assert not tools & set(bundle.mcp_servers[server_id].allow), server_id
def test_bundle_places_each_local_tool_in_its_own_tier():
from app.tools.local import LOCAL_TOOLS
bundle = load_provider("gcp")
for tier, names in bundle.tools.items():
for name in names:
assert LOCAL_TOOLS[name][0] == tier, name
def test_bundle_pins_terraform_providers_and_role_timeouts(tmp_path):
config = minimal_bundle(
tmp_path,
models={"default": "gemini/m", "timeout_seconds": {"target-architect": 420}},
terraform={
"required_providers": {
"cloud": {"source": "example/cloud", "version": "1.2.3"}
}
},
)
write_bundle(tmp_path, config)
bundle = load_provider("test", root=tmp_path)
assert bundle.terraform_providers == {
"cloud": {"source": "example/cloud", "version": "1.2.3"}
}
assert bundle.models.timeout_for("target-architect") == 420
assert bundle.models.timeout_for("decomposer") is None
@pytest.mark.parametrize(
"overrides, message",
[
(
{"terraform": {"required_providers": {"cloud": {"source": "x/cloud"}}}},
"needs a 'source' and an exact 'version'",
),
(
{"models": {"default": "gemini/m", "timeout_seconds": {"x": 0}}},
"must be positive",
),
(
{"models": {"default": "gemini/m", "timeout_seconds": {"x": "slow"}}},
"must be a number",
),
],
)
def test_bad_terraform_pins_and_timeouts_are_rejected(tmp_path, overrides, message):
write_bundle(tmp_path, minimal_bundle(tmp_path, **overrides))
with pytest.raises(ProviderConfigError, match=message):
load_provider("test", root=tmp_path)
def test_gcp_bundle_pins_one_exact_google_provider_version():
pin = load_provider("gcp").terraform_providers["google"]
assert pin["source"] == "hashicorp/google"
assert pin["version"].replace(".", "").isdigit() # exact, not a range