Files
gcp_solution_architecture_a…/tests/agents/test_roles.py
Jonathan Boniface ee1ba38867 feat: a design-time coverage review, measured and left off
A reviewer role reads the requirements and recommendations against the Checks
when the Target architecture is first submitted. Three runs of case 03: it did
not stop the validator sending the design back, and the designs it reviewed
ended with 87 and 93 Checks against 52, so they cost more. It stays behind
DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13).

Also: the schema review no longer reads `true` or `false` in backticks as a
setting name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 12:33:06 +01:00

183 lines
6.3 KiB
Python

"""Role files (agents/*.md) resolved against a Cloud provider bundle."""
from pathlib import Path
import pytest
from app.adk.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
from app.providers import load_provider
from app.workflows.deliverables import DEPENDENCY_GRAPH
AGENTS_DIR = Path(__file__).parents[2] / "agents"
@pytest.fixture(scope="module")
def bundle():
return load_provider("gcp")
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
roles = load_roles(AGENTS_DIR, bundle)
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
# The lesson curator runs outside any Execution, and the coverage reviewer only
# when the deployment switches its review on; the engine needs neither.
assert set(roles) == expected | {"lesson-curator", "coverage-reviewer"}
for role in roles.values():
assert role.instructions.strip()
def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
# Without lessons: what the bundle's role_skills assigns, whatever has been learned.
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
assert roles["orchestrator"].skills == ("google-cloud-solution-architecture",)
assert roles[PILLAR_ADVISOR].skills == ()
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
assert roles["describer"].model == bundle.models.default
assert roles["target-architect"].skills == (
"google-cloud-solution-architecture",
"requirements-chain",
)
assert roles["target-architect"].mcp_servers == ("developer_knowledge",)
assert roles["orchestrator"].model == bundle.models.for_role("orchestrator")
assert roles["decomposer"].model == bundle.models.default
def test_roles_name_only_the_local_tools_they_need(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["iac-author"].tools == (
"terraform_fmt",
"terraform_validate",
"terraform_schema",
"terraform_module",
)
# The architect only looks settings up; it writes and runs no code.
assert roles["target-architect"].tools == ("terraform_schema",)
assert roles["validator"].tools == (
"terraform_validate",
"terraform_plan",
"terraform_schema",
"terraform_module",
)
assert roles["requirements-analyst"].tools == ()
def test_roles_with_long_replies_are_given_longer_for_a_model_request(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["target-architect"].model_timeout_seconds == 420
assert roles["validator"].model_timeout_seconds == 300
assert roles["decomposer"].model_timeout_seconds is None
def test_the_code_author_is_given_the_patterns_for_its_cloud_provider(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert "google-cloud-iac-patterns" in roles["iac-author"].skills
assert "google-cloud-iac-patterns" not in roles["validator"].skills
def test_preview_mcp_servers_can_be_excluded(bundle):
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
assert "asset_inventory" not in roles["requirements-analyst"].mcp_servers
def test_budgets_default_by_role_kind(bundle):
roles = load_roles(AGENTS_DIR, bundle, orchestrator_budget=7, specialist_budget=3)
assert roles["orchestrator"].budget == 7
assert roles["packager"].budget == 3
def test_specialists_include_the_shared_protocol_once(bundle):
roles = load_roles(AGENTS_DIR, bundle)
protocol = (AGENTS_DIR / "partials" / "specialist-protocol.md").read_text(
encoding="utf-8"
)
for spec in DEPENDENCY_GRAPH.values():
assert roles[spec.specialist].instructions.count(protocol.strip()) == 1
assert protocol.strip() not in roles["orchestrator"].instructions
def test_role_files_name_no_provider_specific_skills():
for path in AGENTS_DIR.rglob("*.md"):
text = path.read_text(encoding="utf-8").lower()
assert "google-cloud-" not in text, path.name
# --- validation -----------------------------------------------------------------------
def write_agents(tmp_path: Path, bundle, overrides: dict[str, str]) -> Path:
"""Copy the real role files, then overwrite some."""
target = tmp_path / "agents"
for path in AGENTS_DIR.rglob("*.md"):
destination = target / path.relative_to(AGENTS_DIR)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8")
for name, text in overrides.items():
(target / name).write_text(text, encoding="utf-8")
return target
@pytest.mark.parametrize(
("overrides", "message"),
[
({"decomposer.md": "---\nrole: other\n---\nBody"}, "match"),
(
{
"decomposer.md": "---\nrole: decomposer\ndeliverable: requirements\n---\nB"
},
"requirements",
),
({"decomposer.md": "---\nrole: decomposer\n---\nBody"}, "deliverable"),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\ntools: [terraform_apply]\n---\nB"
},
"terraform_apply",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\nincludes: [missing]\n---\nB"
},
"missing",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\nskills: [x]\n---\nB"
},
"skills",
),
(
{
"decomposer.md": "---\nrole: decomposer\n"
"deliverable: technical_decomposition\n---\n "
},
"instructions",
),
],
)
def test_invalid_role_files_are_rejected(tmp_path, bundle, overrides, message):
agents_dir = write_agents(tmp_path, bundle, overrides)
with pytest.raises(RoleConfigError, match=message):
load_roles(agents_dir, bundle)
def test_missing_role_file_is_rejected(tmp_path, bundle):
agents_dir = write_agents(tmp_path, bundle, {})
(agents_dir / "packager.md").unlink()
with pytest.raises(RoleConfigError, match="packager"):
load_roles(agents_dir, bundle)