A reviewer role reads the requirements and recommendations against the Checks when the Target architecture is first submitted. Three runs of case 03: it did not stop the validator sending the design back, and the designs it reviewed ended with 87 and 93 Checks against 52, so they cost more. It stays behind DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13). Also: the schema review no longer reads `true` or `false` in backticks as a setting name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
183 lines
6.3 KiB
Python
183 lines
6.3 KiB
Python
"""Role files (agents/*.md) resolved against a Cloud provider bundle."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.adk.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
|
|
from app.providers import load_provider
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
|
|
|
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def bundle():
|
|
return load_provider("gcp")
|
|
|
|
|
|
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
|
|
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
|
|
# The lesson curator runs outside any Execution, and the coverage reviewer only
|
|
# when the deployment switches its review on; the engine needs neither.
|
|
assert set(roles) == expected | {"lesson-curator", "coverage-reviewer"}
|
|
for role in roles.values():
|
|
assert role.instructions.strip()
|
|
|
|
|
|
def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
|
|
# Without lessons: what the bundle's role_skills assigns, whatever has been learned.
|
|
roles = load_roles(AGENTS_DIR, bundle, lessons=False)
|
|
|
|
assert roles["orchestrator"].skills == ("google-cloud-solution-architecture",)
|
|
assert roles[PILLAR_ADVISOR].skills == ()
|
|
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
|
|
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
|
|
assert roles["describer"].model == bundle.models.default
|
|
assert roles["target-architect"].skills == (
|
|
"google-cloud-solution-architecture",
|
|
"requirements-chain",
|
|
)
|
|
assert roles["target-architect"].mcp_servers == ("developer_knowledge",)
|
|
assert roles["orchestrator"].model == bundle.models.for_role("orchestrator")
|
|
assert roles["decomposer"].model == bundle.models.default
|
|
|
|
|
|
def test_roles_name_only_the_local_tools_they_need(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert roles["iac-author"].tools == (
|
|
"terraform_fmt",
|
|
"terraform_validate",
|
|
"terraform_schema",
|
|
"terraform_module",
|
|
)
|
|
# The architect only looks settings up; it writes and runs no code.
|
|
assert roles["target-architect"].tools == ("terraform_schema",)
|
|
assert roles["validator"].tools == (
|
|
"terraform_validate",
|
|
"terraform_plan",
|
|
"terraform_schema",
|
|
"terraform_module",
|
|
)
|
|
assert roles["requirements-analyst"].tools == ()
|
|
|
|
|
|
def test_roles_with_long_replies_are_given_longer_for_a_model_request(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert roles["target-architect"].model_timeout_seconds == 420
|
|
assert roles["validator"].model_timeout_seconds == 300
|
|
assert roles["decomposer"].model_timeout_seconds is None
|
|
|
|
|
|
def test_the_code_author_is_given_the_patterns_for_its_cloud_provider(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
assert "google-cloud-iac-patterns" in roles["iac-author"].skills
|
|
assert "google-cloud-iac-patterns" not in roles["validator"].skills
|
|
|
|
|
|
def test_preview_mcp_servers_can_be_excluded(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
|
|
|
|
assert "asset_inventory" not in roles["requirements-analyst"].mcp_servers
|
|
|
|
|
|
def test_budgets_default_by_role_kind(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle, orchestrator_budget=7, specialist_budget=3)
|
|
|
|
assert roles["orchestrator"].budget == 7
|
|
assert roles["packager"].budget == 3
|
|
|
|
|
|
def test_specialists_include_the_shared_protocol_once(bundle):
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
protocol = (AGENTS_DIR / "partials" / "specialist-protocol.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
for spec in DEPENDENCY_GRAPH.values():
|
|
assert roles[spec.specialist].instructions.count(protocol.strip()) == 1
|
|
assert protocol.strip() not in roles["orchestrator"].instructions
|
|
|
|
|
|
def test_role_files_name_no_provider_specific_skills():
|
|
for path in AGENTS_DIR.rglob("*.md"):
|
|
text = path.read_text(encoding="utf-8").lower()
|
|
assert "google-cloud-" not in text, path.name
|
|
|
|
|
|
# --- validation -----------------------------------------------------------------------
|
|
|
|
|
|
def write_agents(tmp_path: Path, bundle, overrides: dict[str, str]) -> Path:
|
|
"""Copy the real role files, then overwrite some."""
|
|
target = tmp_path / "agents"
|
|
for path in AGENTS_DIR.rglob("*.md"):
|
|
destination = target / path.relative_to(AGENTS_DIR)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8")
|
|
for name, text in overrides.items():
|
|
(target / name).write_text(text, encoding="utf-8")
|
|
return target
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("overrides", "message"),
|
|
[
|
|
({"decomposer.md": "---\nrole: other\n---\nBody"}, "match"),
|
|
(
|
|
{
|
|
"decomposer.md": "---\nrole: decomposer\ndeliverable: requirements\n---\nB"
|
|
},
|
|
"requirements",
|
|
),
|
|
({"decomposer.md": "---\nrole: decomposer\n---\nBody"}, "deliverable"),
|
|
(
|
|
{
|
|
"decomposer.md": "---\nrole: decomposer\n"
|
|
"deliverable: technical_decomposition\ntools: [terraform_apply]\n---\nB"
|
|
},
|
|
"terraform_apply",
|
|
),
|
|
(
|
|
{
|
|
"decomposer.md": "---\nrole: decomposer\n"
|
|
"deliverable: technical_decomposition\nincludes: [missing]\n---\nB"
|
|
},
|
|
"missing",
|
|
),
|
|
(
|
|
{
|
|
"decomposer.md": "---\nrole: decomposer\n"
|
|
"deliverable: technical_decomposition\nskills: [x]\n---\nB"
|
|
},
|
|
"skills",
|
|
),
|
|
(
|
|
{
|
|
"decomposer.md": "---\nrole: decomposer\n"
|
|
"deliverable: technical_decomposition\n---\n "
|
|
},
|
|
"instructions",
|
|
),
|
|
],
|
|
)
|
|
def test_invalid_role_files_are_rejected(tmp_path, bundle, overrides, message):
|
|
agents_dir = write_agents(tmp_path, bundle, overrides)
|
|
|
|
with pytest.raises(RoleConfigError, match=message):
|
|
load_roles(agents_dir, bundle)
|
|
|
|
|
|
def test_missing_role_file_is_rejected(tmp_path, bundle):
|
|
agents_dir = write_agents(tmp_path, bundle, {})
|
|
(agents_dir / "packager.md").unlink()
|
|
|
|
with pytest.raises(RoleConfigError, match="packager"):
|
|
load_roles(agents_dir, bundle)
|