Files
cp1-agents/outcome-pattern/solution_architecture/cp1-solution-architecture-agent/tests/engine/test_overlay_skills.py
Jonathan Boniface bc84ee4df6
Some checks failed
check agents / check (push) Failing after 13s
Solution architecture outcome pattern: agent and its MCP servers
The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate
stream from the individual and sub-agent patterns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:13:25 +00:00

161 lines
5.6 KiB
Python

"""Overlay Skills (ADR-0004): an organisation's standards, named per Execution.
They come only from folders the deployment configures. A caller names one; it can
never supply one, and an unknown name refuses the start.
"""
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from app.engine.fake_runtime import FakeAgentRuntime, sample_submission
from app.engine.runtime import NoDecider, RoleSpec
from app.assembly import _overlays
from app.engine.agent import ExecutionEngine
from app.engine.router import Router, Thresholds
from app.core.providers import load_provider
from app.engine.recorder import InMemoryRecorder
from app.core.deliverables import DEPENDENCY_GRAPH
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist, "Produce.", budget=15, skills=("provider-skill",)
)
for spec in DEPENDENCY_GRAPH.values()
},
}
OVERLAYS = {"acme-standards": "abc123", "project-x": "def456"}
class Recording(FakeAgentRuntime):
def __init__(self, turns):
super().__init__(turns)
self.skills: dict[str, tuple[str, ...]] = {}
async def run(self, role, task, tools):
self.skills[role.name] = role.skills
return await super().run(role, task, tools)
def engine(tmp_path: Path, **options) -> tuple[ExecutionEngine, Recording]:
runtime = Recording(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
)
built = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
overlays=OVERLAYS,
**options,
)
return built, runtime
async def test_every_specialist_loads_the_overlays_the_execution_names(tmp_path):
built, runtime = engine(tmp_path)
result = await built.start(
"Ingest webhooks",
customer_case="acme",
pre_approved=True,
execution_id="v1",
overlay_skills=["acme-standards"],
)
assert result.status == "completed"
assert set(runtime.skills.values()) == {("provider-skill", "acme-standards")}
analyst = next(run for run in runtime.runs if run.role == "requirements-analyst")
assert analyst.task.context["overlay_skills"] == ["acme-standards"]
# The Execution records which overlays it loaded, and the version of each.
assert result.state["overlay_skills"] == {"acme-standards": "abc123"}
manifest = json.loads(
(tmp_path / "deliverables/cases/acme/executions/v1/execution.json").read_text()
)
assert manifest["overlay_skills"] == {"acme-standards": "abc123"}
lessons = json.loads((tmp_path / "lessons/candidates/acme/v1.json").read_text())
assert lessons["skill_versions"]["acme-standards"] == "abc123"
async def test_an_unknown_overlay_refuses_the_start(tmp_path):
built, runtime = engine(tmp_path)
with pytest.raises(ValueError, match="Unknown overlay Skills: evil-skill"):
await built.start("Ingest webhooks", overlay_skills=["evil-skill"])
with pytest.raises(ValueError, match="must be a list of Skill names"):
await built.start("Ingest webhooks", overlay_skills="acme-standards")
assert runtime.runs == []
assert not (tmp_path / "deliverables").exists()
async def test_the_deployments_default_overlays_are_loaded_with_the_callers(tmp_path):
built, runtime = engine(tmp_path, default_overlays=["acme-standards"])
result = await built.start(
"Ingest webhooks", pre_approved=True, overlay_skills=["project-x"]
)
assert list(result.state["overlay_skills"]) == ["acme-standards", "project-x"]
assert runtime.skills["iac-author"] == (
"provider-skill",
"acme-standards",
"project-x",
)
async def test_without_overlays_a_role_has_only_its_own_skills(tmp_path):
built, runtime = engine(tmp_path)
result = await built.start("Ingest webhooks", pre_approved=True)
assert result.state["overlay_skills"] == {}
assert set(runtime.skills.values()) == {("provider-skill",)}
def test_a_default_overlay_the_deployment_does_not_have_is_an_error(tmp_path):
with pytest.raises(ValueError, match="Unknown default overlay Skills: nope"):
engine(tmp_path, default_overlays=["nope"])
# --- where overlays come from ------------------------------------------------------------
def overlay(folder: Path, name: str) -> None:
(folder / name).mkdir(parents=True)
(folder / name / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: Standards for {name}.\n---\n# {name}\n"
)
def test_overlays_are_found_only_in_the_folders_the_deployment_names(tmp_path, monkeypatch):
overlay(tmp_path / "overlays", "acme-standards")
overlay(tmp_path / "elsewhere", "not-configured")
monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays"))
found = _overlays(load_provider("gcp"))
assert found["acme-standards"].path == tmp_path / "overlays" / "acme-standards"
assert "not-configured" not in found
def test_an_overlay_may_not_take_a_provider_skills_name(tmp_path, monkeypatch):
overlay(tmp_path / "overlays", "requirements-chain")
monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays"))
with pytest.raises(ValueError, match="reuse the names of provider Skills"):
_overlays(load_provider("gcp"))
def test_the_example_overlay_is_a_valid_skill():
assert "example-organisation-standards" in _overlays(load_provider("gcp"))