Some checks failed
check agents / check (push) Failing after 13s
The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate stream from the individual and sub-agent patterns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
161 lines
5.6 KiB
Python
161 lines
5.6 KiB
Python
"""Overlay Skills (ADR-0004): an organisation's standards, named per Execution.
|
|
|
|
They come only from folders the deployment configures. A caller names one; it can
|
|
never supply one, and an unknown name refuses the start.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.engine.fake_runtime import FakeAgentRuntime, sample_submission
|
|
from app.engine.runtime import NoDecider, RoleSpec
|
|
from app.assembly import _overlays
|
|
from app.engine.agent import ExecutionEngine
|
|
from app.engine.router import Router, Thresholds
|
|
from app.core.providers import load_provider
|
|
from app.engine.recorder import InMemoryRecorder
|
|
from app.core.deliverables import DEPENDENCY_GRAPH
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(
|
|
spec.specialist, "Produce.", budget=15, skills=("provider-skill",)
|
|
)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
OVERLAYS = {"acme-standards": "abc123", "project-x": "def456"}
|
|
|
|
|
|
class Recording(FakeAgentRuntime):
|
|
def __init__(self, turns):
|
|
super().__init__(turns)
|
|
self.skills: dict[str, tuple[str, ...]] = {}
|
|
|
|
async def run(self, role, task, tools):
|
|
self.skills[role.name] = role.skills
|
|
return await super().run(role, task, tools)
|
|
|
|
|
|
def engine(tmp_path: Path, **options) -> tuple[ExecutionEngine, Recording]:
|
|
runtime = Recording(
|
|
{
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
)
|
|
built = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
overlays=OVERLAYS,
|
|
**options,
|
|
)
|
|
return built, runtime
|
|
|
|
|
|
async def test_every_specialist_loads_the_overlays_the_execution_names(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
result = await built.start(
|
|
"Ingest webhooks",
|
|
customer_case="acme",
|
|
pre_approved=True,
|
|
execution_id="v1",
|
|
overlay_skills=["acme-standards"],
|
|
)
|
|
|
|
assert result.status == "completed"
|
|
assert set(runtime.skills.values()) == {("provider-skill", "acme-standards")}
|
|
analyst = next(run for run in runtime.runs if run.role == "requirements-analyst")
|
|
assert analyst.task.context["overlay_skills"] == ["acme-standards"]
|
|
# The Execution records which overlays it loaded, and the version of each.
|
|
assert result.state["overlay_skills"] == {"acme-standards": "abc123"}
|
|
manifest = json.loads(
|
|
(tmp_path / "deliverables/cases/acme/executions/v1/execution.json").read_text()
|
|
)
|
|
assert manifest["overlay_skills"] == {"acme-standards": "abc123"}
|
|
lessons = json.loads((tmp_path / "lessons/candidates/acme/v1.json").read_text())
|
|
assert lessons["skill_versions"]["acme-standards"] == "abc123"
|
|
|
|
|
|
async def test_an_unknown_overlay_refuses_the_start(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
with pytest.raises(ValueError, match="Unknown overlay Skills: evil-skill"):
|
|
await built.start("Ingest webhooks", overlay_skills=["evil-skill"])
|
|
with pytest.raises(ValueError, match="must be a list of Skill names"):
|
|
await built.start("Ingest webhooks", overlay_skills="acme-standards")
|
|
|
|
assert runtime.runs == []
|
|
assert not (tmp_path / "deliverables").exists()
|
|
|
|
|
|
async def test_the_deployments_default_overlays_are_loaded_with_the_callers(tmp_path):
|
|
built, runtime = engine(tmp_path, default_overlays=["acme-standards"])
|
|
|
|
result = await built.start(
|
|
"Ingest webhooks", pre_approved=True, overlay_skills=["project-x"]
|
|
)
|
|
|
|
assert list(result.state["overlay_skills"]) == ["acme-standards", "project-x"]
|
|
assert runtime.skills["iac-author"] == (
|
|
"provider-skill",
|
|
"acme-standards",
|
|
"project-x",
|
|
)
|
|
|
|
|
|
async def test_without_overlays_a_role_has_only_its_own_skills(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
result = await built.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.state["overlay_skills"] == {}
|
|
assert set(runtime.skills.values()) == {("provider-skill",)}
|
|
|
|
|
|
def test_a_default_overlay_the_deployment_does_not_have_is_an_error(tmp_path):
|
|
with pytest.raises(ValueError, match="Unknown default overlay Skills: nope"):
|
|
engine(tmp_path, default_overlays=["nope"])
|
|
|
|
|
|
# --- where overlays come from ------------------------------------------------------------
|
|
|
|
|
|
def overlay(folder: Path, name: str) -> None:
|
|
(folder / name).mkdir(parents=True)
|
|
(folder / name / "SKILL.md").write_text(
|
|
f"---\nname: {name}\ndescription: Standards for {name}.\n---\n# {name}\n"
|
|
)
|
|
|
|
|
|
def test_overlays_are_found_only_in_the_folders_the_deployment_names(tmp_path, monkeypatch):
|
|
overlay(tmp_path / "overlays", "acme-standards")
|
|
overlay(tmp_path / "elsewhere", "not-configured")
|
|
monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays"))
|
|
|
|
found = _overlays(load_provider("gcp"))
|
|
|
|
assert found["acme-standards"].path == tmp_path / "overlays" / "acme-standards"
|
|
assert "not-configured" not in found
|
|
|
|
|
|
def test_an_overlay_may_not_take_a_provider_skills_name(tmp_path, monkeypatch):
|
|
overlay(tmp_path / "overlays", "requirements-chain")
|
|
monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays"))
|
|
|
|
with pytest.raises(ValueError, match="reuse the names of provider Skills"):
|
|
_overlays(load_provider("gcp"))
|
|
|
|
|
|
def test_the_example_overlay_is_a_valid_skill():
|
|
assert "example-organisation-standards" in _overlays(load_provider("gcp"))
|