"""Overlay Skills (ADR-0004): an organisation's standards, named per Execution. They come only from folders the deployment configures. A caller names one; it can never supply one, and an unknown name refuses the start. """ import json from pathlib import Path import pytest from langgraph.checkpoint.memory import InMemorySaver from app.engine.fake_runtime import FakeAgentRuntime, sample_submission from app.engine.runtime import NoDecider, RoleSpec from app.assembly import _overlays from app.engine.agent import ExecutionEngine from app.engine.router import Router, Thresholds from app.core.providers import load_provider from app.engine.recorder import InMemoryRecorder from app.core.deliverables import DEPENDENCY_GRAPH ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec( spec.specialist, "Produce.", budget=15, skills=("provider-skill",) ) for spec in DEPENDENCY_GRAPH.values() }, } OVERLAYS = {"acme-standards": "abc123", "project-x": "def456"} class Recording(FakeAgentRuntime): def __init__(self, turns): super().__init__(turns) self.skills: dict[str, tuple[str, ...]] = {} async def run(self, role, task, tools): self.skills[role.name] = role.skills return await super().run(role, task, tools) def engine(tmp_path: Path, **options) -> tuple[ExecutionEngine, Recording]: runtime = Recording( { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } ) built = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), overlays=OVERLAYS, **options, ) return built, runtime async def test_every_specialist_loads_the_overlays_the_execution_names(tmp_path): built, runtime = engine(tmp_path) result = await built.start( "Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1", overlay_skills=["acme-standards"], ) assert result.status == "completed" assert set(runtime.skills.values()) == {("provider-skill", "acme-standards")} analyst = next(run for run in runtime.runs if run.role == "requirements-analyst") assert analyst.task.context["overlay_skills"] == ["acme-standards"] # The Execution records which overlays it loaded, and the version of each. assert result.state["overlay_skills"] == {"acme-standards": "abc123"} manifest = json.loads( (tmp_path / "deliverables/cases/acme/executions/v1/execution.json").read_text() ) assert manifest["overlay_skills"] == {"acme-standards": "abc123"} lessons = json.loads((tmp_path / "lessons/candidates/acme/v1.json").read_text()) assert lessons["skill_versions"]["acme-standards"] == "abc123" async def test_an_unknown_overlay_refuses_the_start(tmp_path): built, runtime = engine(tmp_path) with pytest.raises(ValueError, match="Unknown overlay Skills: evil-skill"): await built.start("Ingest webhooks", overlay_skills=["evil-skill"]) with pytest.raises(ValueError, match="must be a list of Skill names"): await built.start("Ingest webhooks", overlay_skills="acme-standards") assert runtime.runs == [] assert not (tmp_path / "deliverables").exists() async def test_the_deployments_default_overlays_are_loaded_with_the_callers(tmp_path): built, runtime = engine(tmp_path, default_overlays=["acme-standards"]) result = await built.start( "Ingest webhooks", pre_approved=True, overlay_skills=["project-x"] ) assert list(result.state["overlay_skills"]) == ["acme-standards", "project-x"] assert runtime.skills["iac-author"] == ( "provider-skill", "acme-standards", "project-x", ) async def test_without_overlays_a_role_has_only_its_own_skills(tmp_path): built, runtime = engine(tmp_path) result = await built.start("Ingest webhooks", pre_approved=True) assert result.state["overlay_skills"] == {} assert set(runtime.skills.values()) == {("provider-skill",)} def test_a_default_overlay_the_deployment_does_not_have_is_an_error(tmp_path): with pytest.raises(ValueError, match="Unknown default overlay Skills: nope"): engine(tmp_path, default_overlays=["nope"]) # --- where overlays come from ------------------------------------------------------------ def overlay(folder: Path, name: str) -> None: (folder / name).mkdir(parents=True) (folder / name / "SKILL.md").write_text( f"---\nname: {name}\ndescription: Standards for {name}.\n---\n# {name}\n" ) def test_overlays_are_found_only_in_the_folders_the_deployment_names(tmp_path, monkeypatch): overlay(tmp_path / "overlays", "acme-standards") overlay(tmp_path / "elsewhere", "not-configured") monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays")) found = _overlays(load_provider("gcp")) assert found["acme-standards"].path == tmp_path / "overlays" / "acme-standards" assert "not-configured" not in found def test_an_overlay_may_not_take_a_provider_skills_name(tmp_path, monkeypatch): overlay(tmp_path / "overlays", "requirements-chain") monkeypatch.setenv("OVERLAY_SKILLS_DIRS", str(tmp_path / "overlays")) with pytest.raises(ValueError, match="reuse the names of provider Skills"): _overlays(load_provider("gcp")) def test_the_example_overlay_is_a_valid_skill(): assert "example-organisation-standards" in _overlays(load_provider("gcp"))