"""Overlay Skills (ADR-0004): an organisation's standards, named per Execution. They come only from folders the deployment configures. A caller names one; it can never supply one, and an unknown name refuses the start. """ import json from pathlib import Path import pytest from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import FakeAgentRuntime, sample_submission from app.adk.runtime import NoDecider, RoleSpec from app.agent import ExecutionEngine, discover_overlays, skill_dirs from app.config import Settings from app.nodes.router import Router, Thresholds from app.providers import load_provider from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec( spec.specialist, "Produce.", budget=15, skills=("provider-skill",) ) for spec in DEPENDENCY_GRAPH.values() }, } OVERLAYS = {"acme-standards": "abc123", "project-x": "def456"} class Recording(FakeAgentRuntime): def __init__(self, turns): super().__init__(turns) self.skills: dict[str, tuple[str, ...]] = {} async def run(self, role, task, tools): self.skills[role.name] = role.skills return await super().run(role, task, tools) def engine(tmp_path: Path, **options) -> tuple[ExecutionEngine, Recording]: runtime = Recording( { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } ) built = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), overlays=OVERLAYS, **options, ) return built, runtime async def test_every_specialist_loads_the_overlays_the_execution_names(tmp_path): built, runtime = engine(tmp_path) result = await built.start( "Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1", overlay_skills=["acme-standards"], ) assert result.status == "completed" assert set(runtime.skills.values()) == {("provider-skill", "acme-standards")} analyst = next(run for run in runtime.runs if run.role == "requirements-analyst") assert analyst.task.context["overlay_skills"] == ["acme-standards"] # The Execution records which overlays it loaded, and the version of each. assert result.state["overlay_skills"] == {"acme-standards": "abc123"} manifest = json.loads( (tmp_path / "deliverables/cases/acme/executions/v1/execution.json").read_text() ) assert manifest["overlay_skills"] == {"acme-standards": "abc123"} lessons = json.loads((tmp_path / "lessons/candidates/acme/v1.json").read_text()) assert lessons["skill_versions"]["acme-standards"] == "abc123" async def test_an_unknown_overlay_refuses_the_start(tmp_path): built, runtime = engine(tmp_path) with pytest.raises(ValueError, match="Unknown overlay Skills: evil-skill"): await built.start("Ingest webhooks", overlay_skills=["evil-skill"]) with pytest.raises(ValueError, match="must be a list of Skill names"): await built.start("Ingest webhooks", overlay_skills="acme-standards") assert runtime.runs == [] assert not (tmp_path / "deliverables").exists() async def test_the_deployments_default_overlays_are_loaded_with_the_callers(tmp_path): built, runtime = engine(tmp_path, default_overlays=["acme-standards"]) result = await built.start( "Ingest webhooks", pre_approved=True, overlay_skills=["project-x"] ) assert list(result.state["overlay_skills"]) == ["acme-standards", "project-x"] assert runtime.skills["iac-author"] == ( "provider-skill", "acme-standards", "project-x", ) async def test_without_overlays_a_role_has_only_its_own_skills(tmp_path): built, runtime = engine(tmp_path) result = await built.start("Ingest webhooks", pre_approved=True) assert result.state["overlay_skills"] == {} assert set(runtime.skills.values()) == {("provider-skill",)} def test_a_default_overlay_the_deployment_does_not_have_is_an_error(tmp_path): with pytest.raises(ValueError, match="Unknown default overlay Skills: nope"): engine(tmp_path, default_overlays=["nope"]) # --- where overlays come from ------------------------------------------------------------ def overlay(folder: Path, name: str) -> None: (folder / name).mkdir(parents=True) (folder / name / "SKILL.md").write_text( f"---\nname: {name}\ndescription: Standards for {name}.\n---\n# {name}\n" ) def test_overlays_are_found_only_in_the_folders_the_deployment_names(tmp_path): overlay(tmp_path / "overlays", "acme-standards") overlay(tmp_path / "elsewhere", "not-configured") settings = Settings(OVERLAY_SKILLS_DIRS=str(tmp_path / "overlays")) bundle = load_provider("gcp") found = discover_overlays(settings, bundle) assert list(found) == ["acme-standards"] assert skill_dirs(settings, bundle)["acme-standards"] == ( tmp_path / "overlays" / "acme-standards" ) assert set(bundle.skills) < set(skill_dirs(settings, bundle)) def test_an_overlay_may_not_take_a_provider_skills_name(tmp_path): overlay(tmp_path / "overlays", "requirements-chain") settings = Settings(OVERLAY_SKILLS_DIRS=str(tmp_path / "overlays")) with pytest.raises(ValueError, match="reuse the names of provider Skills"): discover_overlays(settings, load_provider("gcp")) def test_the_example_overlay_is_a_valid_skill_and_is_not_loaded_by_default(): settings = Settings() assert "example-organisation-standards" in discover_overlays( settings, load_provider("gcp") ) assert Settings(DEFAULT_OVERLAY_SKILLS="").default_overlays() == []