Files
gcp_solution_architecture_a…/tests/execution/test_overlay_skills.py
Jonathan Boniface 567aede5ca feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 17:45:56 +01:00

170 lines
5.9 KiB
Python

"""Overlay Skills (ADR-0004): an organisation's standards, named per Execution.
They come only from folders the deployment configures. A caller names one; it can
never supply one, and an unknown name refuses the start.
"""
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine, discover_overlays, skill_dirs
from app.config import Settings
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist, "Produce.", budget=15, skills=("provider-skill",)
)
for spec in DEPENDENCY_GRAPH.values()
},
}
OVERLAYS = {"acme-standards": "abc123", "project-x": "def456"}
class Recording(FakeAgentRuntime):
def __init__(self, turns):
super().__init__(turns)
self.skills: dict[str, tuple[str, ...]] = {}
async def run(self, role, task, tools):
self.skills[role.name] = role.skills
return await super().run(role, task, tools)
def engine(tmp_path: Path, **options) -> tuple[ExecutionEngine, Recording]:
runtime = Recording(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
)
built = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
overlays=OVERLAYS,
**options,
)
return built, runtime
async def test_every_specialist_loads_the_overlays_the_execution_names(tmp_path):
built, runtime = engine(tmp_path)
result = await built.start(
"Ingest webhooks",
customer_case="acme",
pre_approved=True,
execution_id="v1",
overlay_skills=["acme-standards"],
)
assert result.status == "completed"
assert set(runtime.skills.values()) == {("provider-skill", "acme-standards")}
analyst = next(run for run in runtime.runs if run.role == "requirements-analyst")
assert analyst.task.context["overlay_skills"] == ["acme-standards"]
# The Execution records which overlays it loaded, and the version of each.
assert result.state["overlay_skills"] == {"acme-standards": "abc123"}
manifest = json.loads(
(tmp_path / "deliverables/cases/acme/executions/v1/execution.json").read_text()
)
assert manifest["overlay_skills"] == {"acme-standards": "abc123"}
lessons = json.loads((tmp_path / "lessons/candidates/acme/v1.json").read_text())
assert lessons["skill_versions"]["acme-standards"] == "abc123"
async def test_an_unknown_overlay_refuses_the_start(tmp_path):
built, runtime = engine(tmp_path)
with pytest.raises(ValueError, match="Unknown overlay Skills: evil-skill"):
await built.start("Ingest webhooks", overlay_skills=["evil-skill"])
with pytest.raises(ValueError, match="must be a list of Skill names"):
await built.start("Ingest webhooks", overlay_skills="acme-standards")
assert runtime.runs == []
assert not (tmp_path / "deliverables").exists()
async def test_the_deployments_default_overlays_are_loaded_with_the_callers(tmp_path):
built, runtime = engine(tmp_path, default_overlays=["acme-standards"])
result = await built.start(
"Ingest webhooks", pre_approved=True, overlay_skills=["project-x"]
)
assert list(result.state["overlay_skills"]) == ["acme-standards", "project-x"]
assert runtime.skills["iac-author"] == (
"provider-skill",
"acme-standards",
"project-x",
)
async def test_without_overlays_a_role_has_only_its_own_skills(tmp_path):
built, runtime = engine(tmp_path)
result = await built.start("Ingest webhooks", pre_approved=True)
assert result.state["overlay_skills"] == {}
assert set(runtime.skills.values()) == {("provider-skill",)}
def test_a_default_overlay_the_deployment_does_not_have_is_an_error(tmp_path):
with pytest.raises(ValueError, match="Unknown default overlay Skills: nope"):
engine(tmp_path, default_overlays=["nope"])
# --- where overlays come from ------------------------------------------------------------
def overlay(folder: Path, name: str) -> None:
(folder / name).mkdir(parents=True)
(folder / name / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: Standards for {name}.\n---\n# {name}\n"
)
def test_overlays_are_found_only_in_the_folders_the_deployment_names(tmp_path):
overlay(tmp_path / "overlays", "acme-standards")
overlay(tmp_path / "elsewhere", "not-configured")
settings = Settings(OVERLAY_SKILLS_DIRS=str(tmp_path / "overlays"))
bundle = load_provider("gcp")
found = discover_overlays(settings, bundle)
assert list(found) == ["acme-standards"]
assert skill_dirs(settings, bundle)["acme-standards"] == (
tmp_path / "overlays" / "acme-standards"
)
assert set(bundle.skills) < set(skill_dirs(settings, bundle))
def test_an_overlay_may_not_take_a_provider_skills_name(tmp_path):
overlay(tmp_path / "overlays", "requirements-chain")
settings = Settings(OVERLAY_SKILLS_DIRS=str(tmp_path / "overlays"))
with pytest.raises(ValueError, match="reuse the names of provider Skills"):
discover_overlays(settings, load_provider("gcp"))
def test_the_example_overlay_is_a_valid_skill_and_is_not_loaded_by_default():
settings = Settings()
assert "example-organisation-standards" in discover_overlays(
settings, load_provider("gcp")
)
assert Settings(DEFAULT_OVERLAY_SKILLS="").default_overlays() == []