Files
gcp_solution_architecture_a…/tests/execution/test_skill_routing.py
Jonathan Boniface 567aede5ca feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 17:45:56 +01:00

122 lines
4.1 KiB
Python

"""Which references a Specialist is told to read first (iteration 5 step 8)."""
from pathlib import Path
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, FakeDecider, sample_submission
from app.adk.runtime import Answer, NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.tools.skill_routing import read_first, references_of, routable
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.deliverables import DeliverableKind as K
PATTERNS = {
"references/cloud-sql.md": "Cloud SQL. A private PostgreSQL instance.",
"references/pubsub.md": "Pub/Sub. A topic and a subscription.",
}
def skill(root: Path, name: str, references: dict[str, str]) -> Path:
folder = root / name
(folder / "references").mkdir(parents=True)
(folder / "SKILL.md").write_text(f"---\nname: {name}\ndescription: d\n---\n")
for file, text in references.items():
(folder / "references" / file).write_text(text)
return folder
def test_a_reference_is_described_by_its_title_and_first_paragraph(tmp_path):
folder = skill(
tmp_path,
"patterns",
{"kms.md": "# Cloud KMS\n\nA key ring and a key.\n\n- A rule.\n"},
)
assert references_of(folder) == {
"references/kms.md": "Cloud KMS. A key ring and a key"
}
def test_only_a_skill_with_enough_references_is_routed_within(tmp_path):
many = skill(tmp_path, "many", {f"r{n}.md": f"# R{n}\n\nText.\n" for n in range(4)})
few = skill(tmp_path, "few", {"one.md": "# One\n\nText.\n"})
none = tmp_path / "none"
none.mkdir()
assert list(routable({"many": many, "few": few, "none": none})) == ["many"]
async def test_the_references_a_task_needs_are_named():
decider = FakeDecider(
{
"patterns/references/cloud-sql.md": Answer(1.8, 0.9),
"patterns/references/pubsub.md": Answer(0.1, 0.9),
}
)
chosen = await read_first(decider, "Write the code.", {"patterns": PATTERNS})
assert chosen == [{"skill": "patterns", "resource": "references/cloud-sql.md"}]
_, questions, _ = decider.asked[0]
assert "Cloud SQL. A private PostgreSQL instance." in (
questions["patterns/references/cloud-sql.md"].instructions
)
async def test_without_a_decision_nothing_is_named():
assert await read_first(NoDecider(), "Write.", {"patterns": PATTERNS}) is None
async def test_a_specialist_with_routable_skills_is_told_where_to_start(tmp_path):
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
seen: dict[str, object] = {}
def recording(kind):
def turn(task):
seen[kind.value] = task.context["read_first"]
return [sample_submission(kind)]
return turn
decider = FakeDecider(
{
"patterns/references/cloud-sql.md": Answer(0.2, 0.9),
"patterns/references/pubsub.md": Answer(1.9, 0.9),
}
)
author = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
engine = ExecutionEngine(
runtime=FakeAgentRuntime(
{
spec.specialist: [recording(kind)]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
skill_references={author: {"patterns": PATTERNS}},
)
await engine.start("Ingest webhooks", pre_approved=True)
assert seen[K.DEPLOYMENT_GUIDANCE.value] == [
{"skill": "patterns", "resource": "references/pubsub.md"}
]
assert seen[K.REQUIREMENTS.value] is None
state, _, _ = decider.asked[0]
assert state.startswith("Write the deployment guidance Deliverable")
assert "Elements of the Target architecture:" in state