Files
gcp_solution_architecture_a…/tests/execution/test_scope_spec.py
Jonathan Boniface afe78640f8
Some checks failed
validation / verify (push) Failing after 10s
feat: start an Execution from a scope spec (ADR-0008, iteration 9 step 1)
A scope spec is a YAML file of what the customer wants and the facts a design
depends on, complete or partly filled. It sits beside the Intent, which stays
in the customer's words.

- skills/scoping holds a provider-neutral catalogue of 63 fields; the Cloud
  provider's bundle supplies the values it allows for the fields left to it.
- app/scope.py reads a spec against the catalogue with no model: it refuses an
  unknown field or a value of the wrong type, and yields what is stated, what
  is left to the agent, what is unknown and how pressing each gap is. A field
  may say ask, decide or not_applicable.
- ExecutionEngine.start(scope=...), /generate, A2A metadata and evaluation
  cases accept one. Its statement is the request, its engagement reference
  names the Customer case, and its run options apply. A pre-approved start is
  refused when an essential field is not settled.
- The spec as supplied is kept unaltered beside a completed copy with
  assumptions, gaps, provenance, readiness and a status.
- Every Specialist is shown the spec and given the Skill: a stated choice
  binds, exclusions are out of scope.
- Covers workloads and platform foundations; case-08-landing-zone starts from
  a benchmark spec.

Plans for scoping (iteration 9) and the work breakdown structure (iteration
10, nothing built), and the design spec they come from. Not run with real
models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 17:53:00 +01:00

199 lines
7.1 KiB
Python

"""An Execution started from a scope spec (ADR-0008)."""
import json
import pytest
import yaml
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.deliverables import DeliverableKind as K
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist, "Produce.", budget=15, skills=("base",)
)
for spec in DEPENDENCY_GRAPH.values()
},
}
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
SPEC = """
metadata: {engagement_id: ENG-77}
engagement: {kind: platform_foundation}
intent:
statement: Build us a landing zone.
exclusions: [application workloads]
estate:
cloud_provider: gcp
mode: greenfield
regions: {primary: europe-west2}
data_residency: United Kingdom
compliance:
frameworks: [DORA]
data_classification: confidential
networking: {topology: shared_vpc_hub_and_spoke, cidr_allocations: decide}
run: {pre_approved: true, terraform_modules: false}
"""
class Recording(FakeAgentRuntime):
def __init__(self, turns):
super().__init__(turns)
self.seen: dict[str, dict] = {}
async def run(self, role, task, tools):
self.seen[role.name] = {
"skills": role.skills,
"scope": task.context.get("scope"),
}
return await super().run(role, task, tools)
def engine(tmp_path) -> tuple[ExecutionEngine, Recording]:
bundle = load_provider("gcp")
runtime = Recording(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
)
built = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
scope_catalogue=bundle.scope_catalogue(),
scope_values=bundle.scope_values,
scope_skill=bundle.scope_skill,
)
return built, runtime
async def test_a_spec_alone_starts_an_execution_and_names_its_customer_case(tmp_path):
built, _ = engine(tmp_path)
result = await built.start("", scope=SPEC, execution_id="v1")
assert result.status == "completed"
state = result.state
# The spec said pre-approved, named the case, and gave the request.
assert state["pre_approved"] is True and state["customer_case"] == "eng-77"
assert state["workflow_request"] == "Build us a landing zone."
assert state["scope"]["status"] == "APPROVED"
async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_path):
built, _ = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
assert (intent / "001.scope.yaml").read_text() == SPEC
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "APPROVED"
assert completed["networking"]["cidr_allocations"] == "decide"
assert completed["provenance"]["estate.regions.primary"] == "supplied"
assert [a["field"] for a in completed["inferred_assumptions"]] == [
"networking.cidr_allocations"
]
history = json.loads((intent / "history.json").read_text())
assert history["intents"][0]["scope"] == {
"completed": "intent/001.scope.completed.yaml",
"supplied": "intent/001.scope.yaml",
}
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["scope"]["status"] == "APPROVED"
assert manifest["scope"]["readiness"]["assumed"] == 1
async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
built, runtime = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
for name, seen in runtime.seen.items():
assert seen["skills"] == ("base", "scoping"), name
assert seen["scope"]["kind"] == "platform_foundation"
stated = runtime.seen[ANALYST]["scope"]["stated"]
assert stated["networking"]["topology"] == "shared_vpc_hub_and_spoke"
assert stated["intent"]["exclusions"] == ["application workloads"]
assert runtime.seen[ANALYST]["scope"]["to_decide"][0]["field"] == (
"networking.cidr_allocations"
)
async def test_without_a_spec_nothing_changes(tmp_path):
built, runtime = engine(tmp_path)
result = await built.start("Ingest webhooks", pre_approved=True)
assert result.state["scope"] is None
assert runtime.seen[ANALYST] == {"skills": ("base",), "scope": None}
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["scope"] is None
assert not list(tmp_path.rglob("*.scope*.yaml"))
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
built, runtime = engine(tmp_path)
gappy = SPEC.replace(" data_residency: United Kingdom\n", "")
with pytest.raises(ValueError, match="estate.data_residency") as refused:
await built.start("", scope=gappy)
assert "nobody to ask" in str(refused.value)
assert runtime.seen == {} and not list(tmp_path.rglob("history.json"))
# With someone present the same spec starts, and the gap is theirs to answer.
waiting = await built.start("", scope=gappy, pre_approved=False, execution_id="v2")
assert waiting.state["scope"]["status"] == "DRAFT"
asked = waiting.state["scope"]["context"]["to_ask"]
assert asked[0] == {
"field": "estate.data_residency",
"prompt": "Where must data stay?",
"severity": "HIGH",
}
async def test_a_spec_that_cannot_be_read_refuses_the_start(tmp_path):
built, runtime = engine(tmp_path)
with pytest.raises(ValueError, match="networking.topolgy is not a field"):
await built.start("", scope=SPEC.replace("topology", "topolgy"))
with pytest.raises(ValueError, match="must be one of"):
await built.start(
"", scope=SPEC.replace("cloud_provider: gcp", "cloud_provider: aws")
)
with pytest.raises(ValueError, match="started for another-case"):
await built.start("", scope=SPEC, customer_case="another-case")
assert runtime.seen == {}
async def test_what_the_caller_says_outright_wins_over_the_specs_run_options(tmp_path):
built, _ = engine(tmp_path)
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
assert waiting.state["pre_approved"] is False
assert waiting.status == "awaiting_user"
async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path):
built, _ = engine(tmp_path)
with pytest.raises(ValueError, match="A request is needed"):
await built.start(
"",
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
pre_approved=False,
)