Files
gcp_solution_architecture_a…/tests/execution/test_scope_spec.py
Jonathan Boniface 545097530b
Some checks failed
validation / verify (push) Failing after 10s
feat: scoping is the first Phase, completed in conversation (iteration 9 step 2)
Every Execution now begins by agreeing the scope, before requirements.

- A new Phase and Deliverable, produced by a scope analyst with two tools:
  scope_read shows what is settled and what is not; scope_set records fields
  with where each value came from (the request, the customer's answer, or an
  assumption with its reason). Code checks each value against the catalogue.
- With someone present the analyst asks for what is missing; with nobody
  present it assumes it, visibly. With no spec it drafts one from the request.
  A value the customer stated in the file is changed only by their own answer.
- A submission cannot leave an essential field unknown. Code writes the
  completed spec, the data every later Specialist reads, and the information
  register, from what was recorded.
- The spec's status follows the Execution: under review, approved, then
  locked as a Baseline. A Change request starts from the delivered scope.
- The requirements analyst derives from the agreed scope and no longer asks
  for what it settles.

About 80 tests assumed requirements came first; tests/scoped.py runs scoping
ahead of a test's own script, and tests about order and counts are updated.
Not run with real models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 18:17:32 +01:00

417 lines
16 KiB
Python

"""An Execution started from a scope spec (ADR-0008)."""
import json
import pytest
import yaml
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.deliverables import DeliverableKind as K
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist, "Produce.", budget=15, skills=("base",)
)
for spec in DEPENDENCY_GRAPH.values()
},
}
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
SCOPE_ANALYST = DEPENDENCY_GRAPH[K.SCOPE].specialist
SPEC = """
metadata: {engagement_id: ENG-77}
engagement: {kind: platform_foundation}
intent:
statement: Build us a landing zone.
exclusions: [application workloads]
estate:
cloud_provider: gcp
mode: greenfield
regions: {primary: europe-west2}
data_residency: United Kingdom
compliance:
frameworks: [DORA]
data_classification: confidential
networking: {topology: shared_vpc_hub_and_spoke, cidr_allocations: decide}
run: {pre_approved: true, terraform_modules: false}
"""
class Recording(FakeAgentRuntime):
def __init__(self, turns):
super().__init__(turns)
self.seen: dict[str, dict] = {}
async def run(self, role, task, tools):
self.seen[role.name] = {
"skills": role.skills,
"scope": task.context.get("scope"),
}
return await super().run(role, task, tools)
def engine(tmp_path) -> tuple[ExecutionEngine, Recording]:
bundle = load_provider("gcp")
runtime = Recording(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
)
built = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
scope_catalogue=bundle.scope_catalogue(),
scope_values=bundle.scope_values,
scope_skill=bundle.scope_skill,
)
return built, runtime
async def test_a_spec_alone_starts_an_execution_and_names_its_customer_case(tmp_path):
built, _ = engine(tmp_path)
result = await built.start("", scope=SPEC, execution_id="v1")
assert result.status == "completed"
state = result.state
# The spec said pre-approved, named the case, and gave the request.
assert state["pre_approved"] is True and state["customer_case"] == "eng-77"
assert state["workflow_request"] == "Build us a landing zone."
assert state["scope"]["status"] == "APPROVED"
async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_path):
built, _ = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
assert (intent / "001.scope.yaml").read_text() == SPEC
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
# Its Execution completed, so this copy is the record a change starts from.
assert completed["metadata"]["status"] == "BASELINE_LOCKED"
assert completed["networking"]["cidr_allocations"] == "decide"
assert completed["provenance"]["estate.regions.primary"] == "supplied"
assert [a["field"] for a in completed["inferred_assumptions"]] == [
"networking.cidr_allocations"
]
history = json.loads((intent / "history.json").read_text())
assert history["intents"][0]["scope"] == {
"completed": "intent/001.scope.completed.yaml",
"supplied": "intent/001.scope.yaml",
}
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["scope"]["status"] == "BASELINE_LOCKED"
assert manifest["scope"]["readiness"]["assumed"] == 1
async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
built, runtime = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
for name, seen in runtime.seen.items():
assert seen["skills"] == ("base", "scoping"), name
assert seen["scope"]["kind"] == "platform_foundation"
stated = runtime.seen[ANALYST]["scope"]["stated"]
assert stated["networking"]["topology"] == "shared_vpc_hub_and_spoke"
assert stated["intent"]["exclusions"] == ["application workloads"]
assert runtime.seen[ANALYST]["scope"]["to_decide"][0]["field"] == (
"networking.cidr_allocations"
)
def settle(**values) -> Call:
"""What a scope analyst records when it must assume what nobody stated."""
return Call(
"scope_set",
{
"entries": [
{
"field": field,
"value": value,
"source": "assumed",
"note": "A default.",
}
for field, value in values.items()
]
},
)
DRAFTED = [
Call(
"scope_set",
{
"entries": [
{
"field": "intent.statement",
"value": "Ingest webhooks",
"source": "request",
},
{"field": "engagement.kind", "value": "workload", "source": "request"},
]
},
),
settle(
**{
"estate.cloud_provider": "gcp",
"estate.mode": "greenfield",
"estate.regions.primary": "europe-west2",
"estate.data_residency": "United Kingdom",
"workload.users.who": "systems",
"workload.users.count": 10,
"workload.availability.target": "99.9%",
"compliance.frameworks": [],
"compliance.data_classification": "internal",
"cost.monthly_budget": 500,
}
),
]
async def test_without_a_spec_the_scope_analyst_drafts_one_from_the_request(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append([*DRAFTED, sample_submission(K.SCOPE)])
result = await built.start(
"Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1"
)
assert result.status == "completed"
intent = tmp_path / "deliverables" / "cases" / "acme" / "intent"
# Nothing was supplied, so there is no supplied file; the completed copy is
# what the analyst drafted.
assert not (intent / "001.scope.yaml").exists()
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["intent"]["statement"] == "Ingest webhooks"
assert completed["metadata"]["customer_case"] == "acme"
assert completed["provenance"]["intent.statement"] == "request"
assert completed["provenance"]["estate.regions.primary"] == "assumed"
assert completed["readiness"]["assumed"] == 10
assumed = {a["field"] for a in completed["inferred_assumptions"]}
assert "cost.monthly_budget" in assumed
# What was assumed is shown apart from what the customer stated.
scope = runtime.seen[ANALYST]["scope"]
assert scope["stated"]["intent"]["statement"] == "Ingest webhooks"
assert "estate" not in scope["stated"]
assert {"field": "cost.monthly_budget", "value": 500, "why": "A default."} in (
scope["assumed"]
)
document = tmp_path / "deliverables" / "cases" / "acme" / "executions" / "v1"
register = (document / "deliverables" / "scope.md").read_text()
assert "## Information register" in register and "### Assumptions" in register
assert "| intent.statement | Ingest webhooks | the request |" in register
async def test_a_scope_with_an_essential_field_unsettled_cannot_be_submitted(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
# It records the statement and nothing else, then tries to submit, twice.
runtime._turns[SCOPE_ANALYST].append(
[DRAFTED[0], sample_submission(K.SCOPE), sample_submission(K.SCOPE)]
)
result = await built.start("Ingest webhooks", pre_approved=True)
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
refused = run.results[1][1]
assert refused.startswith("Not submitted. Essential fields of the scope spec")
assert "estate.regions.primary" in refused
assert result.status == "completed_with_warnings"
async def test_what_the_customer_stated_is_not_overwritten_by_the_analyst(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append(
[
Call("scope_read"),
settle(**{"estate.regions.primary": "us-central1", "nonsense.field": 1}),
Call(
"scope_set",
{
"entries": [
{
"field": "estate.regions.primary",
"value": "europe-west1",
"source": "answered",
}
]
},
),
Call(
"scope_set",
{
"entries": [
{
"field": "cost.monthly_budget",
"value": "lots",
"source": "answered",
}
]
},
),
sample_submission(K.SCOPE),
]
)
await built.start("", scope=SPEC, execution_id="v1")
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
read, assumed, answered, mistyped = (reply for _, reply in run.results[:4])
assert "Kind of engagement: platform_foundation" in read
assert "Left to you to decide: networking.cidr_allocations" in read
assert "estate.regions.primary was stated in the scope spec" in assumed
assert "nonsense.field is not a field" in assumed
# The customer's own answer does change what the file said.
assert answered == "Recorded 1: estate.regions.primary."
assert "cost.monthly_budget: must be a number" in mistyped
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["estate"]["regions"]["primary"] == "europe-west1"
assert completed["provenance"]["estate.regions.primary"] == "answered"
async def test_the_scope_is_approved_before_requirements_begin(tmp_path):
built, runtime = engine(tmp_path)
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
# Only the scope analyst has run, and the customer is asked to approve the scope.
assert {run.role for run in runtime.runs} == {SCOPE_ANALYST}
assert waiting.prompt["approval"]["phase"] == "scoping"
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "UNDER_REVIEW"
approved = await built.resume(
"v1", {"approval": {"phase": "scoping", "decision": "approve"}}
)
assert ANALYST in {run.role for run in runtime.runs}
assert approved.prompt["approval"]["phase"] == "requirements_discovery"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "APPROVED"
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
built, runtime = engine(tmp_path)
gappy = SPEC.replace(" data_residency: United Kingdom\n", "")
with pytest.raises(ValueError, match="estate.data_residency") as refused:
await built.start("", scope=gappy)
assert "nobody to ask" in str(refused.value)
assert runtime.seen == {} and not list(tmp_path.rglob("history.json"))
# With someone present the same spec starts, and the gap is theirs to answer.
waiting = await built.start("", scope=gappy, pre_approved=False, execution_id="v2")
assert waiting.state["scope"]["status"] == "DRAFT"
asked = waiting.state["scope"]["context"]["to_ask"]
assert asked[0] == {
"field": "estate.data_residency",
"prompt": "Where must data stay?",
"severity": "HIGH",
}
async def test_a_spec_that_cannot_be_read_refuses_the_start(tmp_path):
built, runtime = engine(tmp_path)
with pytest.raises(ValueError, match="networking.topolgy is not a field"):
await built.start("", scope=SPEC.replace("topology", "topolgy"))
with pytest.raises(ValueError, match="must be one of"):
await built.start(
"", scope=SPEC.replace("cloud_provider: gcp", "cloud_provider: aws")
)
with pytest.raises(ValueError, match="started for another-case"):
await built.start("", scope=SPEC, customer_case="another-case")
assert runtime.seen == {}
async def test_what_the_caller_says_outright_wins_over_the_specs_run_options(tmp_path):
built, _ = engine(tmp_path)
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
assert waiting.state["pre_approved"] is False
assert waiting.status == "awaiting_user"
async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path):
built, _ = engine(tmp_path)
with pytest.raises(ValueError, match="A request is needed"):
await built.start(
"",
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
pre_approved=False,
)
async def test_a_change_request_starts_from_the_scope_that_was_delivered(tmp_path):
built, runtime = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
for name, turns in runtime._turns.items():
kind = next(
k for k, spec in DEPENDENCY_GRAPH.items() if spec.specialist == name
)
turns.append([sample_submission(kind)])
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append(
[
Call("scope_read"),
Call(
"scope_set",
{
"entries": [
{
"field": "estate.regions.secondary",
"value": "europe-west1",
"source": "answered",
}
]
},
),
sample_submission(K.SCOPE),
]
)
await built.start(
"We now need a second region.",
customer_case="eng-77",
pre_approved=True,
execution_id="v2",
baseline="v1",
)
run = [r for r in runtime.runs if r.role == SCOPE_ANALYST][-1]
# The delivered spec was there to read before anything was recorded.
assert "Kind of engagement: platform_foundation" in run.results[0][1]
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
changed = yaml.safe_load((intent / "002.scope.completed.yaml").read_text())
assert changed["estate"]["regions"] == {
"primary": "europe-west2",
"secondary": "europe-west1",
"resiliency_tier": "ask",
}
assert changed["provenance"]["estate.regions.primary"] == "supplied"
assert changed["provenance"]["estate.regions.secondary"] == "answered"
# The Baseline's own copy is untouched.
first = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert first["estate"]["regions"]["secondary"] == "ask"