Some checks failed
validation / verify (push) Failing after 11s
Case 08 named the customer's stated topology in its design and did not build it; every Check passed because none asked for it. - The catalogue marks the 26 fields that bind a design. The agreed scope lists the ones the customer stated; what was assumed for them does not bind. - A requirement says what it comes from. A submission of requirements is refused when one of the customer's own requirements, or a stated binding field, is named by none, or when the requirement carrying a choice is not a constraint with a condition. The existing chain then owes it a Check. - Generated code may use only the providers the bundle pins: a resource or a required_providers entry of any other is refused before Terraform runs. - The guide opens with how much of the design the customer stated, how much was assumed and how much is unknown. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
432 lines
16 KiB
Python
432 lines
16 KiB
Python
"""An Execution started from a scope spec (ADR-0008)."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
import yaml
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
|
from app.adk.runtime import NoDecider, RoleSpec
|
|
from app.agent import ExecutionEngine
|
|
from app.nodes.router import Router, Thresholds
|
|
from app.providers import load_provider
|
|
from app.recorder import InMemoryRecorder
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
|
from app.workflows.deliverables import DeliverableKind as K
|
|
from tests.scoped import carrying
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(
|
|
spec.specialist, "Produce.", budget=15, skills=("base",)
|
|
)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
|
|
SCOPE_ANALYST = DEPENDENCY_GRAPH[K.SCOPE].specialist
|
|
SPEC = """
|
|
metadata: {engagement_id: ENG-77}
|
|
engagement: {kind: platform_foundation}
|
|
intent:
|
|
statement: Build us a landing zone.
|
|
exclusions: [application workloads]
|
|
estate:
|
|
cloud_provider: gcp
|
|
mode: greenfield
|
|
regions: {primary: europe-west2}
|
|
data_residency: United Kingdom
|
|
compliance:
|
|
frameworks: [DORA]
|
|
data_classification: confidential
|
|
networking: {topology: shared_vpc_hub_and_spoke, cidr_allocations: decide}
|
|
run: {pre_approved: true, terraform_modules: false}
|
|
"""
|
|
|
|
|
|
STATED_CHOICES = [
|
|
"estate.regions.primary",
|
|
"estate.data_residency",
|
|
"compliance.frameworks",
|
|
"compliance.data_classification",
|
|
"networking.topology",
|
|
]
|
|
|
|
|
|
class Recording(FakeAgentRuntime):
|
|
def __init__(self, turns):
|
|
super().__init__(turns)
|
|
self.seen: dict[str, dict] = {}
|
|
|
|
async def run(self, role, task, tools):
|
|
self.seen[role.name] = {
|
|
"skills": role.skills,
|
|
"scope": task.context.get("scope"),
|
|
}
|
|
return await super().run(role, task, tools)
|
|
|
|
|
|
def engine(tmp_path) -> tuple[ExecutionEngine, Recording]:
|
|
bundle = load_provider("gcp")
|
|
runtime = Recording(
|
|
{
|
|
**{
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
},
|
|
# The requirements carry every choice SPEC states, and a Check holds
|
|
# the design to them.
|
|
**carrying(STATED_CHOICES),
|
|
}
|
|
)
|
|
built = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
scope_catalogue=bundle.scope_catalogue(),
|
|
scope_values=bundle.scope_values,
|
|
scope_skill=bundle.scope_skill,
|
|
)
|
|
return built, runtime
|
|
|
|
|
|
async def test_a_spec_alone_starts_an_execution_and_names_its_customer_case(tmp_path):
|
|
built, _ = engine(tmp_path)
|
|
|
|
result = await built.start("", scope=SPEC, execution_id="v1")
|
|
|
|
assert result.status == "completed"
|
|
state = result.state
|
|
# The spec said pre-approved, named the case, and gave the request.
|
|
assert state["pre_approved"] is True and state["customer_case"] == "eng-77"
|
|
assert state["workflow_request"] == "Build us a landing zone."
|
|
assert state["scope"]["status"] == "APPROVED"
|
|
|
|
|
|
async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_path):
|
|
built, _ = engine(tmp_path)
|
|
|
|
await built.start("", scope=SPEC, execution_id="v1")
|
|
|
|
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
|
assert (intent / "001.scope.yaml").read_text() == SPEC
|
|
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
# Its Execution completed, so this copy is the record a change starts from.
|
|
assert completed["metadata"]["status"] == "BASELINE_LOCKED"
|
|
assert completed["networking"]["cidr_allocations"] == "decide"
|
|
assert completed["provenance"]["estate.regions.primary"] == "supplied"
|
|
assert [a["field"] for a in completed["inferred_assumptions"]] == [
|
|
"networking.cidr_allocations"
|
|
]
|
|
history = json.loads((intent / "history.json").read_text())
|
|
assert history["intents"][0]["scope"] == {
|
|
"completed": "intent/001.scope.completed.yaml",
|
|
"supplied": "intent/001.scope.yaml",
|
|
}
|
|
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
|
assert manifest["scope"]["status"] == "BASELINE_LOCKED"
|
|
assert manifest["scope"]["readiness"]["assumed"] == 1
|
|
|
|
|
|
async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
await built.start("", scope=SPEC, execution_id="v1")
|
|
|
|
for name, seen in runtime.seen.items():
|
|
assert seen["skills"] == ("base", "scoping"), name
|
|
assert seen["scope"]["kind"] == "platform_foundation"
|
|
stated = runtime.seen[ANALYST]["scope"]["stated"]
|
|
assert stated["networking"]["topology"] == "shared_vpc_hub_and_spoke"
|
|
assert stated["intent"]["exclusions"] == ["application workloads"]
|
|
assert runtime.seen[ANALYST]["scope"]["to_decide"][0]["field"] == (
|
|
"networking.cidr_allocations"
|
|
)
|
|
|
|
|
|
def settle(**values) -> Call:
|
|
"""What a scope analyst records when it must assume what nobody stated."""
|
|
return Call(
|
|
"scope_set",
|
|
{
|
|
"entries": [
|
|
{
|
|
"field": field,
|
|
"value": value,
|
|
"source": "assumed",
|
|
"note": "A default.",
|
|
}
|
|
for field, value in values.items()
|
|
]
|
|
},
|
|
)
|
|
|
|
|
|
DRAFTED = [
|
|
Call(
|
|
"scope_set",
|
|
{
|
|
"entries": [
|
|
{
|
|
"field": "intent.statement",
|
|
"value": "Ingest webhooks",
|
|
"source": "request",
|
|
},
|
|
{"field": "engagement.kind", "value": "workload", "source": "request"},
|
|
]
|
|
},
|
|
),
|
|
settle(
|
|
**{
|
|
"estate.cloud_provider": "gcp",
|
|
"estate.mode": "greenfield",
|
|
"estate.regions.primary": "europe-west2",
|
|
"estate.data_residency": "United Kingdom",
|
|
"workload.users.who": "systems",
|
|
"workload.users.count": 10,
|
|
"workload.availability.target": "99.9%",
|
|
"compliance.frameworks": [],
|
|
"compliance.data_classification": "internal",
|
|
"cost.monthly_budget": 500,
|
|
}
|
|
),
|
|
]
|
|
|
|
|
|
async def test_without_a_spec_the_scope_analyst_drafts_one_from_the_request(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
runtime._turns[SCOPE_ANALYST].clear()
|
|
runtime._turns[SCOPE_ANALYST].append([*DRAFTED, sample_submission(K.SCOPE)])
|
|
|
|
result = await built.start(
|
|
"Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1"
|
|
)
|
|
|
|
assert result.status == "completed"
|
|
intent = tmp_path / "deliverables" / "cases" / "acme" / "intent"
|
|
# Nothing was supplied, so there is no supplied file; the completed copy is
|
|
# what the analyst drafted.
|
|
assert not (intent / "001.scope.yaml").exists()
|
|
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
assert completed["intent"]["statement"] == "Ingest webhooks"
|
|
assert completed["metadata"]["customer_case"] == "acme"
|
|
assert completed["provenance"]["intent.statement"] == "request"
|
|
assert completed["provenance"]["estate.regions.primary"] == "assumed"
|
|
assert completed["readiness"]["assumed"] == 10
|
|
assumed = {a["field"] for a in completed["inferred_assumptions"]}
|
|
assert "cost.monthly_budget" in assumed
|
|
# What was assumed is shown apart from what the customer stated.
|
|
scope = runtime.seen[ANALYST]["scope"]
|
|
assert scope["stated"]["intent"]["statement"] == "Ingest webhooks"
|
|
assert "estate" not in scope["stated"]
|
|
assert {"field": "cost.monthly_budget", "value": 500, "why": "A default."} in (
|
|
scope["assumed"]
|
|
)
|
|
document = tmp_path / "deliverables" / "cases" / "acme" / "executions" / "v1"
|
|
register = (document / "deliverables" / "scope.md").read_text()
|
|
assert "## Information register" in register and "### Assumptions" in register
|
|
assert "| intent.statement | Ingest webhooks | the request |" in register
|
|
|
|
|
|
async def test_a_scope_with_an_essential_field_unsettled_cannot_be_submitted(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
runtime._turns[SCOPE_ANALYST].clear()
|
|
# It records the statement and nothing else, then tries to submit, twice.
|
|
runtime._turns[SCOPE_ANALYST].append(
|
|
[DRAFTED[0], sample_submission(K.SCOPE), sample_submission(K.SCOPE)]
|
|
)
|
|
|
|
result = await built.start("Ingest webhooks", pre_approved=True)
|
|
|
|
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
|
|
refused = run.results[1][1]
|
|
assert refused.startswith("Not submitted. Essential fields of the scope spec")
|
|
assert "estate.regions.primary" in refused
|
|
assert result.status == "completed_with_warnings"
|
|
|
|
|
|
async def test_what_the_customer_stated_is_not_overwritten_by_the_analyst(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
runtime._turns[SCOPE_ANALYST].clear()
|
|
runtime._turns[SCOPE_ANALYST].append(
|
|
[
|
|
Call("scope_read"),
|
|
settle(**{"estate.regions.primary": "us-central1", "nonsense.field": 1}),
|
|
Call(
|
|
"scope_set",
|
|
{
|
|
"entries": [
|
|
{
|
|
"field": "estate.regions.primary",
|
|
"value": "europe-west1",
|
|
"source": "answered",
|
|
}
|
|
]
|
|
},
|
|
),
|
|
Call(
|
|
"scope_set",
|
|
{
|
|
"entries": [
|
|
{
|
|
"field": "cost.monthly_budget",
|
|
"value": "lots",
|
|
"source": "answered",
|
|
}
|
|
]
|
|
},
|
|
),
|
|
sample_submission(K.SCOPE),
|
|
]
|
|
)
|
|
|
|
await built.start("", scope=SPEC, execution_id="v1")
|
|
|
|
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
|
|
read, assumed, answered, mistyped = (reply for _, reply in run.results[:4])
|
|
assert "Kind of engagement: platform_foundation" in read
|
|
assert "Left to you to decide: networking.cidr_allocations" in read
|
|
assert "estate.regions.primary was stated in the scope spec" in assumed
|
|
assert "nonsense.field is not a field" in assumed
|
|
# The customer's own answer does change what the file said.
|
|
assert answered == "Recorded 1: estate.regions.primary."
|
|
assert "cost.monthly_budget: must be a number" in mistyped
|
|
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
|
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
assert completed["estate"]["regions"]["primary"] == "europe-west1"
|
|
assert completed["provenance"]["estate.regions.primary"] == "answered"
|
|
|
|
|
|
async def test_the_scope_is_approved_before_requirements_begin(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
|
|
|
|
# Only the scope analyst has run, and the customer is asked to approve the scope.
|
|
assert {run.role for run in runtime.runs} == {SCOPE_ANALYST}
|
|
assert waiting.prompt["approval"]["phase"] == "scoping"
|
|
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
|
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
assert completed["metadata"]["status"] == "UNDER_REVIEW"
|
|
|
|
approved = await built.resume(
|
|
"v1", {"approval": {"phase": "scoping", "decision": "approve"}}
|
|
)
|
|
|
|
assert ANALYST in {run.role for run in runtime.runs}
|
|
assert approved.prompt["approval"]["phase"] == "requirements_discovery"
|
|
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
assert completed["metadata"]["status"] == "APPROVED"
|
|
|
|
|
|
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
gappy = SPEC.replace(" data_residency: United Kingdom\n", "")
|
|
|
|
with pytest.raises(ValueError, match="estate.data_residency") as refused:
|
|
await built.start("", scope=gappy)
|
|
|
|
assert "nobody to ask" in str(refused.value)
|
|
assert runtime.seen == {} and not list(tmp_path.rglob("history.json"))
|
|
# With someone present the same spec starts, and the gap is theirs to answer.
|
|
waiting = await built.start("", scope=gappy, pre_approved=False, execution_id="v2")
|
|
assert waiting.state["scope"]["status"] == "DRAFT"
|
|
asked = waiting.state["scope"]["context"]["to_ask"]
|
|
assert asked[0] == {
|
|
"field": "estate.data_residency",
|
|
"prompt": "Where must data stay?",
|
|
"severity": "HIGH",
|
|
}
|
|
|
|
|
|
async def test_a_spec_that_cannot_be_read_refuses_the_start(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
|
|
with pytest.raises(ValueError, match="networking.topolgy is not a field"):
|
|
await built.start("", scope=SPEC.replace("topology", "topolgy"))
|
|
with pytest.raises(ValueError, match="must be one of"):
|
|
await built.start(
|
|
"", scope=SPEC.replace("cloud_provider: gcp", "cloud_provider: aws")
|
|
)
|
|
with pytest.raises(ValueError, match="started for another-case"):
|
|
await built.start("", scope=SPEC, customer_case="another-case")
|
|
assert runtime.seen == {}
|
|
|
|
|
|
async def test_what_the_caller_says_outright_wins_over_the_specs_run_options(tmp_path):
|
|
built, _ = engine(tmp_path)
|
|
|
|
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
|
|
|
|
assert waiting.state["pre_approved"] is False
|
|
assert waiting.status == "awaiting_user"
|
|
|
|
|
|
async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path):
|
|
built, _ = engine(tmp_path)
|
|
|
|
with pytest.raises(ValueError, match="A request is needed"):
|
|
await built.start(
|
|
"",
|
|
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
|
|
pre_approved=False,
|
|
)
|
|
|
|
|
|
async def test_a_change_request_starts_from_the_scope_that_was_delivered(tmp_path):
|
|
built, runtime = engine(tmp_path)
|
|
await built.start("", scope=SPEC, execution_id="v1")
|
|
for name, turns in runtime._turns.items():
|
|
kind = next(
|
|
k for k, spec in DEPENDENCY_GRAPH.items() if spec.specialist == name
|
|
)
|
|
turns.append([sample_submission(kind)])
|
|
runtime._turns[SCOPE_ANALYST].clear()
|
|
runtime._turns[SCOPE_ANALYST].append(
|
|
[
|
|
Call("scope_read"),
|
|
Call(
|
|
"scope_set",
|
|
{
|
|
"entries": [
|
|
{
|
|
"field": "estate.regions.secondary",
|
|
"value": "europe-west1",
|
|
"source": "answered",
|
|
}
|
|
]
|
|
},
|
|
),
|
|
sample_submission(K.SCOPE),
|
|
]
|
|
)
|
|
|
|
await built.start(
|
|
"We now need a second region.",
|
|
customer_case="eng-77",
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
run = [r for r in runtime.runs if r.role == SCOPE_ANALYST][-1]
|
|
# The delivered spec was there to read before anything was recorded.
|
|
assert "Kind of engagement: platform_foundation" in run.results[0][1]
|
|
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
|
changed = yaml.safe_load((intent / "002.scope.completed.yaml").read_text())
|
|
assert changed["estate"]["regions"] == {
|
|
"primary": "europe-west2",
|
|
"secondary": "europe-west1",
|
|
"resiliency_tier": "ask",
|
|
}
|
|
assert changed["provenance"]["estate.regions.primary"] == "supplied"
|
|
assert changed["provenance"]["estate.regions.secondary"] == "answered"
|
|
# The Baseline's own copy is untouched.
|
|
first = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
|
assert first["estate"]["regions"]["secondary"] == "ask"
|