feat: start an Execution from a scope spec (ADR-0008, iteration 9 step 1)
Some checks failed
validation / verify (push) Failing after 10s
Some checks failed
validation / verify (push) Failing after 10s
A scope spec is a YAML file of what the customer wants and the facts a design depends on, complete or partly filled. It sits beside the Intent, which stays in the customer's words. - skills/scoping holds a provider-neutral catalogue of 63 fields; the Cloud provider's bundle supplies the values it allows for the fields left to it. - app/scope.py reads a spec against the catalogue with no model: it refuses an unknown field or a value of the wrong type, and yields what is stated, what is left to the agent, what is unknown and how pressing each gap is. A field may say ask, decide or not_applicable. - ExecutionEngine.start(scope=...), /generate, A2A metadata and evaluation cases accept one. Its statement is the request, its engagement reference names the Customer case, and its run options apply. A pre-approved start is refused when an essential field is not settled. - The spec as supplied is kept unaltered beside a completed copy with assumptions, gaps, provenance, readiness and a status. - Every Specialist is shown the spec and given the Skill: a stated choice binds, exclusions are out of scope. - Covers workloads and platform foundations; case-08-landing-zone starts from a benchmark spec. Plans for scoping (iteration 9) and the work breakdown structure (iteration 10, nothing built), and the design spec they come from. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
198
tests/execution/test_scope_spec.py
Normal file
198
tests/execution/test_scope_spec.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""An Execution started from a scope spec (ADR-0008)."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
from app.workflows.deliverables import DeliverableKind as K
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist, "Produce.", budget=15, skills=("base",)
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
|
||||
SPEC = """
|
||||
metadata: {engagement_id: ENG-77}
|
||||
engagement: {kind: platform_foundation}
|
||||
intent:
|
||||
statement: Build us a landing zone.
|
||||
exclusions: [application workloads]
|
||||
estate:
|
||||
cloud_provider: gcp
|
||||
mode: greenfield
|
||||
regions: {primary: europe-west2}
|
||||
data_residency: United Kingdom
|
||||
compliance:
|
||||
frameworks: [DORA]
|
||||
data_classification: confidential
|
||||
networking: {topology: shared_vpc_hub_and_spoke, cidr_allocations: decide}
|
||||
run: {pre_approved: true, terraform_modules: false}
|
||||
"""
|
||||
|
||||
|
||||
class Recording(FakeAgentRuntime):
|
||||
def __init__(self, turns):
|
||||
super().__init__(turns)
|
||||
self.seen: dict[str, dict] = {}
|
||||
|
||||
async def run(self, role, task, tools):
|
||||
self.seen[role.name] = {
|
||||
"skills": role.skills,
|
||||
"scope": task.context.get("scope"),
|
||||
}
|
||||
return await super().run(role, task, tools)
|
||||
|
||||
|
||||
def engine(tmp_path) -> tuple[ExecutionEngine, Recording]:
|
||||
bundle = load_provider("gcp")
|
||||
runtime = Recording(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
)
|
||||
built = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
scope_catalogue=bundle.scope_catalogue(),
|
||||
scope_values=bundle.scope_values,
|
||||
scope_skill=bundle.scope_skill,
|
||||
)
|
||||
return built, runtime
|
||||
|
||||
|
||||
async def test_a_spec_alone_starts_an_execution_and_names_its_customer_case(tmp_path):
|
||||
built, _ = engine(tmp_path)
|
||||
|
||||
result = await built.start("", scope=SPEC, execution_id="v1")
|
||||
|
||||
assert result.status == "completed"
|
||||
state = result.state
|
||||
# The spec said pre-approved, named the case, and gave the request.
|
||||
assert state["pre_approved"] is True and state["customer_case"] == "eng-77"
|
||||
assert state["workflow_request"] == "Build us a landing zone."
|
||||
assert state["scope"]["status"] == "APPROVED"
|
||||
|
||||
|
||||
async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_path):
|
||||
built, _ = engine(tmp_path)
|
||||
|
||||
await built.start("", scope=SPEC, execution_id="v1")
|
||||
|
||||
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
||||
assert (intent / "001.scope.yaml").read_text() == SPEC
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["metadata"]["status"] == "APPROVED"
|
||||
assert completed["networking"]["cidr_allocations"] == "decide"
|
||||
assert completed["provenance"]["estate.regions.primary"] == "supplied"
|
||||
assert [a["field"] for a in completed["inferred_assumptions"]] == [
|
||||
"networking.cidr_allocations"
|
||||
]
|
||||
history = json.loads((intent / "history.json").read_text())
|
||||
assert history["intents"][0]["scope"] == {
|
||||
"completed": "intent/001.scope.completed.yaml",
|
||||
"supplied": "intent/001.scope.yaml",
|
||||
}
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["scope"]["status"] == "APPROVED"
|
||||
assert manifest["scope"]["readiness"]["assumed"] == 1
|
||||
|
||||
|
||||
async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
|
||||
await built.start("", scope=SPEC, execution_id="v1")
|
||||
|
||||
for name, seen in runtime.seen.items():
|
||||
assert seen["skills"] == ("base", "scoping"), name
|
||||
assert seen["scope"]["kind"] == "platform_foundation"
|
||||
stated = runtime.seen[ANALYST]["scope"]["stated"]
|
||||
assert stated["networking"]["topology"] == "shared_vpc_hub_and_spoke"
|
||||
assert stated["intent"]["exclusions"] == ["application workloads"]
|
||||
assert runtime.seen[ANALYST]["scope"]["to_decide"][0]["field"] == (
|
||||
"networking.cidr_allocations"
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_spec_nothing_changes(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
|
||||
result = await built.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.state["scope"] is None
|
||||
assert runtime.seen[ANALYST] == {"skills": ("base",), "scope": None}
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["scope"] is None
|
||||
assert not list(tmp_path.rglob("*.scope*.yaml"))
|
||||
|
||||
|
||||
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
gappy = SPEC.replace(" data_residency: United Kingdom\n", "")
|
||||
|
||||
with pytest.raises(ValueError, match="estate.data_residency") as refused:
|
||||
await built.start("", scope=gappy)
|
||||
|
||||
assert "nobody to ask" in str(refused.value)
|
||||
assert runtime.seen == {} and not list(tmp_path.rglob("history.json"))
|
||||
# With someone present the same spec starts, and the gap is theirs to answer.
|
||||
waiting = await built.start("", scope=gappy, pre_approved=False, execution_id="v2")
|
||||
assert waiting.state["scope"]["status"] == "DRAFT"
|
||||
asked = waiting.state["scope"]["context"]["to_ask"]
|
||||
assert asked[0] == {
|
||||
"field": "estate.data_residency",
|
||||
"prompt": "Where must data stay?",
|
||||
"severity": "HIGH",
|
||||
}
|
||||
|
||||
|
||||
async def test_a_spec_that_cannot_be_read_refuses_the_start(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="networking.topolgy is not a field"):
|
||||
await built.start("", scope=SPEC.replace("topology", "topolgy"))
|
||||
with pytest.raises(ValueError, match="must be one of"):
|
||||
await built.start(
|
||||
"", scope=SPEC.replace("cloud_provider: gcp", "cloud_provider: aws")
|
||||
)
|
||||
with pytest.raises(ValueError, match="started for another-case"):
|
||||
await built.start("", scope=SPEC, customer_case="another-case")
|
||||
assert runtime.seen == {}
|
||||
|
||||
|
||||
async def test_what_the_caller_says_outright_wins_over_the_specs_run_options(tmp_path):
|
||||
built, _ = engine(tmp_path)
|
||||
|
||||
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
|
||||
|
||||
assert waiting.state["pre_approved"] is False
|
||||
assert waiting.status == "awaiting_user"
|
||||
|
||||
|
||||
async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path):
|
||||
built, _ = engine(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="A request is needed"):
|
||||
await built.start(
|
||||
"",
|
||||
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
|
||||
pre_approved=False,
|
||||
)
|
||||
290
tests/test_scope.py
Normal file
290
tests/test_scope.py
Normal file
@@ -0,0 +1,290 @@
|
||||
"""The scope spec (ADR-0008): read against the catalogue, with no model involved."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from app.providers import load_provider
|
||||
from app.scope import (
|
||||
APPROVED,
|
||||
ASK,
|
||||
DRAFT,
|
||||
ESSENTIAL,
|
||||
UNDER_REVIEW,
|
||||
Catalogue,
|
||||
FieldSpec,
|
||||
ScopeError,
|
||||
read_scope,
|
||||
render_reading,
|
||||
render_template,
|
||||
slug,
|
||||
)
|
||||
|
||||
CATALOGUE = Catalogue(
|
||||
schema_version=1,
|
||||
kinds=("workload", "platform_foundation"),
|
||||
fields={
|
||||
spec.path: spec
|
||||
for spec in (
|
||||
FieldSpec(
|
||||
"metadata.customer_case",
|
||||
tier=ESSENTIAL,
|
||||
aliases=("metadata.engagement_id",),
|
||||
),
|
||||
FieldSpec(
|
||||
"engagement.kind",
|
||||
type="enum",
|
||||
values=("workload", "platform_foundation"),
|
||||
tier=ESSENTIAL,
|
||||
),
|
||||
FieldSpec("intent.statement", type="text", tier=ESSENTIAL),
|
||||
FieldSpec(
|
||||
"intent.requirements",
|
||||
type="records",
|
||||
keys=("id", "statement"),
|
||||
optional_keys=("category",),
|
||||
),
|
||||
FieldSpec(
|
||||
"users.count",
|
||||
type="integer",
|
||||
tier={"workload": ESSENTIAL},
|
||||
question="How many users?",
|
||||
),
|
||||
FieldSpec(
|
||||
"users.peak",
|
||||
type="integer",
|
||||
tier="important",
|
||||
question="How many at once?",
|
||||
default="a tenth of the users",
|
||||
affects="scaling limits",
|
||||
),
|
||||
FieldSpec("estate.region", tier="important", question="Which region?"),
|
||||
FieldSpec("network.topology", type="enum", provider_values=True),
|
||||
FieldSpec("delivery.start", type="date"),
|
||||
FieldSpec("run.pre_approved", type="boolean"),
|
||||
)
|
||||
},
|
||||
)
|
||||
COMPLETE = """
|
||||
metadata: {customer_case: acme-bookings}
|
||||
engagement: {kind: workload}
|
||||
intent:
|
||||
statement: Staff book rooms.
|
||||
requirements:
|
||||
- {id: REQ-01, statement: Rooms can be booked.}
|
||||
users: {count: 400, peak: 40}
|
||||
estate: {region: europe-west2}
|
||||
"""
|
||||
|
||||
|
||||
def read(text: str, **options):
|
||||
return read_scope(text, CATALOGUE, **options)
|
||||
|
||||
|
||||
def test_a_complete_spec_has_nothing_to_ask():
|
||||
scope = read(COMPLETE)
|
||||
|
||||
assert scope.kind == "workload" and scope.customer_case == "acme-bookings"
|
||||
assert scope.statement == "Staff book rooms."
|
||||
assert scope.gaps(ESSENTIAL, "important") == []
|
||||
assert scope.values["users.count"] == 400
|
||||
assert (
|
||||
scope.status() == UNDER_REVIEW and scope.status(pre_approved=True) == APPROVED
|
||||
)
|
||||
|
||||
|
||||
def test_a_field_left_out_or_marked_ask_is_a_gap_with_its_question():
|
||||
scope = read(COMPLETE.replace("count: 400, peak: 40", "peak: ask"))
|
||||
|
||||
assert scope.gaps() == [
|
||||
{"field": "users.count", "prompt": "How many users?", "severity": "HIGH"},
|
||||
{"field": "users.peak", "prompt": "How many at once?", "severity": "MEDIUM"},
|
||||
# Optional fields nobody filled are gaps too, the least pressing.
|
||||
*[gap for gap in scope.gaps() if gap["severity"] == "LOW"],
|
||||
]
|
||||
assert scope.status(pre_approved=True) == DRAFT
|
||||
|
||||
|
||||
def test_decide_is_an_assumption_and_not_applicable_is_neither():
|
||||
scope = read(
|
||||
COMPLETE.replace("peak: 40", "peak: decide").replace(
|
||||
"estate: {region: europe-west2}", "estate: not_applicable"
|
||||
)
|
||||
)
|
||||
|
||||
(assumed,) = scope.assumptions()
|
||||
assert assumed["id"] == "ASSUMP-01" and assumed["field"] == "users.peak"
|
||||
assert "a tenth of the users" in assumed["statement"]
|
||||
assert assumed["impact_if_invalid"] == "Changes scaling limits."
|
||||
# A whole section marked at once covers every field in it.
|
||||
assert scope.states["estate.region"] == "not_applicable"
|
||||
assert all(gap["field"] != "estate.region" for gap in scope.gaps())
|
||||
|
||||
|
||||
def test_which_fields_are_essential_depends_on_the_kind_of_engagement():
|
||||
foundation = read(
|
||||
COMPLETE.replace("workload", "platform_foundation").replace(
|
||||
"users: {count: 400, peak: 40}", ""
|
||||
)
|
||||
)
|
||||
|
||||
assert foundation.gaps(ESSENTIAL) == []
|
||||
assert foundation.tier("users.count") == "optional"
|
||||
|
||||
|
||||
def test_an_unknown_field_is_named_never_ignored():
|
||||
with pytest.raises(ScopeError) as refused:
|
||||
read(COMPLETE + "userz: {count: 1}\nestate: {regoin: x}\n")
|
||||
|
||||
assert "userz is not a field of the scope spec" in refused.value.problems
|
||||
# The second estate key replaced the first; its typo is reported too.
|
||||
assert any("estate.regoin" in problem for problem in refused.value.problems)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "fault"),
|
||||
[
|
||||
(("count: 400", "count: many"), "users.count: must be a whole number"),
|
||||
(("kind: workload", "kind: holiday"), "must be one of workload"),
|
||||
(
|
||||
("{id: REQ-01, statement: Rooms can be booked.}", "{id: REQ-01}"),
|
||||
"needs statement",
|
||||
),
|
||||
(("estate: {region: europe-west2}", "estate: europe"), "estate is a section"),
|
||||
],
|
||||
)
|
||||
def test_a_value_of_the_wrong_kind_is_refused_with_the_reason(change, fault):
|
||||
with pytest.raises(ScopeError) as refused:
|
||||
read(COMPLETE.replace(*change))
|
||||
|
||||
assert any(fault in problem for problem in refused.value.problems)
|
||||
|
||||
|
||||
def test_text_that_is_not_yaml_is_refused():
|
||||
with pytest.raises(ScopeError, match="not valid YAML"):
|
||||
read("intent: [unclosed")
|
||||
with pytest.raises(ScopeError, match="must be a mapping"):
|
||||
read("- just\n- a list\n")
|
||||
|
||||
|
||||
def test_a_provider_supplies_the_values_the_catalogue_leaves_to_it():
|
||||
spec = COMPLETE + "network: {topology: hub_and_spoke}\n"
|
||||
|
||||
with pytest.raises(ScopeError):
|
||||
read(spec, provider_values={"network.topology": ["flat"]})
|
||||
allowed = read(spec, provider_values={"network.topology": ["hub_and_spoke"]})
|
||||
|
||||
assert allowed.values["network.topology"] == "hub_and_spoke"
|
||||
|
||||
|
||||
def test_an_engagement_reference_and_a_customer_case_name_the_same_thing():
|
||||
by_engagement = read(
|
||||
COMPLETE.replace(
|
||||
"customer_case: acme-bookings", "engagement_id: ENG-2026-UK-084"
|
||||
)
|
||||
)
|
||||
|
||||
assert by_engagement.customer_case == "eng-2026-uk-084"
|
||||
assert slug("ENG 2026/UK 084") == "eng-2026-uk-084"
|
||||
with pytest.raises(ScopeError, match="name the same thing and differ"):
|
||||
read(
|
||||
COMPLETE.replace(
|
||||
"{customer_case: acme-bookings}", "{customer_case: a, engagement_id: b}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_the_completed_copy_holds_every_field_and_the_agents_sections():
|
||||
scope = read(
|
||||
COMPLETE.replace("peak: 40", "peak: decide") + "delivery: {start: 2026-11-01}\n"
|
||||
)
|
||||
|
||||
completed = scope.completed(DRAFT)
|
||||
|
||||
assert completed["metadata"]["status"] == DRAFT
|
||||
assert completed["users"] == {"count": 400, "peak": "decide"}
|
||||
assert completed["delivery"]["start"] == "2026-11-01" # plain data, not a date
|
||||
assert completed["network"]["topology"] == ASK
|
||||
assert completed["provenance"]["users.count"] == "supplied"
|
||||
assert completed["provenance"]["users.peak"] == "assumed"
|
||||
assert completed["provenance"]["network.topology"] == "unknown"
|
||||
assert completed["readiness"]["supplied"] == len(scope.values)
|
||||
assert completed["inferred_assumptions"][0]["field"] == "users.peak"
|
||||
assert yaml.safe_load(yaml.safe_dump(completed)) == completed
|
||||
|
||||
|
||||
def test_a_completed_copy_can_be_read_back_as_a_spec():
|
||||
"""The agent's own sections are replaced, not rejected, on the way back in."""
|
||||
scope = read(COMPLETE.replace("peak: 40", "peak: decide"))
|
||||
again = read(yaml.safe_dump(scope.completed(DRAFT)))
|
||||
|
||||
assert again.values == scope.values and again.states == scope.states
|
||||
|
||||
|
||||
def test_a_specialist_is_shown_what_is_stated_and_what_is_open():
|
||||
scope = read(
|
||||
COMPLETE.replace("peak: 40", "peak: decide") + "run: {pre_approved: true}\n"
|
||||
)
|
||||
|
||||
context = scope.for_context()
|
||||
|
||||
assert context["kind"] == "workload"
|
||||
assert context["stated"]["users"] == {"count": 400}
|
||||
assert context["stated"]["intent"]["requirements"][0]["id"] == "REQ-01"
|
||||
assert context["to_decide"][0]["field"] == "users.peak"
|
||||
# How to run the agent, and the case's name, are not the Specialists' business.
|
||||
assert "run" not in context["stated"] and "metadata" not in context["stated"]
|
||||
|
||||
|
||||
def test_a_template_and_a_reading_are_written_for_a_person():
|
||||
template = yaml.safe_load(render_template(CATALOGUE, "workload"))
|
||||
reading = render_reading(read(COMPLETE.replace("count: 400, peak: 40", "peak: 40")))
|
||||
|
||||
assert template["engagement"]["kind"] == "workload"
|
||||
assert template["users"] == {"count": ASK, "peak": ASK}
|
||||
assert "delivery" not in template # optional fields are left out of a template
|
||||
assert "Essential, needed before work can start:" in reading
|
||||
assert "users.count: How many users?" in reading
|
||||
|
||||
|
||||
# --- the real catalogue and the benchmark spec ------------------------------------------
|
||||
|
||||
|
||||
def test_the_gcp_bundle_reads_the_benchmark_landing_zone_spec():
|
||||
bundle = load_provider("gcp")
|
||||
text = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "eval"
|
||||
/ "datasets"
|
||||
/ "specs"
|
||||
/ "landing-zone.yaml"
|
||||
).read_text()
|
||||
|
||||
scope = read_scope(
|
||||
text, bundle.scope_catalogue(), provider_values=bundle.scope_values
|
||||
)
|
||||
|
||||
assert scope.kind == "platform_foundation"
|
||||
assert scope.customer_case == "case-08-landing-zone"
|
||||
# Complete enough to start with nobody to ask.
|
||||
assert scope.gaps(ESSENTIAL) == []
|
||||
assert scope.values["networking.topology"] == "shared_vpc_hub_and_spoke"
|
||||
assert scope.states["workload.users.count"] == "not_applicable"
|
||||
assert [item["id"] for item in scope.values["intent.requirements"]] == [
|
||||
"REQ-01",
|
||||
"REQ-02",
|
||||
"REQ-03",
|
||||
"REQ-04",
|
||||
]
|
||||
|
||||
|
||||
def test_every_essential_field_of_the_catalogue_has_a_question_to_ask():
|
||||
catalogue = load_provider("gcp").scope_catalogue()
|
||||
|
||||
for kind in catalogue.kinds:
|
||||
for path, spec in catalogue.fields.items():
|
||||
if spec.tier_for(kind) != "optional":
|
||||
assert spec.question, (
|
||||
f"{path} is {spec.tier_for(kind)} and has no question"
|
||||
)
|
||||
Reference in New Issue
Block a user