feat: scoping is the first Phase, completed in conversation (iteration 9 step 2)
Some checks failed
validation / verify (push) Failing after 10s
Some checks failed
validation / verify (push) Failing after 10s
Every Execution now begins by agreeing the scope, before requirements. - A new Phase and Deliverable, produced by a scope analyst with two tools: scope_read shows what is settled and what is not; scope_set records fields with where each value came from (the request, the customer's answer, or an assumption with its reason). Code checks each value against the catalogue. - With someone present the analyst asks for what is missing; with nobody present it assumes it, visibly. With no spec it drafts one from the request. A value the customer stated in the file is changed only by their own answer. - A submission cannot leave an essential field unknown. Code writes the completed spec, the data every later Specialist reads, and the information register, from what was recorded. - The spec's status follows the Execution: under review, approved, then locked as a Baseline. A Change request starts from the delivered scope. - The requirements analyst derives from the agreed scope and no longer asks for what it settles. About 80 tests assumed requirements came first; tests/scoped.py runs scoping ahead of a test's own script, and tests about order and counts are updated. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,7 +6,7 @@ import pytest
|
||||
import yaml
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
@@ -25,6 +25,7 @@ ROLES = {
|
||||
},
|
||||
}
|
||||
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
|
||||
SCOPE_ANALYST = DEPENDENCY_GRAPH[K.SCOPE].specialist
|
||||
SPEC = """
|
||||
metadata: {engagement_id: ENG-77}
|
||||
engagement: {kind: platform_foundation}
|
||||
@@ -100,7 +101,8 @@ async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_p
|
||||
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
||||
assert (intent / "001.scope.yaml").read_text() == SPEC
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["metadata"]["status"] == "APPROVED"
|
||||
# Its Execution completed, so this copy is the record a change starts from.
|
||||
assert completed["metadata"]["status"] == "BASELINE_LOCKED"
|
||||
assert completed["networking"]["cidr_allocations"] == "decide"
|
||||
assert completed["provenance"]["estate.regions.primary"] == "supplied"
|
||||
assert [a["field"] for a in completed["inferred_assumptions"]] == [
|
||||
@@ -112,7 +114,7 @@ async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_p
|
||||
"supplied": "intent/001.scope.yaml",
|
||||
}
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["scope"]["status"] == "APPROVED"
|
||||
assert manifest["scope"]["status"] == "BASELINE_LOCKED"
|
||||
assert manifest["scope"]["readiness"]["assumed"] == 1
|
||||
|
||||
|
||||
@@ -132,16 +134,179 @@ async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_spec_nothing_changes(tmp_path):
|
||||
def settle(**values) -> Call:
|
||||
"""What a scope analyst records when it must assume what nobody stated."""
|
||||
return Call(
|
||||
"scope_set",
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"field": field,
|
||||
"value": value,
|
||||
"source": "assumed",
|
||||
"note": "A default.",
|
||||
}
|
||||
for field, value in values.items()
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
DRAFTED = [
|
||||
Call(
|
||||
"scope_set",
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"field": "intent.statement",
|
||||
"value": "Ingest webhooks",
|
||||
"source": "request",
|
||||
},
|
||||
{"field": "engagement.kind", "value": "workload", "source": "request"},
|
||||
]
|
||||
},
|
||||
),
|
||||
settle(
|
||||
**{
|
||||
"estate.cloud_provider": "gcp",
|
||||
"estate.mode": "greenfield",
|
||||
"estate.regions.primary": "europe-west2",
|
||||
"estate.data_residency": "United Kingdom",
|
||||
"workload.users.who": "systems",
|
||||
"workload.users.count": 10,
|
||||
"workload.availability.target": "99.9%",
|
||||
"compliance.frameworks": [],
|
||||
"compliance.data_classification": "internal",
|
||||
"cost.monthly_budget": 500,
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
async def test_without_a_spec_the_scope_analyst_drafts_one_from_the_request(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
runtime._turns[SCOPE_ANALYST].clear()
|
||||
runtime._turns[SCOPE_ANALYST].append([*DRAFTED, sample_submission(K.SCOPE)])
|
||||
|
||||
result = await built.start(
|
||||
"Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1"
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
intent = tmp_path / "deliverables" / "cases" / "acme" / "intent"
|
||||
# Nothing was supplied, so there is no supplied file; the completed copy is
|
||||
# what the analyst drafted.
|
||||
assert not (intent / "001.scope.yaml").exists()
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["intent"]["statement"] == "Ingest webhooks"
|
||||
assert completed["metadata"]["customer_case"] == "acme"
|
||||
assert completed["provenance"]["intent.statement"] == "request"
|
||||
assert completed["provenance"]["estate.regions.primary"] == "assumed"
|
||||
assert completed["readiness"]["assumed"] == 10
|
||||
assumed = {a["field"] for a in completed["inferred_assumptions"]}
|
||||
assert "cost.monthly_budget" in assumed
|
||||
# What was assumed is shown apart from what the customer stated.
|
||||
scope = runtime.seen[ANALYST]["scope"]
|
||||
assert scope["stated"]["intent"]["statement"] == "Ingest webhooks"
|
||||
assert "estate" not in scope["stated"]
|
||||
assert {"field": "cost.monthly_budget", "value": 500, "why": "A default."} in (
|
||||
scope["assumed"]
|
||||
)
|
||||
document = tmp_path / "deliverables" / "cases" / "acme" / "executions" / "v1"
|
||||
register = (document / "deliverables" / "scope.md").read_text()
|
||||
assert "## Information register" in register and "### Assumptions" in register
|
||||
assert "| intent.statement | Ingest webhooks | the request |" in register
|
||||
|
||||
|
||||
async def test_a_scope_with_an_essential_field_unsettled_cannot_be_submitted(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
runtime._turns[SCOPE_ANALYST].clear()
|
||||
# It records the statement and nothing else, then tries to submit, twice.
|
||||
runtime._turns[SCOPE_ANALYST].append(
|
||||
[DRAFTED[0], sample_submission(K.SCOPE), sample_submission(K.SCOPE)]
|
||||
)
|
||||
|
||||
result = await built.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.state["scope"] is None
|
||||
assert runtime.seen[ANALYST] == {"skills": ("base",), "scope": None}
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["scope"] is None
|
||||
assert not list(tmp_path.rglob("*.scope*.yaml"))
|
||||
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
|
||||
refused = run.results[1][1]
|
||||
assert refused.startswith("Not submitted. Essential fields of the scope spec")
|
||||
assert "estate.regions.primary" in refused
|
||||
assert result.status == "completed_with_warnings"
|
||||
|
||||
|
||||
async def test_what_the_customer_stated_is_not_overwritten_by_the_analyst(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
runtime._turns[SCOPE_ANALYST].clear()
|
||||
runtime._turns[SCOPE_ANALYST].append(
|
||||
[
|
||||
Call("scope_read"),
|
||||
settle(**{"estate.regions.primary": "us-central1", "nonsense.field": 1}),
|
||||
Call(
|
||||
"scope_set",
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"field": "estate.regions.primary",
|
||||
"value": "europe-west1",
|
||||
"source": "answered",
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
Call(
|
||||
"scope_set",
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"field": "cost.monthly_budget",
|
||||
"value": "lots",
|
||||
"source": "answered",
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
sample_submission(K.SCOPE),
|
||||
]
|
||||
)
|
||||
|
||||
await built.start("", scope=SPEC, execution_id="v1")
|
||||
|
||||
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
|
||||
read, assumed, answered, mistyped = (reply for _, reply in run.results[:4])
|
||||
assert "Kind of engagement: platform_foundation" in read
|
||||
assert "Left to you to decide: networking.cidr_allocations" in read
|
||||
assert "estate.regions.primary was stated in the scope spec" in assumed
|
||||
assert "nonsense.field is not a field" in assumed
|
||||
# The customer's own answer does change what the file said.
|
||||
assert answered == "Recorded 1: estate.regions.primary."
|
||||
assert "cost.monthly_budget: must be a number" in mistyped
|
||||
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["estate"]["regions"]["primary"] == "europe-west1"
|
||||
assert completed["provenance"]["estate.regions.primary"] == "answered"
|
||||
|
||||
|
||||
async def test_the_scope_is_approved_before_requirements_begin(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
|
||||
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
|
||||
|
||||
# Only the scope analyst has run, and the customer is asked to approve the scope.
|
||||
assert {run.role for run in runtime.runs} == {SCOPE_ANALYST}
|
||||
assert waiting.prompt["approval"]["phase"] == "scoping"
|
||||
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["metadata"]["status"] == "UNDER_REVIEW"
|
||||
|
||||
approved = await built.resume(
|
||||
"v1", {"approval": {"phase": "scoping", "decision": "approve"}}
|
||||
)
|
||||
|
||||
assert ANALYST in {run.role for run in runtime.runs}
|
||||
assert approved.prompt["approval"]["phase"] == "requirements_discovery"
|
||||
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert completed["metadata"]["status"] == "APPROVED"
|
||||
|
||||
|
||||
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
|
||||
@@ -196,3 +361,56 @@ async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path
|
||||
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
|
||||
pre_approved=False,
|
||||
)
|
||||
|
||||
|
||||
async def test_a_change_request_starts_from_the_scope_that_was_delivered(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
await built.start("", scope=SPEC, execution_id="v1")
|
||||
for name, turns in runtime._turns.items():
|
||||
kind = next(
|
||||
k for k, spec in DEPENDENCY_GRAPH.items() if spec.specialist == name
|
||||
)
|
||||
turns.append([sample_submission(kind)])
|
||||
runtime._turns[SCOPE_ANALYST].clear()
|
||||
runtime._turns[SCOPE_ANALYST].append(
|
||||
[
|
||||
Call("scope_read"),
|
||||
Call(
|
||||
"scope_set",
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"field": "estate.regions.secondary",
|
||||
"value": "europe-west1",
|
||||
"source": "answered",
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
sample_submission(K.SCOPE),
|
||||
]
|
||||
)
|
||||
|
||||
await built.start(
|
||||
"We now need a second region.",
|
||||
customer_case="eng-77",
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
run = [r for r in runtime.runs if r.role == SCOPE_ANALYST][-1]
|
||||
# The delivered spec was there to read before anything was recorded.
|
||||
assert "Kind of engagement: platform_foundation" in run.results[0][1]
|
||||
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
|
||||
changed = yaml.safe_load((intent / "002.scope.completed.yaml").read_text())
|
||||
assert changed["estate"]["regions"] == {
|
||||
"primary": "europe-west2",
|
||||
"secondary": "europe-west1",
|
||||
"resiliency_tier": "ask",
|
||||
}
|
||||
assert changed["provenance"]["estate.regions.primary"] == "supplied"
|
||||
assert changed["provenance"]["estate.regions.secondary"] == "answered"
|
||||
# The Baseline's own copy is untouched.
|
||||
first = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
|
||||
assert first["estate"]["regions"]["secondary"] == "ask"
|
||||
|
||||
Reference in New Issue
Block a user