feat: scoping is the first Phase, completed in conversation (iteration 9 step 2)
Some checks failed
validation / verify (push) Failing after 10s

Every Execution now begins by agreeing the scope, before requirements.

- A new Phase and Deliverable, produced by a scope analyst with two tools:
  scope_read shows what is settled and what is not; scope_set records fields
  with where each value came from (the request, the customer's answer, or an
  assumption with its reason). Code checks each value against the catalogue.
- With someone present the analyst asks for what is missing; with nobody
  present it assumes it, visibly. With no spec it drafts one from the request.
  A value the customer stated in the file is changed only by their own answer.
- A submission cannot leave an essential field unknown. Code writes the
  completed spec, the data every later Specialist reads, and the information
  register, from what was recorded.
- The spec's status follows the Execution: under review, approved, then
  locked as a Baseline. A Change request starts from the delivered scope.
- The requirements analyst derives from the agreed scope and no longer asks
  for what it settles.

About 80 tests assumed requirements came first; tests/scoped.py runs scoping
ahead of a test's own script, and tests about order and counts are updated.
Not run with real models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-11 18:17:32 +01:00
parent afe78640f8
commit 545097530b
38 changed files with 1054 additions and 123 deletions

View File

@@ -23,6 +23,8 @@ from app.nodes.intake import Intake
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
AGENTS_DIR = Path(__file__).parents[2] / "agents"
K = DeliverableKind

View File

@@ -18,7 +18,6 @@ from app.agent import ExecutionEngine
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
AGENTS_DIR = Path(__file__).parents[2] / "agents"
@@ -42,11 +41,6 @@ def call(tool: str, **args) -> list[types.Part]:
return [types.Part(function_call=types.FunctionCall(name=tool, args=args))]
def requirements_markdown() -> str:
sections = DEPENDENCY_GRAPH[DeliverableKind.REQUIREMENTS].sections
return "\n".join(f"## {section}\nDetails." for section in sections)
def task_context(request: LlmRequest) -> dict:
text = (request.contents[0].parts or [])[0].text or ""
return json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0])
@@ -58,26 +52,24 @@ async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
models = {
"orchestrator": RoleScriptedLlm(
script=[
call("delegate", kinds=["requirements"]),
call("delegate", kinds=["scope"]),
call("ask_user", questions=[]),
call("delegate", kinds=["requirements"]),
call("delegate", kinds=["scope"]),
call("ask_user", questions=["Anything to add?"]),
]
),
"requirements-analyst": RoleScriptedLlm(
"scope-analyst": RoleScriptedLlm(
script=[
call("load_skill", skill_name="google-cloud-solution-architecture"),
call("load_skill", skill_name="scoping"),
call(
"needs_input", questions=["What availability target do you need?"]
),
call(
"submit_deliverable",
content=requirements_markdown(),
content="# Scope\n## Scope statement\nRegional event ingestion.\n",
attachments=[
{
"path": "requirements.json",
"content": SAMPLE_DATA["requirements.json"],
}
{"path": name, "content": SAMPLE_DATA[name]}
for name in ("scope.yaml", "scope.json")
],
),
]
@@ -101,7 +93,7 @@ async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
assert asked.prompt == {
"questions": [
{
"from": "requirements",
"from": "scope",
"question": "What availability target do you need?",
}
]
@@ -109,11 +101,8 @@ async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
answered = await engine.resume(asked.execution_id, {"reply": "99.9% in one region"})
assert (
answered.state["ledger"]["deliverables"]["requirements"]["status"]
== "submitted"
)
analyst_second_run = models["requirements-analyst"].requests[-1]
assert answered.state["ledger"]["deliverables"]["scope"]["status"] == "submitted"
analyst_second_run = models["scope-analyst"].requests[-1]
assert task_context(analyst_second_run)["answers"] == [
{
"questions": ["What availability target do you need?"],
@@ -122,7 +111,5 @@ async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
]
orchestrator_prompt = models["orchestrator"].requests[0].config.system_instruction
assert "You coordinate one solution-architecture Execution" in orchestrator_prompt
analyst_prompt = (
models["requirements-analyst"].requests[0].config.system_instruction
)
analyst_prompt = models["scope-analyst"].requests[0].config.system_instruction
assert "## How you work" in analyst_prompt

View File

@@ -9,6 +9,8 @@ from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.workflows.target_architecture import parse_graph, render_mermaid
from eval.scoring import CaseSpec, score_execution, template_headings
from tests.scoped import ScopedEngine as ExecutionEngine
from tests.scoped import ScopedRuntime as FakeAgentRuntime
TEMPLATE = (
Path(__file__).parents[2]
@@ -427,9 +429,8 @@ async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.fake_runtime import Call, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from eval.solution_eval import run_with_scripted_user

View File

@@ -24,6 +24,8 @@ from app.baseline import BaselineError, load_baseline
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
CASE = "acme-webhooks"

View File

@@ -45,6 +45,7 @@ def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle():
def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist():
assert PHASE_ORDER == (
PhaseId.SCOPING,
PhaseId.REQUIREMENTS_DISCOVERY,
PhaseId.SOLUTION_ARCHITECTURE,
PhaseId.SOLUTION_IMPLEMENTATION,
@@ -61,7 +62,8 @@ def test_the_target_architecture_sits_between_the_design_and_the_code():
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
K = DeliverableKind
assert depends[K.REQUIREMENTS] == set()
assert depends[K.SCOPE] == set()
assert depends[K.REQUIREMENTS] == {K.SCOPE}
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
@@ -89,16 +91,18 @@ def test_the_design_is_approved_before_code_is_written():
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
design = (
DeliverableKind.SCOPE,
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.DESIGN_RECOMMENDATIONS,
DeliverableKind.TARGET_ARCHITECTURE,
)
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
assert inputs_of(DeliverableKind.SCOPE) == ()
assert inputs_of(DeliverableKind.REQUIREMENTS) == (DeliverableKind.SCOPE,)
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:4]
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:5]
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
DeliverableKind.SOLUTION_GUIDE
}

View File

@@ -10,11 +10,13 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.fake_runtime import Call, sample_submission
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from tests.scoped import ScopedEngine
from tests.scoped import ScopedRuntime as FakeAgentRuntime
K = DeliverableKind
P = PhaseId
@@ -71,7 +73,7 @@ def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine:
"output_root": tmp_path,
}
options.update(overrides)
return ExecutionEngine(**options)
return ScopedEngine(**options)
def happy_path_turns(**extra_specialist_turns) -> dict:
@@ -132,7 +134,7 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
recorded_types = [e.type for e in recorder.events[result.execution_id]]
assert recorded_types.count("approved") == 5
assert recorded_types.count("approved") == 6 # scoping, then the five after it
async def test_independent_specialists_run_together_between_orchestrator_turns(
@@ -161,6 +163,7 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
)
dependencies = describer_run.task.context["dependencies"]
assert list(dependencies) == [
"scope",
"requirements",
"technical_decomposition",
"product_selection",
@@ -718,7 +721,7 @@ async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
]
}
)
engine = make_engine(tmp_path, runtime, max_orchestrator_turns=2)
engine = make_engine(tmp_path, runtime, max_orchestrator_turns=3)
result = await engine.start(
"Ingestion",

View File

@@ -25,6 +25,14 @@ def approve(ledger: Ledger, phase: P) -> None:
ledger.approve(phase, actor="user")
def scoped(**options) -> Ledger:
"""A Ledger whose scope is agreed: where requirements discovery begins."""
ledger = Ledger(**options)
produce(ledger, K.SCOPE)
approve(ledger, P.SCOPING)
return ledger
def through_phase_one(ledger: Ledger) -> None:
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
approve(ledger, P.REQUIREMENTS_DISCOVERY)
@@ -49,22 +57,29 @@ def through_phase_three(ledger: Ledger) -> None:
# --- ready work and the two acceptance thresholds -------------------------------
def test_new_execution_has_only_requirements_ready():
def test_a_new_execution_has_only_the_scope_ready():
ledger = Ledger()
assert ledger.ready_work() == (K.REQUIREMENTS,)
assert ledger.ready_work() == (K.SCOPE,)
assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P)
def test_once_the_scope_is_approved_requirements_are_ready():
ledger = scoped()
assert ledger.ready_work() == (K.REQUIREMENTS,)
assert ledger.phase_status(P.SCOPING) is PhaseStatus.APPROVED
def test_same_phase_dependency_is_met_once_submitted():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,)
def test_cross_phase_dependency_needs_the_earlier_phase_approved():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert K.PRODUCT_SELECTION not in ledger.ready_work()
@@ -75,7 +90,7 @@ def test_cross_phase_dependency_needs_the_earlier_phase_approved():
def test_independent_work_becomes_ready_together():
ledger = Ledger()
ledger = scoped()
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
@@ -93,7 +108,7 @@ def test_independent_work_becomes_ready_together():
def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.phases_ready_for_approval() == ()
@@ -106,7 +121,7 @@ def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
ledger = Ledger()
ledger = scoped()
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator")
@@ -115,7 +130,7 @@ def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
def test_start_moves_ready_work_in_progress_and_returns_attempts():
ledger = Ledger()
ledger = scoped()
attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator")
@@ -125,7 +140,7 @@ def test_start_moves_ready_work_in_progress_and_returns_attempts():
def test_submit_requires_the_current_attempt():
ledger = Ledger()
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
@@ -136,7 +151,7 @@ def test_submit_requires_the_current_attempt():
def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
ledger = Ledger()
ledger = scoped()
first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, first, reason="needs input")
second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
@@ -149,14 +164,14 @@ def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
def test_submit_refuses_work_not_in_progress():
ledger = Ledger()
ledger = scoped()
with pytest.raises(LedgerRefusal, match="in_progress"):
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
def test_release_returns_work_to_pending_without_a_revision():
ledger = Ledger()
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, attempt, reason="needs input")
@@ -172,7 +187,7 @@ def test_release_returns_work_to_pending_without_a_revision():
def test_request_approval_refuses_until_every_deliverable_is_submitted():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
@@ -182,7 +197,7 @@ def test_request_approval_refuses_until_every_deliverable_is_submitted():
def test_approval_moves_through_awaiting_to_approved_by_the_user():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
@@ -198,14 +213,14 @@ def test_approval_moves_through_awaiting_to_approved_by_the_user():
def test_approve_refuses_a_phase_not_awaiting_approval():
ledger = Ledger()
ledger = scoped()
with pytest.raises(LedgerRefusal, match="awaiting_approval"):
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
def test_pre_approved_ledger_grants_approval_immediately():
ledger = Ledger(pre_approved=True)
ledger = scoped(pre_approved=True)
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
@@ -216,7 +231,7 @@ def test_pre_approved_ledger_grants_approval_immediately():
def test_request_changes_reopens_the_phase_for_further_work():
ledger = Ledger()
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
@@ -228,7 +243,7 @@ def test_request_changes_reopens_the_phase_for_further_work():
def test_finish_is_allowed_only_once_packaging_is_approved():
ledger = Ledger()
ledger = scoped()
through_phase_three(ledger)
assert not ledger.can_finish()
@@ -242,7 +257,7 @@ def test_finish_is_allowed_only_once_packaging_is_approved():
def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
ledger = Ledger()
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
@@ -274,7 +289,7 @@ def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
ledger = Ledger()
ledger = scoped()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
@@ -289,14 +304,14 @@ def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
def test_reopen_refuses_deliverables_that_were_never_submitted():
ledger = Ledger()
ledger = scoped()
with pytest.raises(LedgerRefusal, match="submitted"):
ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator")
def test_reopen_escalates_at_the_revision_cap_without_changing_state():
ledger = Ledger(max_revisions=2)
ledger = scoped(max_revisions=2)
produce(ledger, K.REQUIREMENTS)
for _ in range(2):
ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator")
@@ -313,7 +328,7 @@ def test_reopen_escalates_at_the_revision_cap_without_changing_state():
def test_user_can_override_the_revision_cap():
ledger = Ledger(max_revisions=1)
ledger = scoped(max_revisions=1)
produce(ledger, K.REQUIREMENTS)
ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
@@ -327,7 +342,7 @@ def test_user_can_override_the_revision_cap():
def test_default_revision_cap_is_three():
assert Ledger().max_revisions == 3
assert scoped().max_revisions == 3
# --- records and persistence -------------------------------------------------------
@@ -335,16 +350,16 @@ def test_default_revision_cap_is_three():
def test_transitions_are_recorded_as_ordered_events():
ledger = Ledger()
produce(ledger, K.REQUIREMENTS)
produce(ledger, K.SCOPE)
assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [
(1, "started", "requirements"),
(2, "submitted", "requirements"),
(1, "started", "scope"),
(2, "submitted", "scope"),
]
def test_round_trips_through_plain_data():
ledger = Ledger(max_revisions=5)
ledger = scoped(max_revisions=5)
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user")
@@ -358,7 +373,7 @@ def test_round_trips_through_plain_data():
def test_revising_the_target_architecture_returns_the_code_to_draft():
ledger = Ledger()
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(

View File

@@ -10,6 +10,8 @@ from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.states.lessons import refusal
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
@@ -195,9 +197,9 @@ async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_pa
"reason": "stopped",
}
]
assert not (
tmp_path / "deliverables" / "cases" / "unassigned" / "executions"
).exists()
# Only the scope had been produced, so only the scope was written.
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
assert written == {"scope.md"}
async def test_a_cloud_read_refusal_keeps_only_its_leading_words(tmp_path):

View File

@@ -48,14 +48,14 @@ async def test_an_execution_waiting_for_its_user_has_written_what_exists(tmp_pat
waiting = await engine.start("Ingest", customer_case="acme", execution_id="v1")
# It stopped for the first Approval: only the first Phase has been produced.
# It stopped for the first Approval: only the scope has been produced.
assert waiting.status == "awaiting_user"
out = folder(tmp_path, "v1")
manifest = json.loads((out / "execution.json").read_text())
assert manifest["status"] == "in_progress"
assert set(manifest["deliverables"]) == {"requirements", "technical_decomposition"}
assert (out / "deliverables" / "requirements.md").read_text()
assert (out / "requirements.json").is_file()
assert set(manifest["deliverables"]) == {"scope"}
assert (out / "deliverables" / "scope.md").read_text()
assert (out / "scope.json").is_file() and (out / "scope.yaml").is_file()
assert manifest["solution_guide"] is None
# Unfinished work is never something a Change request can start from.
try:

View File

@@ -5,6 +5,7 @@ from app.adk.roles import PILLAR_ADVISOR
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.tools.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.tools.toolkit import make_tool_factory
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
PILLARS = {
"security": "waf-security",

View File

@@ -28,6 +28,8 @@ from app.workflows.target_architecture import (
parse_graph,
render_mermaid,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId

View File

@@ -25,6 +25,9 @@ from app.nodes.router import (
)
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
from tests.scoped import scoped_ledger
K = DeliverableKind
@@ -122,6 +125,8 @@ async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path)
assert result.status == "completed"
assert "orchestrator" not in {run.role for run in runtime.runs}
assert actions(result) == [
"delegate", # the scope
"request_approval",
"delegate", # requirements
"delegate", # technical decomposition
"request_approval",
@@ -198,14 +203,18 @@ async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
turns = {
**once(),
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
REPLY_WRITER: [
[Call("write_reply", {"text": "The scope, ready."})],
[Call("write_reply", {"text": "Two requirements, ready."})],
],
}
engine, runtime = engine_for(tmp_path, turns, writer=True)
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
# The writer ran once for the scope and once for the requirements.
writer_run = [run for run in runtime.runs if run.role == REPLY_WRITER][-1]
assert writer_run.tools == ("write_reply",)
assert writer_run.task.context["facts"]["deliverables"] == [
"requirements",
@@ -478,6 +487,7 @@ async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_pa
[(state, questions, _)] = decider.asked
assert "The user's message: Use customer-managed keys on the topic." in state
assert questions["change_target"].options == (
"scope",
"requirements",
"technical_decomposition",
"product_selection",
@@ -689,9 +699,8 @@ def test_a_quoted_question_is_removed_from_what_the_user_said():
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
from app.nodes.router import TurnView
from app.states.ledger import Ledger
ledger = Ledger()
ledger = scoped_ledger()
ledger.start([K.REQUIREMENTS], actor="orchestrator")
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
decider = FakeDecider(
@@ -765,9 +774,8 @@ async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
):
"""Evaluation case 04: a requirement said again in reply to a later question."""
from app.nodes.router import Readings, TurnView
from app.states.ledger import Ledger
ledger = Ledger()
ledger = scoped_ledger()
ledger.start([K.REQUIREMENTS], actor="orchestrator")
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
view = TurnView(

View File

@@ -6,7 +6,7 @@ import pytest
import yaml
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
@@ -25,6 +25,7 @@ ROLES = {
},
}
ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist
SCOPE_ANALYST = DEPENDENCY_GRAPH[K.SCOPE].specialist
SPEC = """
metadata: {engagement_id: ENG-77}
engagement: {kind: platform_foundation}
@@ -100,7 +101,8 @@ async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_p
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
assert (intent / "001.scope.yaml").read_text() == SPEC
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "APPROVED"
# Its Execution completed, so this copy is the record a change starts from.
assert completed["metadata"]["status"] == "BASELINE_LOCKED"
assert completed["networking"]["cidr_allocations"] == "decide"
assert completed["provenance"]["estate.regions.primary"] == "supplied"
assert [a["field"] for a in completed["inferred_assumptions"]] == [
@@ -112,7 +114,7 @@ async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_p
"supplied": "intent/001.scope.yaml",
}
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["scope"]["status"] == "APPROVED"
assert manifest["scope"]["status"] == "BASELINE_LOCKED"
assert manifest["scope"]["readiness"]["assumed"] == 1
@@ -132,16 +134,179 @@ async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path):
)
async def test_without_a_spec_nothing_changes(tmp_path):
def settle(**values) -> Call:
"""What a scope analyst records when it must assume what nobody stated."""
return Call(
"scope_set",
{
"entries": [
{
"field": field,
"value": value,
"source": "assumed",
"note": "A default.",
}
for field, value in values.items()
]
},
)
DRAFTED = [
Call(
"scope_set",
{
"entries": [
{
"field": "intent.statement",
"value": "Ingest webhooks",
"source": "request",
},
{"field": "engagement.kind", "value": "workload", "source": "request"},
]
},
),
settle(
**{
"estate.cloud_provider": "gcp",
"estate.mode": "greenfield",
"estate.regions.primary": "europe-west2",
"estate.data_residency": "United Kingdom",
"workload.users.who": "systems",
"workload.users.count": 10,
"workload.availability.target": "99.9%",
"compliance.frameworks": [],
"compliance.data_classification": "internal",
"cost.monthly_budget": 500,
}
),
]
async def test_without_a_spec_the_scope_analyst_drafts_one_from_the_request(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append([*DRAFTED, sample_submission(K.SCOPE)])
result = await built.start(
"Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1"
)
assert result.status == "completed"
intent = tmp_path / "deliverables" / "cases" / "acme" / "intent"
# Nothing was supplied, so there is no supplied file; the completed copy is
# what the analyst drafted.
assert not (intent / "001.scope.yaml").exists()
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["intent"]["statement"] == "Ingest webhooks"
assert completed["metadata"]["customer_case"] == "acme"
assert completed["provenance"]["intent.statement"] == "request"
assert completed["provenance"]["estate.regions.primary"] == "assumed"
assert completed["readiness"]["assumed"] == 10
assumed = {a["field"] for a in completed["inferred_assumptions"]}
assert "cost.monthly_budget" in assumed
# What was assumed is shown apart from what the customer stated.
scope = runtime.seen[ANALYST]["scope"]
assert scope["stated"]["intent"]["statement"] == "Ingest webhooks"
assert "estate" not in scope["stated"]
assert {"field": "cost.monthly_budget", "value": 500, "why": "A default."} in (
scope["assumed"]
)
document = tmp_path / "deliverables" / "cases" / "acme" / "executions" / "v1"
register = (document / "deliverables" / "scope.md").read_text()
assert "## Information register" in register and "### Assumptions" in register
assert "| intent.statement | Ingest webhooks | the request |" in register
async def test_a_scope_with_an_essential_field_unsettled_cannot_be_submitted(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
# It records the statement and nothing else, then tries to submit, twice.
runtime._turns[SCOPE_ANALYST].append(
[DRAFTED[0], sample_submission(K.SCOPE), sample_submission(K.SCOPE)]
)
result = await built.start("Ingest webhooks", pre_approved=True)
assert result.state["scope"] is None
assert runtime.seen[ANALYST] == {"skills": ("base",), "scope": None}
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["scope"] is None
assert not list(tmp_path.rglob("*.scope*.yaml"))
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
refused = run.results[1][1]
assert refused.startswith("Not submitted. Essential fields of the scope spec")
assert "estate.regions.primary" in refused
assert result.status == "completed_with_warnings"
async def test_what_the_customer_stated_is_not_overwritten_by_the_analyst(tmp_path):
built, runtime = engine(tmp_path)
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append(
[
Call("scope_read"),
settle(**{"estate.regions.primary": "us-central1", "nonsense.field": 1}),
Call(
"scope_set",
{
"entries": [
{
"field": "estate.regions.primary",
"value": "europe-west1",
"source": "answered",
}
]
},
),
Call(
"scope_set",
{
"entries": [
{
"field": "cost.monthly_budget",
"value": "lots",
"source": "answered",
}
]
},
),
sample_submission(K.SCOPE),
]
)
await built.start("", scope=SPEC, execution_id="v1")
run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST)
read, assumed, answered, mistyped = (reply for _, reply in run.results[:4])
assert "Kind of engagement: platform_foundation" in read
assert "Left to you to decide: networking.cidr_allocations" in read
assert "estate.regions.primary was stated in the scope spec" in assumed
assert "nonsense.field is not a field" in assumed
# The customer's own answer does change what the file said.
assert answered == "Recorded 1: estate.regions.primary."
assert "cost.monthly_budget: must be a number" in mistyped
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["estate"]["regions"]["primary"] == "europe-west1"
assert completed["provenance"]["estate.regions.primary"] == "answered"
async def test_the_scope_is_approved_before_requirements_begin(tmp_path):
built, runtime = engine(tmp_path)
waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1")
# Only the scope analyst has run, and the customer is asked to approve the scope.
assert {run.role for run in runtime.runs} == {SCOPE_ANALYST}
assert waiting.prompt["approval"]["phase"] == "scoping"
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "UNDER_REVIEW"
approved = await built.resume(
"v1", {"approval": {"phase": "scoping", "decision": "approve"}}
)
assert ANALYST in {run.role for run in runtime.runs}
assert approved.prompt["approval"]["phase"] == "requirements_discovery"
completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert completed["metadata"]["status"] == "APPROVED"
async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path):
@@ -196,3 +361,56 @@ async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path
scope=SPEC.replace(" statement: Build us a landing zone.\n", ""),
pre_approved=False,
)
async def test_a_change_request_starts_from_the_scope_that_was_delivered(tmp_path):
built, runtime = engine(tmp_path)
await built.start("", scope=SPEC, execution_id="v1")
for name, turns in runtime._turns.items():
kind = next(
k for k, spec in DEPENDENCY_GRAPH.items() if spec.specialist == name
)
turns.append([sample_submission(kind)])
runtime._turns[SCOPE_ANALYST].clear()
runtime._turns[SCOPE_ANALYST].append(
[
Call("scope_read"),
Call(
"scope_set",
{
"entries": [
{
"field": "estate.regions.secondary",
"value": "europe-west1",
"source": "answered",
}
]
},
),
sample_submission(K.SCOPE),
]
)
await built.start(
"We now need a second region.",
customer_case="eng-77",
pre_approved=True,
execution_id="v2",
baseline="v1",
)
run = [r for r in runtime.runs if r.role == SCOPE_ANALYST][-1]
# The delivered spec was there to read before anything was recorded.
assert "Kind of engagement: platform_foundation" in run.results[0][1]
intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent"
changed = yaml.safe_load((intent / "002.scope.completed.yaml").read_text())
assert changed["estate"]["regions"] == {
"primary": "europe-west2",
"secondary": "europe-west1",
"resiliency_tier": "ask",
}
assert changed["provenance"]["estate.regions.primary"] == "supplied"
assert changed["provenance"]["estate.regions.secondary"] == "answered"
# The Baseline's own copy is untouched.
first = yaml.safe_load((intent / "001.scope.completed.yaml").read_text())
assert first["estate"]["regions"]["secondary"] == "ask"

80
tests/scoped.py Normal file
View File

@@ -0,0 +1,80 @@
"""Helpers for tests about what comes after scoping.
Every Execution now begins with the scoping Phase (ADR-0008). A test about
requirements, design or validation is not about that, so these run scoping ahead
of the test's own script and leave the test where it always started: requirements
ready, nothing recorded of how the scope was agreed.
"""
from __future__ import annotations
from typing import Any
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
from app.adk.runtime import Submitted
from app.agent import ExecutionEngine, Interaction
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
SCOPE_ANALYST = DEPENDENCY_GRAPH[DeliverableKind.SCOPE].specialist
ORCHESTRATOR = "orchestrator"
class ScopedRuntime(FakeAgentRuntime):
"""A scripted runtime that first has the scope produced and its Approval asked
for, outside the script and outside `runs`."""
def __init__(self, turns: Any) -> None:
super().__init__(turns)
self._next = "delegate"
async def run(self, role, task, tools): # type: ignore[no-untyped-def]
by_name = {tool.name: tool for tool in tools}
if role.name == SCOPE_ANALYST and not self.remaining(SCOPE_ANALYST):
# Submitted as it stands: a test about a later Deliverable's draft is
# not asking whether the scope's draft was shown first.
call = sample_submission(DeliverableKind.SCOPE)
return Submitted(call.args["content"], call.args["attachments"])
if role.name == ORCHESTRATOR and self._next == "delegate":
self._next = "approve"
asked = await by_name["delegate"].handler(
kinds=[DeliverableKind.SCOPE.value]
)
return asked.outcome
if role.name == ORCHESTRATOR and self._next == "approve":
self._next = "done"
asked = await by_name["request_approval"].handler(
phase=PhaseId.SCOPING.value, summary="The scope is ready."
)
if asked.outcome is not None:
return asked.outcome # a user is present: their Approval is awaited
return await super().run(role, task, tools)
class ScopedEngine(ExecutionEngine):
"""An engine whose `start` grants the scoping Approval when it is asked for, so
an interactive test meets the first prompt it was written about."""
async def start(self, *args: Any, **kwargs: Any) -> Interaction:
interaction = await super().start(*args, **kwargs)
approval = (interaction.prompt or {}).get("approval") or {}
if approval.get("phase") == PhaseId.SCOPING.value:
interaction = await self.resume(
interaction.execution_id,
{"approval": {"phase": PhaseId.SCOPING.value, "decision": "approve"}},
)
return interaction
def scoped_ledger(**options: Any): # type: ignore[no-untyped-def]
"""A Ledger whose scope is agreed: where requirements discovery begins."""
from app.states.ledger import Ledger, PhaseStatus
ledger = Ledger(**options)
attempt = ledger.start([DeliverableKind.SCOPE], actor=ORCHESTRATOR)
ledger.submit(
DeliverableKind.SCOPE, attempt[DeliverableKind.SCOPE], actor=SCOPE_ANALYST
)
ledger.request_approval(PhaseId.SCOPING, actor=ORCHESTRATOR)
if ledger.phase_status(PhaseId.SCOPING) is PhaseStatus.AWAITING_APPROVAL:
ledger.approve(PhaseId.SCOPING, actor="user")
return ledger

View File

@@ -57,12 +57,11 @@ async def test_preview_mcp_servers_follow_the_setting(tmp_path):
async def test_the_assembly_routes_by_default_and_runs_without_a_jev_key(tmp_path):
"""With no Jev key the router follows its playbook on rules alone."""
from app.adk.fake_runtime import sample_submission
from app.adk.fake_runtime import Call
from app.adk.runtime import NoDecider
from app.workflows.deliverables import DeliverableKind
runtime = FakeAgentRuntime(
{"requirements-analyst": [[sample_submission(DeliverableKind.REQUIREMENTS)]]}
{"scope-analyst": [[Call("needs_input", {"questions": ["Which region?"]})]]}
)
async with open_assembly(
@@ -74,9 +73,10 @@ async def test_the_assembly_routes_by_default_and_runs_without_a_jev_key(tmp_pat
interaction = await assembly.engine.start("Design ingestion")
assert isinstance(assembly.decider, NoDecider)
# The router delegated requirements with no Orchestrator model, then the
# decomposer (unscripted here) stopped and its question reached the user.
assert [run.role for run in runtime.runs] == ["requirements-analyst"]
# The router delegated the scope with no Orchestrator model, and the scope
# analyst's question reached the user.
assert [run.role for run in runtime.runs] == ["scope-analyst"]
assert "scope_set" in runtime.runs[0].tools
assert interaction.status == "awaiting_user"
assert "reply-writer" in assembly.roles