feat: Change requests against a Baseline (ADR-0006)

A caller restates a request to say how requirements have changed, naming a
completed Execution of the same Customer case as the Baseline.

- app/baseline.py reads a Baseline from its files and refuses one that
  does not exist, did not complete or belongs to another case, before
  anything runs or is recorded. A Baseline is never modified.
- Requirements carry a change (unchanged, added, changed, removed) and a
  reason. Submission is refused when a Baseline requirement disappears, an
  id is reused, or an unchanged requirement's text differs. A removed
  requirement stays on record and asks nothing downstream.
- Each Specialist starts from its delivered version; the graph and the code
  are its working copy, and the description is carried over when the
  design's shape is the same.
- app/workflows/changes.py compares versions by id and traces what each
  changed requirement reaches; anything else that changed is listed as
  unrequested. Approvals show the change for their Phase, and changes.json
  is written with the full Deliverables.
- Intake no longer starts a message read as a change: it names the
  Execution to give as the Baseline.
- A fifth evaluation case runs a request and then a change to it, with
  four checks that the change is recorded, reaches Checks, stays contained
  and costs less than its Baseline.

Tested with scripted agents only; no Change request has run with real
models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 11:38:15 +01:00
parent db28d02f56
commit 5ef434d00e
32 changed files with 1824 additions and 40 deletions

View File

@@ -387,3 +387,66 @@ def test_a_pre_approved_task_is_not_read_as_a_question(tmp_path):
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert decider.asked == []
assert runtime.runs
def delivered_case(tmp_path: Path, case: str = "acme") -> None:
folder = tmp_path / "deliverables" / "cases" / case / "intent"
folder.mkdir(parents=True)
(folder / "history.json").write_text(
'{"customer_case": "acme", "intents": [{"version": 1, "workflow_request": '
'"Design ingestion.", "execution_id": "v1", "status": "completed"}]}'
)
def test_a_change_to_delivered_work_names_the_baseline_and_starts_nothing(tmp_path):
delivered_case(tmp_path)
decider = FakeDecider({"intake": Answer("change_request", 0.98)})
client, runtime = client_for(tmp_path, {}, decider)
with client:
task = send(
client,
[{"text": "Availability must now be 99.99%."}],
metadata={"customer_case": "acme"},
)
text, data = status_parts(task)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert 'send this again with "baseline" set to "v1"' in text
assert data == {"status": "answered", "intake": "change_request", "baseline": "v1"}
assert runtime.runs == []
def test_a_caller_who_says_it_is_new_work_is_not_second_guessed(tmp_path):
delivered_case(tmp_path)
decider = FakeDecider({"intake": Answer("change_request", 0.98)})
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
client, runtime = client_for(tmp_path, turns, decider)
with client:
task = send(
client,
[{"text": "Availability must now be 99.99%."}],
metadata={"customer_case": "acme", "new_request": True},
)
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
assert decider.asked == [] and len(runtime.runs) == 1
def test_a_baseline_that_cannot_be_one_fails_the_task_and_generate(tmp_path):
client, runtime = client_for(tmp_path, {})
with client:
task = send(
client,
[{"text": "A change."}],
metadata={"customer_case": "acme", "baseline": "nope"},
)
response = client.post(
"/generate",
json={"request": "A change.", "customer_case": "acme", "baseline": "nope"},
)
assert task["status"]["state"] == "TASK_STATE_FAILED"
assert "is not an Execution of Customer case acme" in status_parts(task)[0]
assert response.status_code == 400
assert "is not an Execution of Customer case acme" in response.json()["error"]
assert runtime.runs == []

View File

@@ -1,5 +1,6 @@
"""Scoring a finished Execution against a benchmark case, without models or network."""
import json
from pathlib import Path
from app.adk.fake_runtime import SAMPLE_DATA
@@ -614,3 +615,133 @@ def test_calibration_without_labels_still_measures_stability():
assert result["labelled"] == 0
assert result["judges"]["jev"]["agreement_with_labels"] is None
assert result["judges"]["jev"]["stability"] == 1.0
# --- a Change request stays contained ---------------------------------------------------
def change_interaction(graph_change=None, requirement=None) -> Interaction:
from app.adk.fake_runtime import SAMPLE_DATA
names = {
"requirements": "requirements.json",
"technical_decomposition": "components.json",
"design_recommendations": "recommendations.json",
"target_architecture": "target-architecture.json",
}
def version() -> dict:
return {
kind: {"attachments": {name: SAMPLE_DATA[name]}}
for kind, name in names.items()
}
baseline, current = version(), version()
requirements = json.loads(SAMPLE_DATA["requirements.json"])
for item in requirements:
item["change"] = "unchanged"
if requirement is not False:
requirements[1].update(
requirement
or {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
)
current["requirements"]["attachments"] = {
"requirements.json": json.dumps(requirements)
}
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
if graph_change:
graph_change(graph)
current["target_architecture"]["attachments"] = {
"target-architecture.json": json.dumps(graph)
}
state = {"baseline": {"deliverables": baseline}, "deliverables": current}
return Interaction("v2", "completed", None, state)
def results(checks) -> dict[str, bool]:
return {check.name: check.passed for check in checks}
def test_a_contained_change_passes_every_change_check():
from eval.scoring import score_change
def tighten(graph):
graph["checks"][0]["statement"] = "The service runs at least three instances."
checks = score_change(change_interaction(tighten), 4_000_000, 1_500_000)
assert results(checks) == {
"change_is_recorded": True,
"changed_requirements_reach_checks": True,
"change_is_contained": True,
"change_costs_less_than_baseline": True,
}
def test_a_changed_requirement_that_changes_no_check_fails():
from eval.scoring import score_change
checks = score_change(change_interaction(), 4_000_000, 1_500_000)
by_name = {check.name: check for check in checks}
assert not by_name["changed_requirements_reach_checks"].passed
assert by_name["changed_requirements_reach_checks"].detail == (
"no Check changed for: NFR-1"
)
def test_a_change_that_cost_more_than_the_baseline_or_changed_nothing_fails():
from eval.scoring import score_change
dearer = results(score_change(change_interaction(), 2_000_000, 3_000_000))
nothing = results(
score_change(change_interaction(requirement=False), 2_000_000, 1_000_000)
)
assert not dearer["change_costs_less_than_baseline"]
assert not nothing["change_is_recorded"]
def test_many_changes_nobody_asked_for_fail_containment():
from eval.scoring import score_change
def rewrite(graph):
# Everything about the identity changes; NFR-1 reaches none of it.
graph["elements"][1]["name"] = "A different identity"
graph["checks"][1]["statement"] = "Something else."
graph["elements"].append(
{
"id": "E-9",
"name": "An extra key",
"product": "Example KMS",
"resource_type": "example_key",
}
)
graph["checks"].append(
{"id": "K-9", "applies_to": "E-9", "statement": "x", "source": "DR-1"}
)
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
interaction = change_interaction(rewrite)
for version in (
interaction.state["baseline"]["deliverables"],
interaction.state["deliverables"],
):
version["technical_decomposition"]["attachments"] = {
"components.json": json.dumps(components)
}
by_name = {check.name: check for check in score_change(interaction, 4, 1)}
assert not by_name["change_is_contained"].passed
assert by_name["change_is_contained"].detail.startswith("4 of 5 items changed")
def test_the_change_case_loads_with_its_statement_of_change():
from eval.scoring import load_cases
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
case = next(c for c in cases if c.id == "case-05-change-retention")
assert "7 years" in case.change
assert all(not c.change for c in cases if c.id != case.id)

View File

@@ -0,0 +1,633 @@
"""Change requests (ADR-0006): an Execution that starts from a Baseline.
A Baseline is a completed Execution of the same Customer case. A Change request is a
new Execution that begins with the Baseline's Deliverables; the Baseline itself is
never modified.
"""
import hashlib
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import (
SAMPLE_DATA,
Call,
FakeAgentRuntime,
sample_submission,
)
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.baseline import BaselineError, load_baseline
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
K = DeliverableKind
CASE = "acme-webhooks"
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
def revised(**changes) -> Call:
"""The requirements of a Change request: the Baseline's, each marked, plus what
`changes` adds or alters by id."""
requirements = json.loads(SAMPLE_DATA["requirements.json"])
for item in requirements:
item["change"] = "unchanged"
if item["id"] in changes:
item.update(changes.pop(item["id"]))
requirements += [{"id": key, **value} for key, value in changes.items()]
call = sample_submission(K.REQUIREMENTS)
return Call(
call.tool,
{
**call.args,
"attachments": {"requirements.json": json.dumps(requirements)},
},
)
RETENTION = {
"NFR-1": {
"change": "changed",
"text": "99.99% availability.",
"reason": "The service is now customer-facing.",
}
}
def changed(**overrides) -> dict:
"""Scripted turns for a Change request that changes NFR-1 and nothing else."""
turns = once()
turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]]
turns.update(overrides)
return turns
def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine:
return ExecutionEngine(
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
**options,
)
async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"):
"""A completed Execution on disk, to be a Baseline."""
result = await engine_for(tmp_path, once()).start(
"Ingest webhooks",
customer_case=case,
pre_approved=True,
execution_id=execution_id,
)
assert result.status == "completed"
return result
def digest(folder: Path) -> dict[str, str]:
return {
str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted(folder.rglob("*"))
if path.is_file()
}
def history(tmp_path: Path, case: str = CASE) -> list[dict]:
path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json"
return json.loads(path.read_text())["intents"]
# --- step 2: start against a Baseline --------------------------------------------------
async def test_a_baseline_is_read_from_its_files(tmp_path):
await delivered(tmp_path)
baseline = load_baseline(tmp_path, CASE, "v1")
assert baseline["execution_id"] == "v1"
assert baseline["workflow_request"] == "Ingest webhooks"
assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
graph = baseline["deliverables"]["target_architecture"]["attachments"]
assert "target-architecture.json" in graph
assert baseline["deliverables"]["requirements"]["content"].startswith("#")
async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path):
await delivered(tmp_path)
result = await engine_for(tmp_path, changed()).start(
"Retention goes from 30 days to 7 years.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
assert result.status == "completed"
assert result.state["baseline"]["execution_id"] == "v1"
first, second = history(tmp_path)
assert first["baseline"] is None
assert second["baseline"] == "v1" and second["execution_id"] == "v2"
case = tmp_path / "deliverables" / "cases" / CASE
manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text())
assert manifest["baseline"] == "v1"
assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text()
async def test_the_baseline_is_never_modified(tmp_path):
await delivered(tmp_path)
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
before = digest(folder)
await engine_for(tmp_path, changed()).start(
"Add a second region.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
assert digest(folder) == before
async def test_a_change_request_may_itself_be_a_baseline(tmp_path):
await delivered(tmp_path)
for version, previous in (("v2", "v1"), ("v3", "v2")):
await engine_for(tmp_path, changed()).start(
"Another change.",
customer_case=CASE,
pre_approved=True,
execution_id=version,
baseline=previous,
)
assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"]
assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2"
async def test_without_a_baseline_an_execution_starts_clean(tmp_path):
result = await delivered(tmp_path)
assert result.state["baseline"] is None
manifest = json.loads(
(
tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json"
).read_text()
)
assert manifest["baseline"] is None
@pytest.mark.parametrize(
"case, baseline, message",
[
(CASE, "nope", "is not an Execution of Customer case acme-webhooks"),
("other-client", "v1", "belongs to another Customer case"),
(None, "v1", "needs a customer_case"),
(CASE, "../v1", "is not an Execution id"),
],
)
async def test_a_baseline_that_cannot_be_one_refuses_the_start(
tmp_path, case, baseline, message
):
await delivered(tmp_path)
engine = engine_for(tmp_path, once())
with pytest.raises(BaselineError, match=message):
await engine.start(
"A change.", customer_case=case, execution_id="v2", baseline=baseline
)
# Refused before anything ran or was recorded.
assert [item["execution_id"] for item in history(tmp_path)] == ["v1"]
assert not (tmp_path / "deliverables/cases/other-client").exists()
async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path):
from tests.execution.test_router import failing
turns = once()
turns["validator"] = [[failing("K-1")]]
engine = engine_for(tmp_path, turns, max_revisions=0)
unfinished = await engine.start(
"Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1"
)
assert unfinished.status == "completed_with_warnings"
with pytest.raises(BaselineError, match="did not complete"):
load_baseline(tmp_path, CASE, "v1")
# --- step 3: the requirements say what changed -------------------------------------------
def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]:
from app.workflows.deliverables import check_submission
baseline = baseline or {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
}
return check_submission(
K.REQUIREMENTS,
revised_call.args["content"],
revised_call.args["attachments"],
baseline=baseline,
)
def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted():
call = revised(
**{
"NFR-1": RETENTION["NFR-1"],
"NFR-2": {
"kind": "non_functional",
"text": "A second region.",
"conditions": ["The service runs in two regions."],
"change": "added",
"reason": "Customers are now in two continents.",
},
}
)
assert requirements_check(call) == ()
def test_every_requirement_in_a_change_request_says_whether_it_changed():
call = sample_submission(K.REQUIREMENTS) # no `change` on anything
missing = requirements_check(call)
assert "requirements.json: FR-1 needs a `change`" in missing[0]
assert len(missing) == 2
def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text():
no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}})
quietly = revised(**{"NFR-1": {"text": "99.99% availability."}})
quietly_other = revised(
**{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}}
)
assert requirements_check(no_reason) == (
"requirements.json: NFR-1 is changed and needs a `reason`",
)
assert (
"NFR-1 is marked unchanged, but its text differs"
in (requirements_check(quietly)[0])
)
assert (
"FR-1 is marked unchanged, but its text differs"
in (requirements_check(quietly_other)[0])
)
def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused():
call = revised(**{"NFR-1": RETENTION["NFR-1"]})
requirements = json.loads(call.args["attachments"]["requirements.json"])
dropped = Call(
call.tool,
{
**call.args,
"attachments": {"requirements.json": json.dumps(requirements[1:])},
},
)
reused = revised(
**{
"NFR-1": {
"change": "added",
"text": "Something else entirely.",
"reason": "New.",
}
}
)
invented = revised(
**{
"NFR-1": RETENTION["NFR-1"],
"NFR-9": {
"kind": "non_functional",
"text": "x",
"conditions": ["x"],
"change": "changed",
"reason": "y",
},
}
)
assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0]
assert (
"NFR-1 is marked added, but the Baseline has that id"
in (requirements_check(reused)[0])
)
assert (
"NFR-9 is marked changed, but the Baseline has no such requirement"
in (requirements_check(invented)[0])
)
def test_a_change_request_must_change_something():
assert (
"nothing is marked added, changed or removed"
in (requirements_check(revised())[0])
)
def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream():
from app.workflows.chain import read_list
call = revised(
**{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}}
)
upstream = {"requirements": {"attachments": call.args["attachments"]}}
assert requirements_check(call) == ()
live = read_list(upstream, "requirements", "requirements.json")
assert [item["id"] for item in live] == ["FR-1"]
assert '"NFR-1"' in call.args["attachments"]["requirements.json"]
# --- step 4: later Specialists revise from the Baseline ---------------------------------
async def test_each_specialist_is_given_its_delivered_version_and_what_changed(
tmp_path,
):
await delivered(tmp_path)
runtime = FakeAgentRuntime(changed())
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
by_role = {run.role: run for run in runtime.runs}
analyst = by_role["requirements-analyst"].task.context["baseline"]
assert analyst["execution_id"] == "v1"
assert analyst["original_request"] == "Ingest webhooks"
assert "requirements.json" in analyst["deliverable"]["attachments"]
assert analyst["changes"] is None # nothing has changed yet
architect = by_role["target-architect"].task.context
assert architect["baseline"]["changes"]["requirements"] == [
{
"id": "NFR-1",
"change": "changed",
"reason": "The service is now customer-facing.",
"text": "99.99% availability.",
}
]
# Its working copy already holds the delivered graph.
assert architect["baseline"]["deliverable"]["attachments"][
"target-architecture.json"
]
assert "graph_read" in by_role["target-architect"].tools
# The guide is assembled afresh: the Baseline's is not handed back.
assert by_role["packager"].task.context["baseline"]["deliverable"] is None
async def test_a_description_whose_design_did_not_change_shape_is_carried_over(
tmp_path,
):
await delivered(tmp_path)
runtime = FakeAgentRuntime(changed())
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
result = await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
# The same Elements and Relationships as the Baseline: the describer is not run.
assert "describer" not in {run.role for run in runtime.runs}
assert result.state["deliverables"]["architecture_description"]["content"]
async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path):
await delivered(tmp_path)
seen: dict = {}
def author(task):
seen["working_copy"] = task.context["working_copy"]
return [sample_submission(K.DEPLOYMENT_GUIDANCE)]
engine = engine_for(tmp_path, changed(**{"iac-author": [author]}))
await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
# It is told nothing of a working copy on a first attempt; the tools hold it.
assert seen["working_copy"] is None
# --- what changed, worked out from the ids ------------------------------------------------
def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]:
"""A Baseline and a revised set of Deliverables, as data files."""
def deliverables() -> dict:
return {
kind: {"attachments": {name: SAMPLE_DATA[name]}}
for kind, name in (
("requirements", "requirements.json"),
("technical_decomposition", "components.json"),
("design_recommendations", "recommendations.json"),
("target_architecture", "target-architecture.json"),
)
}
before, after = deliverables(), deliverables()
change = requirement_change or RETENTION
after["requirements"]["attachments"] = revised(
**json.loads(json.dumps(change))
).args["attachments"]
if graph_change:
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph_change(graph)
after["target_architecture"]["attachments"] = {
"target-architecture.json": json.dumps(graph)
}
return before, after
def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested():
from app.workflows.changes import change_summary
def tighten(graph):
graph["checks"][0]["statement"] = "The service runs at least three instances."
summary = change_summary(*versions(tighten))
assert [item["id"] for item in summary["requirements"]] == ["NFR-1"]
assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []}
assert summary["elements"] == {"added": [], "changed": [], "removed": []}
assert summary["unrequested"] == {}
def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested():
from app.workflows.changes import change_summary
def wander(graph):
# K-2 comes from DR-1, which did not change, on E-2, which realises nothing
# the changed requirement touches. NFR-1 reaches K-1 and E-1 only.
graph["checks"][1]["statement"] = "The identity has no keys."
change = {
"NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
}
before, after = versions(wander, change)
# NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2.
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
for version in (before, after):
version["technical_decomposition"]["attachments"] = {
"components.json": json.dumps(components)
}
summary = change_summary(before, after)
assert summary["checks"]["changed"] == ["K-2"]
assert summary["unrequested"] == {"checks": ["K-2"]}
def test_added_and_removed_items_are_found_by_id():
from app.workflows.changes import change_summary
def grow(graph):
graph["elements"].append(
{
"id": "E-3",
"name": "Second region service",
"component": "C-1",
"product": "Example Run",
"resource_type": "example_run_service",
}
)
graph["checks"] = [graph["checks"][0]] + [
{"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"}
]
summary = change_summary(*versions(grow))
assert summary["elements"]["added"] == ["E-3"]
assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]}
# K-2 came from DR-1 on E-2: nothing that changed reaches it.
assert summary["unrequested"] == {"checks": ["K-2"]}
# --- step 5: Approvals show the change --------------------------------------------------
async def test_an_approval_in_a_change_request_shows_what_that_phase_changes(
tmp_path,
):
from app.presentation import describe_prompt
await delivered(tmp_path)
engine = engine_for(tmp_path, changed())
first = await engine.start(
"Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1"
)
approval = first.prompt["approval"]
assert approval["phase"] == "requirements_discovery"
assert approval["baseline"] == "v1"
assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"]
text = describe_prompt(first.prompt)
assert "This changes the architecture delivered in v1." in text
assert (
"- Requirement NFR-1 changed: 99.99% availability. (The service is now "
"customer-facing.)"
) in text
second = await engine.resume(
"v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}}
)
# The design did not change, and the Approval says so.
design = second.prompt["approval"]
assert design["phase"] == "solution_architecture" and design["changes"] == {}
assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt)
async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path):
first = await engine_for(tmp_path, once()).start("Ingest webhooks")
assert "changes" not in first.prompt["approval"]
# --- step 6: the change summary and the line of versions ----------------------------------
async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path):
from app.workflows.changes import change_summary
await delivered(tmp_path)
result = await engine_for(tmp_path, changed()).start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
case = tmp_path / "deliverables" / "cases" / CASE / "executions"
written = json.loads((case / "v2" / "changes.json").read_text())
assert [item["id"] for item in written["requirements"]] == ["NFR-1"]
assert written["unrequested"] == {}
# It matches a comparison of the two Executions' data files.
assert written == change_summary(
load_baseline(tmp_path, CASE, "v1")["deliverables"],
result.state["deliverables"],
)
# The Change request stands on its own: every Deliverable is written in full.
manifest = json.loads((case / "v2" / "execution.json").read_text())
assert manifest["changes"] == "changes.json"
assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
assert (case / "v2" / "solution-architecture-guide.md").is_file()
assert not (case / "v1" / "changes.json").exists()

View File

@@ -11,7 +11,12 @@ def delivered(tmp_path, case: str, statuses: list[str]) -> None:
folder = tmp_path / "deliverables" / "cases" / case / "intent"
folder.mkdir(parents=True)
intents = [
{"version": n, "workflow_request": f"Design thing {n}.", "status": status}
{
"version": n,
"workflow_request": f"Design thing {n}.",
"status": status,
"execution_id": f"v{n}",
}
for n, status in enumerate(statuses, start=1)
]
(folder / "history.json").write_text(
@@ -53,6 +58,7 @@ async def test_a_change_needs_something_delivered_to_change(tmp_path):
no_case = await intake.read("Raise availability to 99.99%.")
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
assert with_history.latest == "v1" # what the change would change
assert "- Design thing 1." in decider.asked[0][0]
assert "Design thing 2." not in decider.asked[0][0] # still in progress
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST