"""Change requests (ADR-0006): an Execution that starts from a Baseline. A Baseline is a completed Execution of the same Customer case. A Change request is a new Execution that begins with the Baseline's Deliverables; the Baseline itself is never modified. """ import hashlib import json from pathlib import Path import pytest from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import ( SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission, ) from app.adk.runtime import NoDecider, RoleSpec from app.agent import ExecutionEngine from app.baseline import BaselineError, load_baseline from app.nodes.router import Router, Thresholds from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811 from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811 K = DeliverableKind CASE = "acme-webhooks" ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def once() -> dict: return { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } def revised(**changes) -> Call: """The requirements of a Change request: the Baseline's, each marked, plus what `changes` adds or alters by id.""" requirements = json.loads(SAMPLE_DATA["requirements.json"]) for item in requirements: item["change"] = "unchanged" if item["id"] in changes: item.update(changes.pop(item["id"])) requirements += [{"id": key, **value} for key, value in changes.items()] call = sample_submission(K.REQUIREMENTS) return Call( call.tool, { **call.args, "attachments": {"requirements.json": json.dumps(requirements)}, }, ) RETENTION = { "NFR-1": { "change": "changed", "text": "99.99% availability.", "reason": "The service is now customer-facing.", } } def changed(**overrides) -> dict: """Scripted turns for a Change request that changes NFR-1 and nothing else.""" turns = once() turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]] turns.update(overrides) return turns def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine: return ExecutionEngine( runtime=FakeAgentRuntime(turns), roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), **options, ) async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"): """A completed Execution on disk, to be a Baseline.""" result = await engine_for(tmp_path, once()).start( "Ingest webhooks", customer_case=case, pre_approved=True, execution_id=execution_id, ) assert result.status == "completed" return result def digest(folder: Path) -> dict[str, str]: return { str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(folder.rglob("*")) if path.is_file() } def history(tmp_path: Path, case: str = CASE) -> list[dict]: path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json" return json.loads(path.read_text())["intents"] # --- step 2: start against a Baseline -------------------------------------------------- async def test_a_baseline_is_read_from_its_files(tmp_path): await delivered(tmp_path) baseline = load_baseline(tmp_path, CASE, "v1") assert baseline["execution_id"] == "v1" assert baseline["workflow_request"] == "Ingest webhooks" assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH} graph = baseline["deliverables"]["target_architecture"]["attachments"] assert "target-architecture.json" in graph assert baseline["deliverables"]["requirements"]["content"].startswith("#") async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path): await delivered(tmp_path) result = await engine_for(tmp_path, changed()).start( "Retention goes from 30 days to 7 years.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) assert result.status == "completed" assert result.state["baseline"]["execution_id"] == "v1" first, second = history(tmp_path) assert first["baseline"] is None assert second["baseline"] == "v1" and second["execution_id"] == "v2" case = tmp_path / "deliverables" / "cases" / CASE manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text()) assert manifest["baseline"] == "v1" assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text() async def test_the_baseline_is_never_modified(tmp_path): await delivered(tmp_path) folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1" before = digest(folder) await engine_for(tmp_path, changed()).start( "Add a second region.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) assert digest(folder) == before async def test_a_change_request_may_itself_be_a_baseline(tmp_path): await delivered(tmp_path) for version, previous in (("v2", "v1"), ("v3", "v2")): await engine_for(tmp_path, changed()).start( "Another change.", customer_case=CASE, pre_approved=True, execution_id=version, baseline=previous, ) assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"] assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2" async def test_without_a_baseline_an_execution_starts_clean(tmp_path): result = await delivered(tmp_path) assert result.state["baseline"] is None manifest = json.loads( ( tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json" ).read_text() ) assert manifest["baseline"] is None @pytest.mark.parametrize( "case, baseline, message", [ (CASE, "nope", "is not an Execution of Customer case acme-webhooks"), ("other-client", "v1", "belongs to another Customer case"), (None, "v1", "needs a customer_case"), (CASE, "../v1", "is not an Execution id"), ], ) async def test_a_baseline_that_cannot_be_one_refuses_the_start( tmp_path, case, baseline, message ): await delivered(tmp_path) engine = engine_for(tmp_path, once()) with pytest.raises(BaselineError, match=message): await engine.start( "A change.", customer_case=case, execution_id="v2", baseline=baseline ) # Refused before anything ran or was recorded. assert [item["execution_id"] for item in history(tmp_path)] == ["v1"] assert not (tmp_path / "deliverables/cases/other-client").exists() async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path): from tests.execution.test_router import failing turns = once() turns["validator"] = [[failing("K-1")]] engine = engine_for(tmp_path, turns, max_revisions=0) unfinished = await engine.start( "Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1" ) assert unfinished.status == "completed_with_warnings" with pytest.raises(BaselineError, match="did not complete"): load_baseline(tmp_path, CASE, "v1") # --- step 3: the requirements say what changed ------------------------------------------- def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]: from app.workflows.deliverables import check_submission baseline = baseline or { "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} } return check_submission( K.REQUIREMENTS, revised_call.args["content"], revised_call.args["attachments"], baseline=baseline, ) def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted(): call = revised( **{ "NFR-1": RETENTION["NFR-1"], "NFR-2": { "kind": "non_functional", "text": "A second region.", "conditions": ["The service runs in two regions."], "change": "added", "reason": "Customers are now in two continents.", }, } ) assert requirements_check(call) == () def test_every_requirement_in_a_change_request_says_whether_it_changed(): call = sample_submission(K.REQUIREMENTS) # no `change` on anything missing = requirements_check(call) assert "requirements.json: FR-1 needs a `change`" in missing[0] assert len(missing) == 2 def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text(): no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}}) quietly = revised(**{"NFR-1": {"text": "99.99% availability."}}) quietly_other = revised( **{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}} ) assert requirements_check(no_reason) == ( "requirements.json: NFR-1 is changed and needs a `reason`", ) assert ( "NFR-1 is marked unchanged, but its text differs" in (requirements_check(quietly)[0]) ) assert ( "FR-1 is marked unchanged, but its text differs" in (requirements_check(quietly_other)[0]) ) def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused(): call = revised(**{"NFR-1": RETENTION["NFR-1"]}) requirements = json.loads(call.args["attachments"]["requirements.json"]) dropped = Call( call.tool, { **call.args, "attachments": {"requirements.json": json.dumps(requirements[1:])}, }, ) reused = revised( **{ "NFR-1": { "change": "added", "text": "Something else entirely.", "reason": "New.", } } ) invented = revised( **{ "NFR-1": RETENTION["NFR-1"], "NFR-9": { "kind": "non_functional", "text": "x", "conditions": ["x"], "change": "changed", "reason": "y", }, } ) assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0] assert ( "NFR-1 is marked added, but the Baseline has that id" in (requirements_check(reused)[0]) ) assert ( "NFR-9 is marked changed, but the Baseline has no such requirement" in (requirements_check(invented)[0]) ) def test_a_change_request_must_change_something(): assert ( "nothing is marked added, changed or removed" in (requirements_check(revised())[0]) ) def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream(): from app.workflows.chain import read_list call = revised( **{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}} ) upstream = {"requirements": {"attachments": call.args["attachments"]}} assert requirements_check(call) == () live = read_list(upstream, "requirements", "requirements.json") assert [item["id"] for item in live] == ["FR-1"] assert '"NFR-1"' in call.args["attachments"]["requirements.json"] # --- step 4: later Specialists revise from the Baseline --------------------------------- async def test_each_specialist_is_given_its_delivered_version_and_what_changed( tmp_path, ): await delivered(tmp_path) runtime = FakeAgentRuntime(changed()) engine = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), ) await engine.start( "Availability goes up.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) by_role = {run.role: run for run in runtime.runs} analyst = by_role["requirements-analyst"].task.context["baseline"] assert analyst["execution_id"] == "v1" assert analyst["original_request"] == "Ingest webhooks" assert "requirements.json" in analyst["deliverable"]["attachments"] assert analyst["changes"] is None # nothing has changed yet architect = by_role["target-architect"].task.context assert architect["baseline"]["changes"]["requirements"] == [ { "id": "NFR-1", "change": "changed", "reason": "The service is now customer-facing.", "text": "99.99% availability.", } ] # Its working copy already holds the delivered graph. assert architect["baseline"]["deliverable"]["attachments"][ "target-architecture.json" ] assert "graph_read" in by_role["target-architect"].tools # The guide is assembled afresh: the Baseline's is not handed back. assert by_role["packager"].task.context["baseline"]["deliverable"] is None async def test_a_description_whose_design_did_not_change_shape_is_carried_over( tmp_path, ): await delivered(tmp_path) runtime = FakeAgentRuntime(changed()) engine = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), ) result = await engine.start( "Availability goes up.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) # The same Elements and Relationships as the Baseline: the describer is not run. assert "describer" not in {run.role for run in runtime.runs} assert result.state["deliverables"]["architecture_description"]["content"] async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path): await delivered(tmp_path) seen: dict = {} def author(task): seen["working_copy"] = task.context["working_copy"] return [sample_submission(K.DEPLOYMENT_GUIDANCE)] engine = engine_for(tmp_path, changed(**{"iac-author": [author]})) await engine.start( "Availability goes up.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) # It is told nothing of a working copy on a first attempt; the tools hold it. assert seen["working_copy"] is None # --- what changed, worked out from the ids ------------------------------------------------ def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]: """A Baseline and a revised set of Deliverables, as data files.""" def deliverables() -> dict: return { kind: {"attachments": {name: SAMPLE_DATA[name]}} for kind, name in ( ("requirements", "requirements.json"), ("technical_decomposition", "components.json"), ("design_recommendations", "recommendations.json"), ("target_architecture", "target-architecture.json"), ) } before, after = deliverables(), deliverables() change = requirement_change or RETENTION after["requirements"]["attachments"] = revised( **json.loads(json.dumps(change)) ).args["attachments"] if graph_change: graph = json.loads(SAMPLE_DATA["target-architecture.json"]) graph_change(graph) after["target_architecture"]["attachments"] = { "target-architecture.json": json.dumps(graph) } return before, after def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested(): from app.workflows.changes import change_summary def tighten(graph): graph["checks"][0]["statement"] = "The service runs at least three instances." summary = change_summary(*versions(tighten)) assert [item["id"] for item in summary["requirements"]] == ["NFR-1"] assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []} assert summary["elements"] == {"added": [], "changed": [], "removed": []} assert summary["unrequested"] == {} def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested(): from app.workflows.changes import change_summary def wander(graph): # K-2 comes from DR-1, which did not change, on E-2, which realises nothing # the changed requirement touches. NFR-1 reaches K-1 and E-1 only. graph["checks"][1]["statement"] = "The identity has no keys." change = { "NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."} } before, after = versions(wander, change) # NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2. components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}] for version in (before, after): version["technical_decomposition"]["attachments"] = { "components.json": json.dumps(components) } summary = change_summary(before, after) assert summary["checks"]["changed"] == ["K-2"] assert summary["unrequested"] == {"checks": ["K-2"]} def test_added_and_removed_items_are_found_by_id(): from app.workflows.changes import change_summary def grow(graph): graph["elements"].append( { "id": "E-3", "name": "Second region service", "component": "C-1", "product": "Example Run", "resource_type": "example_run_service", } ) graph["checks"] = [graph["checks"][0]] + [ {"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"} ] summary = change_summary(*versions(grow)) assert summary["elements"]["added"] == ["E-3"] assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]} # K-2 came from DR-1 on E-2: nothing that changed reaches it. assert summary["unrequested"] == {"checks": ["K-2"]} # --- step 5: Approvals show the change -------------------------------------------------- async def test_an_approval_in_a_change_request_shows_what_that_phase_changes( tmp_path, ): from app.presentation import describe_prompt await delivered(tmp_path) engine = engine_for(tmp_path, changed()) first = await engine.start( "Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1" ) approval = first.prompt["approval"] assert approval["phase"] == "requirements_discovery" assert approval["baseline"] == "v1" assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"] text = describe_prompt(first.prompt) assert "This changes the architecture delivered in v1." in text assert ( "- Requirement NFR-1 changed: 99.99% availability. (The service is now " "customer-facing.)" ) in text second = await engine.resume( "v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}} ) # The design did not change, and the Approval says so. design = second.prompt["approval"] assert design["phase"] == "solution_architecture" and design["changes"] == {} assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt) async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path): first = await engine_for(tmp_path, once()).start("Ingest webhooks") assert "changes" not in first.prompt["approval"] # --- step 6: the change summary and the line of versions ---------------------------------- async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path): from app.workflows.changes import change_summary await delivered(tmp_path) result = await engine_for(tmp_path, changed()).start( "Availability goes up.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) case = tmp_path / "deliverables" / "cases" / CASE / "executions" written = json.loads((case / "v2" / "changes.json").read_text()) assert [item["id"] for item in written["requirements"]] == ["NFR-1"] assert written["unrequested"] == {} # It matches a comparison of the two Executions' data files. assert written == change_summary( load_baseline(tmp_path, CASE, "v1")["deliverables"], result.state["deliverables"], ) # The Change request stands on its own: every Deliverable is written in full. manifest = json.loads((case / "v2" / "execution.json").read_text()) assert manifest["changes"] == "changes.json" assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH} assert (case / "v2" / "solution-architecture-guide.md").is_file() assert not (case / "v1" / "changes.json").exists() # --- only the pillars a change touches are consulted again ------------------------------ async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path): """The design advisor is told which pillars the change affects, and has the Baseline's advice for the rest.""" from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer from app.tools.pillars import ADVICE_KEY await delivered(tmp_path) # A Baseline whose design advisor consulted two pillars. folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1" advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."} (folder / "working.json").write_text( json.dumps({"design_recommendations": {ADVICE_KEY: advice}}) ) seen: dict = {} def advisor(task): seen["pillars_to_consult"] = task.context["pillars_to_consult"] return [sample_submission(K.DESIGN_RECOMMENDATIONS)] decider = FakeDecider( {"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)} ) runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]})) engine = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), decider=decider, ) result = await engine.start( "Availability goes up.", customer_case=CASE, pre_approved=True, execution_id="v2", baseline="v1", ) assert seen["pillars_to_consult"] == ["reliability"] state, _, _ = decider.asked[0] assert "Requirement NFR-1 changed: 99.99% availability." in state # The advice is carried into the new Execution, and written for the next change. assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice assert json.loads((folder.parent / "v2" / "working.json").read_text()) == { "design_recommendations": {ADVICE_KEY: advice} } async def test_a_first_consultation_is_given_no_hint(tmp_path): seen: dict = {} def advisor(task): seen["pillars_to_consult"] = task.context["pillars_to_consult"] return [sample_submission(K.DESIGN_RECOMMENDATIONS)] from app.adk.fake_runtime import FakeDecider decider = FakeDecider({}) turns = once() turns["design-advisor"] = [advisor] engine = ExecutionEngine( runtime=FakeAgentRuntime(turns), roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), decider=decider, ) await engine.start("Ingest webhooks", pre_approved=True) assert seen["pillars_to_consult"] is None and decider.asked == [] # --- changed, or only reworded? --------------------------------------------------------- async def test_an_item_that_was_only_reworded_is_listed_apart_from_real_changes(): from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer from app.tools.meaning import settle_meaning from app.workflows.changes import change_summary def revise(graph): graph["checks"][0]["statement"] = "The service runs at least three instances." graph["checks"][1]["statement"] = "This service alone uses the identity." before, after = versions(revise) summary = change_summary(before, after) assert summary["checks"]["changed"] == ["K-1", "K-2"] decider = FakeDecider({"K-1": Answer(0.95, 0.9), "K-2": Answer(0.1, 0.8)}) settled = await settle_meaning(decider, summary, before, after, scope="v2") assert settled["checks"]["changed"] == ["K-1"] assert settled["checks"]["reworded"] == ["K-2"] assert settled["meaning"] == {"K-1": 0.95, "K-2": 0.1} # K-2 was outside the reach of what changed; reworded, it is no longer unrequested. assert settled["unrequested"] == {} _, questions, scope = decider.asked[0] assert scope == "v2" assert "The identity is used by this service only." in questions["K-2"].instructions assert "This service alone uses the identity." in questions["K-2"].instructions # The summary given in is not altered. assert summary["checks"]["changed"] == ["K-1", "K-2"] async def test_without_a_decision_everything_that_differs_stays_changed(): from app.tools.meaning import settle_meaning from app.workflows.changes import change_summary def revise(graph): graph["checks"][1]["statement"] = "This service alone uses the identity." before, after = versions(revise) summary = change_summary(before, after) settled = await settle_meaning(NoDecider(), summary, before, after) assert settled["checks"]["changed"] == ["K-2"] assert "reworded" not in settled["checks"] and "meaning" not in settled