"""Whole Executions of the toy Workflow: the engine on a subject it was not built from.""" import json import pytest from execution_engine.adk.fake_runtime import Call from tests.toy.support import ( approve, fails, folder, make_engine, manifest, once_each, submits, ) from tests.toy.workflow import ( CHECK, CHECKING, MENU, MENU_FILE, PACKING, PLAN, PLANNING, SHOPPING_FILE, SHOPPING_LIST, ) THREE_DISHES = json.dumps( [ {"id": "D-1", "name": "Sandwiches"}, {"id": "D-2", "name": "Lemonade"}, {"id": "D-3", "name": "Cake"}, ] ) FOR_THREE = json.dumps( [ {"dish": "D-1", "item": "bread", "quantity": 2}, {"dish": "D-2", "item": "lemons", "quantity": 6}, {"dish": "D-3", "item": "flour", "quantity": 1}, ] ) THREE_VERDICTS = json.dumps( { "dry_run": {"verdict": "pass", "evidence": "Fits the basket."}, "checks": [ {"check": dish, "verdict": "pass", "evidence": "Enough."} for dish in ("D-1", "D-2", "D-3") ], "proposed_additions": [], } ) def actions(result) -> list[str]: return [decision["action"] for decision in result.state["decisions"]] async def test_a_pre_approved_execution_runs_to_the_end_and_writes_its_files(tmp_path): engine, runtime = make_engine(tmp_path, once_each()) result = await engine.start( "A picnic for six", pre_approved=True, customer_case="park", execution_id="e1" ) assert result.status == "completed" assert [run.role for run in runtime.runs] == [ "menu-writer", "shopper", "checker", "packer", ] out = folder(tmp_path, "park", "e1") assert {path.name for path in out.iterdir()} == { "deliverables", "execution.json", "menu.json", "shopping.json", "verdicts.json", "picnic-plan.md", } record = manifest(tmp_path, "park", "e1") assert record["final_document"] == "picnic-plan.md" assert record["phases"] == { PLANNING: "approved", CHECKING: "approved", PACKING: "approved", } # The final Deliverable is also the file at the top of the folder. assert (out / "picnic-plan.md").read_text() == ( out / "deliverables" / "plan.md" ).read_text() # What the Workflow records of its own, and nothing of any other subject. assert record["dishes"] == ["D-1", "D-2"] assert not {"solution_guide", "terraform_modules", "delivery_plan"} & set(record) async def test_no_spec_is_needed_and_none_is_recorded(tmp_path): engine, runtime = make_engine(tmp_path, once_each()) result = await engine.start("A picnic", pre_approved=True, execution_id="e1") assert result.state["scope"] is None assert manifest(tmp_path, "unassigned", "e1")["scope"] is None assert all(run.task.context["scope"] is None for run in runtime.runs) async def test_code_adds_its_view_to_a_submission(tmp_path): engine, _ = make_engine(tmp_path, once_each()) result = await engine.start("A picnic", pre_approved=True) shopping = result.state["deliverables"][SHOPPING_LIST]["content"] assert shopping.rstrip().endswith("## Totals\n\n3 items for 2 dishes.") async def test_each_specialist_is_given_what_it_works_from(tmp_path): engine, runtime = make_engine(tmp_path, once_each()) await engine.start("A picnic", pre_approved=True) by_role = {run.role: run.task.context for run in runtime.runs} assert by_role["menu-writer"]["dependencies"] == {} assert set(by_role["shopper"]["dependencies"]) == {MENU} assert by_role["shopper"]["contract"] == { "sections": ["What to buy"], "attachments": [SHOPPING_FILE], } # The check works from the list; the menu arrives as its data file alone. assert set(by_role["checker"]["dependencies"]) == {MENU, SHOPPING_LIST} assert by_role["checker"]["dependencies"][MENU] == { "attachments": {MENU_FILE: runtime.runs[0].outcome.attachments[MENU_FILE]} } # A section code wrote for a reader is not shown to the next Specialist, which # has the same facts as data. The final Deliverable assembles it, so it is. assert ( "## Totals" not in by_role["checker"]["dependencies"][SHOPPING_LIST]["content"] ) assert set(by_role["packer"]["dependencies"]) == {MENU, SHOPPING_LIST, CHECK} assert "## Totals" in by_role["packer"]["dependencies"][SHOPPING_LIST]["content"] async def test_a_submission_that_breaks_the_chain_is_refused(tmp_path): turns = once_each() nothing_for_lemonade = json.dumps([{"dish": "D-1", "item": "bread"}]) turns["shopper"] = [ [ *submits(SHOPPING_LIST, **{SHOPPING_FILE: nothing_for_lemonade}), *submits(SHOPPING_LIST), ] ] engine, runtime = make_engine(tmp_path, turns) result = await engine.start("A picnic", pre_approved=True) assert result.status == "completed" refused, accepted = runtime.runs[1].results assert "nothing is bought for dish D-2" in refused[1] assert accepted[1] == "Submitted shopping_list." async def test_with_a_user_each_phase_waits_for_its_approval(tmp_path): engine, runtime = make_engine(tmp_path, once_each()) first = await engine.start("A picnic", execution_id="e1") assert first.status == "awaiting_user" assert first.prompt["approval"]["phase"] == PLANNING assert first.prompt["approval"]["deliverables"] == [MENU, SHOPPING_LIST] # The summary is assembled from the Workflow's own facts. assert "2 dishes on the menu." in first.prompt["approval"]["summary"] assert [run.role for run in runtime.runs] == ["menu-writer", "shopper"] second = await engine.resume("e1", approve(PLANNING)) assert second.prompt["approval"]["phase"] == CHECKING assert ( "Validation: 2 Checks passed, 0 failed" in second.prompt["approval"]["summary"] ) third = await engine.resume("e1", approve(CHECKING)) assert third.prompt["approval"]["deliverables"] == [PLAN] done = await engine.resume("e1", approve(PACKING)) assert done.status == "completed" async def test_a_specialists_question_reaches_the_user_and_the_answer_comes_back( tmp_path, ): turns = once_each() turns["menu-writer"] = [ [Call("needs_input", {"questions": ["Any allergies?"]})], lambda task: ( submits(MENU) if task.context["answers"][-1]["reply"] == "None" else [] ), ] engine, _ = make_engine(tmp_path, turns) asked = await engine.start("A picnic", execution_id="e1") assert asked.prompt == {"questions": [{"from": MENU, "question": "Any allergies?"}]} answered = await engine.resume("e1", {"reply": "None"}) assert answered.prompt["approval"]["phase"] == PLANNING async def test_a_failed_check_sends_the_list_back_and_the_menu_stands(tmp_path): turns = once_each() turns["shopper"] = [ submits(SHOPPING_LIST), lambda task: submits(SHOPPING_LIST) if task.context["feedback"] else [], ] turns["checker"] = [fails(D_2="fail"), submits(CHECK)] engine, runtime = make_engine(tmp_path, turns) result = await engine.start("A picnic", pre_approved=True) assert result.status == "completed" # The Workflow says what a failed Check reopens: the list, not the menu. assert [run.role for run in runtime.runs] == [ "menu-writer", "shopper", "checker", "shopper", "checker", "packer", ] assert "revise_code" in actions(result) assert runtime.runs[3].task.context["feedback"] == ["D-2 failed: D-2 is short."] ledger = result.state["ledger"]["deliverables"] assert ledger[SHOPPING_LIST]["revisions"] == 1 assert ledger[MENU]["revisions"] == 0 async def test_a_check_that_fails_twice_running_sends_the_menu_back(tmp_path): turns = once_each() turns["menu-writer"] = [submits(MENU), submits(MENU)] turns["shopper"] = [submits(SHOPPING_LIST)] * 3 turns["checker"] = [fails(D_2="fail"), fails(D_2="fail"), submits(CHECK)] engine, runtime = make_engine(tmp_path, turns) result = await engine.start("A picnic", pre_approved=True) assert result.status == "completed" assert "revise_target_architecture" in actions(result) assert result.state["ledger"]["deliverables"][MENU]["revisions"] == 1 assert [run.role for run in runtime.runs].count("menu-writer") == 2 async def test_a_user_who_asks_for_changes_at_an_approval_gets_a_revision(tmp_path): turns = once_each() turns["shopper"] = [submits(SHOPPING_LIST), submits(SHOPPING_LIST)] engine, runtime = make_engine(tmp_path, turns) await engine.start("A picnic", execution_id="e1") again = await engine.resume( "e1", { "approval": { "phase": PLANNING, "decision": "changes", "feedback": "More lemons on the shopping list, please.", } }, ) # Named outright, so it is revised with no decision model to say which. assert [run.role for run in runtime.runs] == ["menu-writer", "shopper", "shopper"] assert "More lemons" in runtime.runs[2].task.context["feedback"][0] assert again.prompt["approval"]["phase"] == PLANNING async def test_a_change_request_starts_from_its_baseline_and_says_what_changed( tmp_path, ): engine, _ = make_engine(tmp_path, once_each()) await engine.start( "A picnic", pre_approved=True, customer_case="park", execution_id="first" ) turns = { "menu-writer": [ lambda task: ( submits(MENU, **{MENU_FILE: THREE_DISHES}) if task.context["baseline"]["deliverable"] else [] ) ], "shopper": [submits(SHOPPING_LIST, **{SHOPPING_FILE: FOR_THREE})], "checker": [submits(CHECK, **{"verdicts.json": THREE_VERDICTS})], # The final Deliverable is written afresh; the Baseline's is not handed back. "packer": [ lambda task: ( submits(PLAN) if task.context["baseline"]["deliverable"] is None else [] ) ], } changed, runtime = make_engine(tmp_path, turns) result = await changed.start( "Add a cake.", pre_approved=True, customer_case="park", execution_id="second", baseline="first", ) assert result.status == "completed" out = folder(tmp_path, "park", "second") assert json.loads((out / "changes.json").read_text()) == { "added": ["D-3"], "removed": [], } assert manifest(tmp_path, "park", "second")["baseline"] == "first" # The shopper is told what has changed upstream of its Deliverable, in data. assert runtime.runs[1].task.context["baseline"]["changes"] == { "added": ["D-3"], "removed": [], } # The Baseline is never modified. assert manifest(tmp_path, "park", "first")["dishes"] == ["D-1", "D-2"] async def test_a_baseline_from_another_customer_case_is_refused(tmp_path): engine, _ = make_engine(tmp_path, once_each()) await engine.start( "A picnic", pre_approved=True, customer_case="park", execution_id="first" ) other, _ = make_engine(tmp_path, {}) with pytest.raises(ValueError, match="another Customer case"): await other.start( "Add a cake.", pre_approved=True, customer_case="beach", baseline="first" ) async def test_an_option_takes_its_default_the_callers_value_or_its_baselines( tmp_path, ): engine_turns = once_each() for role in engine_turns: engine_turns[role] = engine_turns[role] * 3 engine, runtime = make_engine(tmp_path, engine_turns) default = await engine.start("A picnic", pre_approved=True, customer_case="park") chosen = await engine.start( "A picnic", pre_approved=True, customer_case="park", execution_id="six", guests=6, ) followed = await engine.start( "The same again.", pre_approved=True, customer_case="park", baseline="six" ) assert default.state["options"] == {"guests": 4} assert chosen.state["options"] == {"guests": 6} assert followed.state["options"] == {"guests": 6} assert manifest(tmp_path, "park", "six")["options"] == {"guests": 6} # Every Specialist is told, through the Workflow's own entry in its task. assert {run.task.context["guests"] for run in runtime.runs[4:8]} == {6} @pytest.mark.parametrize( ("options", "message"), [ ({"guests": 0}, "guests must be a whole number"), ({"guests": "many"}, "guests must be a whole number"), ({"terraform_modules": True}, "Unknown options: terraform_modules"), ], ) async def test_an_option_the_workflow_does_not_allow_refuses_the_start( tmp_path, options, message ): engine, runtime = make_engine(tmp_path, once_each()) with pytest.raises(ValueError, match=message): await engine.start("A picnic", pre_approved=True, **options) assert runtime.runs == [] async def test_with_nobody_to_answer_a_specialist_that_cannot_finish_is_stopped( tmp_path, ): turns = once_each() asks = [Call("needs_input", {"questions": ["Which park?"]})] turns["menu-writer"] = [asks] * 10 engine, runtime = make_engine(tmp_path, turns) result = await engine.start("A picnic", pre_approved=True) assert result.status == "completed_with_warnings" assert [run.role for run in runtime.runs] == ["menu-writer"] * 3 assert result.state["warnings"] == [ ( "Stopped: menu-writer ended 3 runs running without submitting menu, with " "nobody to answer it." ) ] async def test_a_specialist_that_finishes_on_its_last_chance_is_not_stopped(tmp_path): turns = once_each() asks = [Call("needs_input", {"questions": ["Which park?"]})] turns["menu-writer"] = [asks, asks, submits(MENU)] engine, _runtime = make_engine(tmp_path, turns) result = await engine.start("A picnic", pre_approved=True) assert result.status == "completed" assert result.state["unfinished_runs"] == {}