Files
orchestration_engine/tests/toy/test_execution.py

417 lines
14 KiB
Python

"""Whole Executions of the toy Workflow: the engine on a subject it was not built from."""
import json
import pytest
from execution_engine.adk.fake_runtime import Call
from tests.toy.support import (
approve,
fails,
folder,
make_engine,
manifest,
once_each,
submits,
)
from tests.toy.workflow import (
CHECK,
CHECKING,
MENU,
MENU_FILE,
PACKING,
PLAN,
PLANNING,
SHOPPING_FILE,
SHOPPING_LIST,
)
THREE_DISHES = json.dumps(
[
{"id": "D-1", "name": "Sandwiches"},
{"id": "D-2", "name": "Lemonade"},
{"id": "D-3", "name": "Cake"},
]
)
FOR_THREE = json.dumps(
[
{"dish": "D-1", "item": "bread", "quantity": 2},
{"dish": "D-2", "item": "lemons", "quantity": 6},
{"dish": "D-3", "item": "flour", "quantity": 1},
]
)
THREE_VERDICTS = json.dumps(
{
"dry_run": {"verdict": "pass", "evidence": "Fits the basket."},
"checks": [
{"check": dish, "verdict": "pass", "evidence": "Enough."}
for dish in ("D-1", "D-2", "D-3")
],
"proposed_additions": [],
}
)
def actions(result) -> list[str]:
return [decision["action"] for decision in result.state["decisions"]]
async def test_a_pre_approved_execution_runs_to_the_end_and_writes_its_files(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
result = await engine.start(
"A picnic for six", pre_approved=True, customer_case="park", execution_id="e1"
)
assert result.status == "completed"
assert [run.role for run in runtime.runs] == [
"menu-writer",
"shopper",
"checker",
"packer",
]
out = folder(tmp_path, "park", "e1")
assert {path.name for path in out.iterdir()} == {
"deliverables",
"execution.json",
"menu.json",
"shopping.json",
"verdicts.json",
"picnic-plan.md",
}
record = manifest(tmp_path, "park", "e1")
assert record["final_document"] == "picnic-plan.md"
assert record["phases"] == {
PLANNING: "approved",
CHECKING: "approved",
PACKING: "approved",
}
# The final Deliverable is also the file at the top of the folder.
assert (out / "picnic-plan.md").read_text() == (
out / "deliverables" / "plan.md"
).read_text()
# What the Workflow records of its own, and nothing of any other subject.
assert record["dishes"] == ["D-1", "D-2"]
assert not {"solution_guide", "terraform_modules", "delivery_plan"} & set(record)
async def test_no_spec_is_needed_and_none_is_recorded(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
result = await engine.start("A picnic", pre_approved=True, execution_id="e1")
assert result.state["scope"] is None
assert manifest(tmp_path, "unassigned", "e1")["scope"] is None
assert all(run.task.context["scope"] is None for run in runtime.runs)
async def test_code_adds_its_view_to_a_submission(tmp_path):
engine, _ = make_engine(tmp_path, once_each())
result = await engine.start("A picnic", pre_approved=True)
shopping = result.state["deliverables"][SHOPPING_LIST]["content"]
assert shopping.rstrip().endswith("## Totals\n\n3 items for 2 dishes.")
async def test_each_specialist_is_given_what_it_works_from(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
await engine.start("A picnic", pre_approved=True)
by_role = {run.role: run.task.context for run in runtime.runs}
assert by_role["menu-writer"]["dependencies"] == {}
assert set(by_role["shopper"]["dependencies"]) == {MENU}
assert by_role["shopper"]["contract"] == {
"sections": ["What to buy"],
"attachments": [SHOPPING_FILE],
}
# The check works from the list; the menu arrives as its data file alone.
assert set(by_role["checker"]["dependencies"]) == {MENU, SHOPPING_LIST}
assert by_role["checker"]["dependencies"][MENU] == {
"attachments": {MENU_FILE: runtime.runs[0].outcome.attachments[MENU_FILE]}
}
# A section code wrote for a reader is not shown to the next Specialist, which
# has the same facts as data. The final Deliverable assembles it, so it is.
assert (
"## Totals" not in by_role["checker"]["dependencies"][SHOPPING_LIST]["content"]
)
assert set(by_role["packer"]["dependencies"]) == {MENU, SHOPPING_LIST, CHECK}
assert "## Totals" in by_role["packer"]["dependencies"][SHOPPING_LIST]["content"]
async def test_a_submission_that_breaks_the_chain_is_refused(tmp_path):
turns = once_each()
nothing_for_lemonade = json.dumps([{"dish": "D-1", "item": "bread"}])
turns["shopper"] = [
[
*submits(SHOPPING_LIST, **{SHOPPING_FILE: nothing_for_lemonade}),
*submits(SHOPPING_LIST),
]
]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
refused, accepted = runtime.runs[1].results
assert "nothing is bought for dish D-2" in refused[1]
assert accepted[1] == "Submitted shopping_list."
async def test_with_a_user_each_phase_waits_for_its_approval(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
first = await engine.start("A picnic", execution_id="e1")
assert first.status == "awaiting_user"
assert first.prompt["approval"]["phase"] == PLANNING
assert first.prompt["approval"]["deliverables"] == [MENU, SHOPPING_LIST]
# The summary is assembled from the Workflow's own facts.
assert "2 dishes on the menu." in first.prompt["approval"]["summary"]
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper"]
second = await engine.resume("e1", approve(PLANNING))
assert second.prompt["approval"]["phase"] == CHECKING
assert (
"Validation: 2 Checks passed, 0 failed" in second.prompt["approval"]["summary"]
)
third = await engine.resume("e1", approve(CHECKING))
assert third.prompt["approval"]["deliverables"] == [PLAN]
done = await engine.resume("e1", approve(PACKING))
assert done.status == "completed"
async def test_a_specialists_question_reaches_the_user_and_the_answer_comes_back(
tmp_path,
):
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
lambda task: (
submits(MENU) if task.context["answers"][-1]["reply"] == "None" else []
),
]
engine, _ = make_engine(tmp_path, turns)
asked = await engine.start("A picnic", execution_id="e1")
assert asked.prompt == {"questions": [{"from": MENU, "question": "Any allergies?"}]}
answered = await engine.resume("e1", {"reply": "None"})
assert answered.prompt["approval"]["phase"] == PLANNING
async def test_a_failed_check_sends_the_list_back_and_the_menu_stands(tmp_path):
turns = once_each()
turns["shopper"] = [
submits(SHOPPING_LIST),
lambda task: submits(SHOPPING_LIST) if task.context["feedback"] else [],
]
turns["checker"] = [fails(D_2="fail"), submits(CHECK)]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
# The Workflow says what a failed Check reopens: the list, not the menu.
assert [run.role for run in runtime.runs] == [
"menu-writer",
"shopper",
"checker",
"shopper",
"checker",
"packer",
]
assert "revise_code" in actions(result)
assert runtime.runs[3].task.context["feedback"] == ["D-2 failed: D-2 is short."]
ledger = result.state["ledger"]["deliverables"]
assert ledger[SHOPPING_LIST]["revisions"] == 1
assert ledger[MENU]["revisions"] == 0
async def test_a_check_that_fails_twice_running_sends_the_menu_back(tmp_path):
turns = once_each()
turns["menu-writer"] = [submits(MENU), submits(MENU)]
turns["shopper"] = [submits(SHOPPING_LIST)] * 3
turns["checker"] = [fails(D_2="fail"), fails(D_2="fail"), submits(CHECK)]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
assert "revise_target_architecture" in actions(result)
assert result.state["ledger"]["deliverables"][MENU]["revisions"] == 1
assert [run.role for run in runtime.runs].count("menu-writer") == 2
async def test_a_user_who_asks_for_changes_at_an_approval_gets_a_revision(tmp_path):
turns = once_each()
turns["shopper"] = [submits(SHOPPING_LIST), submits(SHOPPING_LIST)]
engine, runtime = make_engine(tmp_path, turns)
await engine.start("A picnic", execution_id="e1")
again = await engine.resume(
"e1",
{
"approval": {
"phase": PLANNING,
"decision": "changes",
"feedback": "More lemons on the shopping list, please.",
}
},
)
# Named outright, so it is revised with no decision model to say which.
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper", "shopper"]
assert "More lemons" in runtime.runs[2].task.context["feedback"][0]
assert again.prompt["approval"]["phase"] == PLANNING
async def test_a_change_request_starts_from_its_baseline_and_says_what_changed(
tmp_path,
):
engine, _ = make_engine(tmp_path, once_each())
await engine.start(
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
)
turns = {
"menu-writer": [
lambda task: (
submits(MENU, **{MENU_FILE: THREE_DISHES})
if task.context["baseline"]["deliverable"]
else []
)
],
"shopper": [submits(SHOPPING_LIST, **{SHOPPING_FILE: FOR_THREE})],
"checker": [submits(CHECK, **{"verdicts.json": THREE_VERDICTS})],
# The final Deliverable is written afresh; the Baseline's is not handed back.
"packer": [
lambda task: (
submits(PLAN) if task.context["baseline"]["deliverable"] is None else []
)
],
}
changed, runtime = make_engine(tmp_path, turns)
result = await changed.start(
"Add a cake.",
pre_approved=True,
customer_case="park",
execution_id="second",
baseline="first",
)
assert result.status == "completed"
out = folder(tmp_path, "park", "second")
assert json.loads((out / "changes.json").read_text()) == {
"added": ["D-3"],
"removed": [],
}
assert manifest(tmp_path, "park", "second")["baseline"] == "first"
# The shopper is told what has changed upstream of its Deliverable, in data.
assert runtime.runs[1].task.context["baseline"]["changes"] == {
"added": ["D-3"],
"removed": [],
}
# The Baseline is never modified.
assert manifest(tmp_path, "park", "first")["dishes"] == ["D-1", "D-2"]
async def test_a_baseline_from_another_customer_case_is_refused(tmp_path):
engine, _ = make_engine(tmp_path, once_each())
await engine.start(
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
)
other, _ = make_engine(tmp_path, {})
with pytest.raises(ValueError, match="another Customer case"):
await other.start(
"Add a cake.", pre_approved=True, customer_case="beach", baseline="first"
)
async def test_an_option_takes_its_default_the_callers_value_or_its_baselines(
tmp_path,
):
engine_turns = once_each()
for role in engine_turns:
engine_turns[role] = engine_turns[role] * 3
engine, runtime = make_engine(tmp_path, engine_turns)
default = await engine.start("A picnic", pre_approved=True, customer_case="park")
chosen = await engine.start(
"A picnic",
pre_approved=True,
customer_case="park",
execution_id="six",
guests=6,
)
followed = await engine.start(
"The same again.", pre_approved=True, customer_case="park", baseline="six"
)
assert default.state["options"] == {"guests": 4}
assert chosen.state["options"] == {"guests": 6}
assert followed.state["options"] == {"guests": 6}
assert manifest(tmp_path, "park", "six")["options"] == {"guests": 6}
# Every Specialist is told, through the Workflow's own entry in its task.
assert {run.task.context["guests"] for run in runtime.runs[4:8]} == {6}
@pytest.mark.parametrize(
("options", "message"),
[
({"guests": 0}, "guests must be a whole number"),
({"guests": "many"}, "guests must be a whole number"),
({"terraform_modules": True}, "Unknown options: terraform_modules"),
],
)
async def test_an_option_the_workflow_does_not_allow_refuses_the_start(
tmp_path, options, message
):
engine, runtime = make_engine(tmp_path, once_each())
with pytest.raises(ValueError, match=message):
await engine.start("A picnic", pre_approved=True, **options)
assert runtime.runs == []
async def test_with_nobody_to_answer_a_specialist_that_cannot_finish_is_stopped(
tmp_path,
):
turns = once_each()
asks = [Call("needs_input", {"questions": ["Which park?"]})]
turns["menu-writer"] = [asks] * 10
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed_with_warnings"
assert [run.role for run in runtime.runs] == ["menu-writer"] * 3
assert result.state["warnings"] == [
(
"Stopped: menu-writer ended 3 runs running without submitting menu, with "
"nobody to answer it."
)
]
async def test_a_specialist_that_finishes_on_its_last_chance_is_not_stopped(tmp_path):
turns = once_each()
asks = [Call("needs_input", {"questions": ["Which park?"]})]
turns["menu-writer"] = [asks, asks, submits(MENU)]
engine, _runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
assert result.state["unfinished_runs"] == {}