feat: working orhcestration engine
This commit is contained in:
416
tests/toy/test_execution.py
Normal file
416
tests/toy/test_execution.py
Normal file
@@ -0,0 +1,416 @@
|
||||
"""Whole Executions of the toy Workflow: the engine on a subject it was not built from."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call
|
||||
from tests.toy.support import (
|
||||
approve,
|
||||
fails,
|
||||
folder,
|
||||
make_engine,
|
||||
manifest,
|
||||
once_each,
|
||||
submits,
|
||||
)
|
||||
from tests.toy.workflow import (
|
||||
CHECK,
|
||||
CHECKING,
|
||||
MENU,
|
||||
MENU_FILE,
|
||||
PACKING,
|
||||
PLAN,
|
||||
PLANNING,
|
||||
SHOPPING_FILE,
|
||||
SHOPPING_LIST,
|
||||
)
|
||||
|
||||
THREE_DISHES = json.dumps(
|
||||
[
|
||||
{"id": "D-1", "name": "Sandwiches"},
|
||||
{"id": "D-2", "name": "Lemonade"},
|
||||
{"id": "D-3", "name": "Cake"},
|
||||
]
|
||||
)
|
||||
FOR_THREE = json.dumps(
|
||||
[
|
||||
{"dish": "D-1", "item": "bread", "quantity": 2},
|
||||
{"dish": "D-2", "item": "lemons", "quantity": 6},
|
||||
{"dish": "D-3", "item": "flour", "quantity": 1},
|
||||
]
|
||||
)
|
||||
THREE_VERDICTS = json.dumps(
|
||||
{
|
||||
"dry_run": {"verdict": "pass", "evidence": "Fits the basket."},
|
||||
"checks": [
|
||||
{"check": dish, "verdict": "pass", "evidence": "Enough."}
|
||||
for dish in ("D-1", "D-2", "D-3")
|
||||
],
|
||||
"proposed_additions": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def actions(result) -> list[str]:
|
||||
return [decision["action"] for decision in result.state["decisions"]]
|
||||
|
||||
|
||||
async def test_a_pre_approved_execution_runs_to_the_end_and_writes_its_files(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start(
|
||||
"A picnic for six", pre_approved=True, customer_case="park", execution_id="e1"
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"menu-writer",
|
||||
"shopper",
|
||||
"checker",
|
||||
"packer",
|
||||
]
|
||||
out = folder(tmp_path, "park", "e1")
|
||||
assert {path.name for path in out.iterdir()} == {
|
||||
"deliverables",
|
||||
"execution.json",
|
||||
"menu.json",
|
||||
"shopping.json",
|
||||
"verdicts.json",
|
||||
"picnic-plan.md",
|
||||
}
|
||||
record = manifest(tmp_path, "park", "e1")
|
||||
assert record["final_document"] == "picnic-plan.md"
|
||||
assert record["phases"] == {
|
||||
PLANNING: "approved",
|
||||
CHECKING: "approved",
|
||||
PACKING: "approved",
|
||||
}
|
||||
# The final Deliverable is also the file at the top of the folder.
|
||||
assert (out / "picnic-plan.md").read_text() == (
|
||||
out / "deliverables" / "plan.md"
|
||||
).read_text()
|
||||
# What the Workflow records of its own, and nothing of any other subject.
|
||||
assert record["dishes"] == ["D-1", "D-2"]
|
||||
assert not {"solution_guide", "terraform_modules", "delivery_plan"} & set(record)
|
||||
|
||||
|
||||
async def test_no_spec_is_needed_and_none_is_recorded(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True, execution_id="e1")
|
||||
|
||||
assert result.state["scope"] is None
|
||||
assert manifest(tmp_path, "unassigned", "e1")["scope"] is None
|
||||
assert all(run.task.context["scope"] is None for run in runtime.runs)
|
||||
|
||||
|
||||
async def test_code_adds_its_view_to_a_submission(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
shopping = result.state["deliverables"][SHOPPING_LIST]["content"]
|
||||
assert shopping.rstrip().endswith("## Totals\n\n3 items for 2 dishes.")
|
||||
|
||||
|
||||
async def test_each_specialist_is_given_what_it_works_from(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
by_role = {run.role: run.task.context for run in runtime.runs}
|
||||
assert by_role["menu-writer"]["dependencies"] == {}
|
||||
assert set(by_role["shopper"]["dependencies"]) == {MENU}
|
||||
assert by_role["shopper"]["contract"] == {
|
||||
"sections": ["What to buy"],
|
||||
"attachments": [SHOPPING_FILE],
|
||||
}
|
||||
# The check works from the list; the menu arrives as its data file alone.
|
||||
assert set(by_role["checker"]["dependencies"]) == {MENU, SHOPPING_LIST}
|
||||
assert by_role["checker"]["dependencies"][MENU] == {
|
||||
"attachments": {MENU_FILE: runtime.runs[0].outcome.attachments[MENU_FILE]}
|
||||
}
|
||||
# A section code wrote for a reader is not shown to the next Specialist, which
|
||||
# has the same facts as data. The final Deliverable assembles it, so it is.
|
||||
assert (
|
||||
"## Totals" not in by_role["checker"]["dependencies"][SHOPPING_LIST]["content"]
|
||||
)
|
||||
assert set(by_role["packer"]["dependencies"]) == {MENU, SHOPPING_LIST, CHECK}
|
||||
assert "## Totals" in by_role["packer"]["dependencies"][SHOPPING_LIST]["content"]
|
||||
|
||||
|
||||
async def test_a_submission_that_breaks_the_chain_is_refused(tmp_path):
|
||||
turns = once_each()
|
||||
nothing_for_lemonade = json.dumps([{"dish": "D-1", "item": "bread"}])
|
||||
turns["shopper"] = [
|
||||
[
|
||||
*submits(SHOPPING_LIST, **{SHOPPING_FILE: nothing_for_lemonade}),
|
||||
*submits(SHOPPING_LIST),
|
||||
]
|
||||
]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
refused, accepted = runtime.runs[1].results
|
||||
assert "nothing is bought for dish D-2" in refused[1]
|
||||
assert accepted[1] == "Submitted shopping_list."
|
||||
|
||||
|
||||
async def test_with_a_user_each_phase_waits_for_its_approval(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
first = await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
assert first.status == "awaiting_user"
|
||||
assert first.prompt["approval"]["phase"] == PLANNING
|
||||
assert first.prompt["approval"]["deliverables"] == [MENU, SHOPPING_LIST]
|
||||
# The summary is assembled from the Workflow's own facts.
|
||||
assert "2 dishes on the menu." in first.prompt["approval"]["summary"]
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper"]
|
||||
|
||||
second = await engine.resume("e1", approve(PLANNING))
|
||||
|
||||
assert second.prompt["approval"]["phase"] == CHECKING
|
||||
assert (
|
||||
"Validation: 2 Checks passed, 0 failed" in second.prompt["approval"]["summary"]
|
||||
)
|
||||
|
||||
third = await engine.resume("e1", approve(CHECKING))
|
||||
|
||||
assert third.prompt["approval"]["deliverables"] == [PLAN]
|
||||
|
||||
done = await engine.resume("e1", approve(PACKING))
|
||||
|
||||
assert done.status == "completed"
|
||||
|
||||
|
||||
async def test_a_specialists_question_reaches_the_user_and_the_answer_comes_back(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [
|
||||
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
||||
lambda task: (
|
||||
submits(MENU) if task.context["answers"][-1]["reply"] == "None" else []
|
||||
),
|
||||
]
|
||||
engine, _ = make_engine(tmp_path, turns)
|
||||
|
||||
asked = await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
assert asked.prompt == {"questions": [{"from": MENU, "question": "Any allergies?"}]}
|
||||
|
||||
answered = await engine.resume("e1", {"reply": "None"})
|
||||
|
||||
assert answered.prompt["approval"]["phase"] == PLANNING
|
||||
|
||||
|
||||
async def test_a_failed_check_sends_the_list_back_and_the_menu_stands(tmp_path):
|
||||
turns = once_each()
|
||||
turns["shopper"] = [
|
||||
submits(SHOPPING_LIST),
|
||||
lambda task: submits(SHOPPING_LIST) if task.context["feedback"] else [],
|
||||
]
|
||||
turns["checker"] = [fails(D_2="fail"), submits(CHECK)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
# The Workflow says what a failed Check reopens: the list, not the menu.
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"menu-writer",
|
||||
"shopper",
|
||||
"checker",
|
||||
"shopper",
|
||||
"checker",
|
||||
"packer",
|
||||
]
|
||||
assert "revise_code" in actions(result)
|
||||
assert runtime.runs[3].task.context["feedback"] == ["D-2 failed: D-2 is short."]
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert ledger[SHOPPING_LIST]["revisions"] == 1
|
||||
assert ledger[MENU]["revisions"] == 0
|
||||
|
||||
|
||||
async def test_a_check_that_fails_twice_running_sends_the_menu_back(tmp_path):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [submits(MENU), submits(MENU)]
|
||||
turns["shopper"] = [submits(SHOPPING_LIST)] * 3
|
||||
turns["checker"] = [fails(D_2="fail"), fails(D_2="fail"), submits(CHECK)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "revise_target_architecture" in actions(result)
|
||||
assert result.state["ledger"]["deliverables"][MENU]["revisions"] == 1
|
||||
assert [run.role for run in runtime.runs].count("menu-writer") == 2
|
||||
|
||||
|
||||
async def test_a_user_who_asks_for_changes_at_an_approval_gets_a_revision(tmp_path):
|
||||
turns = once_each()
|
||||
turns["shopper"] = [submits(SHOPPING_LIST), submits(SHOPPING_LIST)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
again = await engine.resume(
|
||||
"e1",
|
||||
{
|
||||
"approval": {
|
||||
"phase": PLANNING,
|
||||
"decision": "changes",
|
||||
"feedback": "More lemons on the shopping list, please.",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
# Named outright, so it is revised with no decision model to say which.
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper", "shopper"]
|
||||
assert "More lemons" in runtime.runs[2].task.context["feedback"][0]
|
||||
assert again.prompt["approval"]["phase"] == PLANNING
|
||||
|
||||
|
||||
async def test_a_change_request_starts_from_its_baseline_and_says_what_changed(
|
||||
tmp_path,
|
||||
):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
await engine.start(
|
||||
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
|
||||
)
|
||||
turns = {
|
||||
"menu-writer": [
|
||||
lambda task: (
|
||||
submits(MENU, **{MENU_FILE: THREE_DISHES})
|
||||
if task.context["baseline"]["deliverable"]
|
||||
else []
|
||||
)
|
||||
],
|
||||
"shopper": [submits(SHOPPING_LIST, **{SHOPPING_FILE: FOR_THREE})],
|
||||
"checker": [submits(CHECK, **{"verdicts.json": THREE_VERDICTS})],
|
||||
# The final Deliverable is written afresh; the Baseline's is not handed back.
|
||||
"packer": [
|
||||
lambda task: (
|
||||
submits(PLAN) if task.context["baseline"]["deliverable"] is None else []
|
||||
)
|
||||
],
|
||||
}
|
||||
changed, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await changed.start(
|
||||
"Add a cake.",
|
||||
pre_approved=True,
|
||||
customer_case="park",
|
||||
execution_id="second",
|
||||
baseline="first",
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
out = folder(tmp_path, "park", "second")
|
||||
assert json.loads((out / "changes.json").read_text()) == {
|
||||
"added": ["D-3"],
|
||||
"removed": [],
|
||||
}
|
||||
assert manifest(tmp_path, "park", "second")["baseline"] == "first"
|
||||
# The shopper is told what has changed upstream of its Deliverable, in data.
|
||||
assert runtime.runs[1].task.context["baseline"]["changes"] == {
|
||||
"added": ["D-3"],
|
||||
"removed": [],
|
||||
}
|
||||
# The Baseline is never modified.
|
||||
assert manifest(tmp_path, "park", "first")["dishes"] == ["D-1", "D-2"]
|
||||
|
||||
|
||||
async def test_a_baseline_from_another_customer_case_is_refused(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
await engine.start(
|
||||
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
|
||||
)
|
||||
other, _ = make_engine(tmp_path, {})
|
||||
|
||||
with pytest.raises(ValueError, match="another Customer case"):
|
||||
await other.start(
|
||||
"Add a cake.", pre_approved=True, customer_case="beach", baseline="first"
|
||||
)
|
||||
|
||||
|
||||
async def test_an_option_takes_its_default_the_callers_value_or_its_baselines(
|
||||
tmp_path,
|
||||
):
|
||||
engine_turns = once_each()
|
||||
for role in engine_turns:
|
||||
engine_turns[role] = engine_turns[role] * 3
|
||||
engine, runtime = make_engine(tmp_path, engine_turns)
|
||||
|
||||
default = await engine.start("A picnic", pre_approved=True, customer_case="park")
|
||||
chosen = await engine.start(
|
||||
"A picnic",
|
||||
pre_approved=True,
|
||||
customer_case="park",
|
||||
execution_id="six",
|
||||
guests=6,
|
||||
)
|
||||
followed = await engine.start(
|
||||
"The same again.", pre_approved=True, customer_case="park", baseline="six"
|
||||
)
|
||||
|
||||
assert default.state["options"] == {"guests": 4}
|
||||
assert chosen.state["options"] == {"guests": 6}
|
||||
assert followed.state["options"] == {"guests": 6}
|
||||
assert manifest(tmp_path, "park", "six")["options"] == {"guests": 6}
|
||||
# Every Specialist is told, through the Workflow's own entry in its task.
|
||||
assert {run.task.context["guests"] for run in runtime.runs[4:8]} == {6}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("options", "message"),
|
||||
[
|
||||
({"guests": 0}, "guests must be a whole number"),
|
||||
({"guests": "many"}, "guests must be a whole number"),
|
||||
({"terraform_modules": True}, "Unknown options: terraform_modules"),
|
||||
],
|
||||
)
|
||||
async def test_an_option_the_workflow_does_not_allow_refuses_the_start(
|
||||
tmp_path, options, message
|
||||
):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
await engine.start("A picnic", pre_approved=True, **options)
|
||||
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_with_nobody_to_answer_a_specialist_that_cannot_finish_is_stopped(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once_each()
|
||||
asks = [Call("needs_input", {"questions": ["Which park?"]})]
|
||||
turns["menu-writer"] = [asks] * 10
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer"] * 3
|
||||
assert result.state["warnings"] == [
|
||||
(
|
||||
"Stopped: menu-writer ended 3 runs running without submitting menu, with "
|
||||
"nobody to answer it."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_a_specialist_that_finishes_on_its_last_chance_is_not_stopped(tmp_path):
|
||||
turns = once_each()
|
||||
asks = [Call("needs_input", {"questions": ["Which park?"]})]
|
||||
turns["menu-writer"] = [asks, asks, submits(MENU)]
|
||||
engine, _runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.state["unfinished_runs"] == {}
|
||||
Reference in New Issue
Block a user