feat: working orhcestration engine

This commit is contained in:
2026-10-21 19:50:24 +01:00
commit ce5e2f20cd
161 changed files with 31794 additions and 0 deletions

0
tests/toy/__init__.py Normal file
View File

View File

@@ -0,0 +1,7 @@
---
role: checker
deliverable: check
budget: 12
includes: [protocol]
---
You check that every dish can be made from the shopping list.

View File

@@ -0,0 +1,6 @@
---
role: menu-writer
deliverable: menu
includes: [protocol]
---
You write the menu: the dishes, each with an id.

View File

@@ -0,0 +1,4 @@
---
role: orchestrator
---
You coordinate a picnic plan. You produce nothing yourself.

View File

@@ -0,0 +1,6 @@
---
role: packer
deliverable: plan
includes: [protocol]
---
You put the menu, the shopping list and the check together as the picnic plan.

View File

@@ -0,0 +1,3 @@
## How you work
You produce exactly one Deliverable, named in your task. Submit it with `submit_deliverable` when it meets its `contract`, or ask with `needs_input`.

View File

@@ -0,0 +1,4 @@
---
role: reply-writer
---
You write what the user reads, from the facts you are given.

View File

@@ -0,0 +1,7 @@
---
role: shopper
deliverable: shopping_list
tools: [tally]
includes: [protocol]
---
You write the shopping list: what to buy for every dish on the menu.

View File

@@ -0,0 +1,19 @@
# A provider bundle for the engine's own tests. Nothing here is a real provider.
name: toy
skills_dirs:
- skills
models:
default: scripted/none
roles:
checker: scripted/careful
timeout_seconds:
checker: 240
role_skills:
menu-writer: [picnic-planning]
shopper: [picnic-planning]
tools:
general: [tally]

View File

@@ -0,0 +1,10 @@
---
name: house-rules
description: >-
A stand-in overlay Skill for the engine's own tests: one household's rules for a
picnic. Use on any Deliverable of an Execution that names this overlay.
---
# House rules
Nothing here is a rule. It exists so a test can name an overlay Skill.

View File

@@ -0,0 +1,10 @@
---
name: picnic-planning
description: >-
A stand-in Skill for the engine's own tests: how a picnic is planned. Use when
writing the menu or the shopping list of a picnic plan.
---
# Picnic planning
Nothing here is guidance. It exists so the toy Workflow's roles have a Skill.

96
tests/toy/support.py Normal file
View File

@@ -0,0 +1,96 @@
"""What the toy Workflow's tests share: its roles, an engine, and scripted turns."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from execution_engine.adk.runtime import NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.toy.workflow import CHECK, SAMPLES, TOY, VERDICTS_FILE
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in TOY.deliverables.values()
},
}
def submits(kind: str, **files: str) -> list[Call]:
"""One turn that submits a passing Deliverable, with `files` in place of its
sample attachments."""
call = sample_submission(TOY, kind)
return [
Call(
call.tool,
{**call.args, "attachments": {**call.args["attachments"], **files}},
)
]
def verdicts(**by_dish: str) -> str:
"""verdicts.json with the sample's verdicts, and `by_dish` in place of them:
`verdicts(D_2="fail")`."""
data = json.loads(SAMPLES[VERDICTS_FILE])
for check in data["checks"]:
verdict = by_dish.get(check["check"].replace("-", "_"))
if verdict:
check.update(verdict=verdict, evidence=f"{check['check']} is short.")
return json.dumps(data)
def fails(**by_dish: str) -> list[Call]:
return submits(CHECK, **{VERDICTS_FILE: verdicts(**by_dish)})
def once_each() -> dict[str, list[Any]]:
"""Every Specialist submits a passing Deliverable, once."""
return {spec.specialist: [submits(kind)] for kind, spec in TOY.deliverables.items()}
def make_engine(
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
"""An engine on the toy Workflow whose Orchestrator routes by its playbook's
rules: no decision model, and no script for the Orchestrator."""
runtime = FakeAgentRuntime(turns)
options: dict[str, Any] = {
"workflow": TOY,
"runtime": runtime,
"roles": ROLES,
"recorder": InMemoryRecorder(),
"checkpointer": InMemorySaver(),
"output_root": tmp_path,
"orchestrator": Router(NoDecider(), None, Thresholds()),
}
options.update(overrides)
return ExecutionEngine(**options), runtime
def approve(phase: str) -> dict[str, Any]:
return {"approval": {"phase": phase, "decision": "approve"}}
def folder(tmp_path: Path, customer_case: str, execution_id: str) -> Path:
return (
tmp_path
/ "deliverables"
/ "cases"
/ customer_case
/ "executions"
/ execution_id
)
def manifest(tmp_path: Path, customer_case: str, execution_id: str) -> dict[str, Any]:
path = folder(tmp_path, customer_case, execution_id) / "execution.json"
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
return data

118
tests/toy/test_assembly.py Normal file
View File

@@ -0,0 +1,118 @@
"""Assembling an agent from settings, its own folder and a Workflow."""
import dataclasses
from typing import Any
import pytest
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider
from execution_engine.agent import open_assembly
from execution_engine.config import Settings
from execution_engine.recorder import InMemoryRecorder
from tests.toy.support import once_each, submits
from tests.toy.workflow import ROOT, SHOPPING_LIST, TOY
def settings(**overrides: Any) -> Settings:
values: dict[str, Any] = {
"CLOUD_PROVIDER": "toy",
"CHECKPOINTER": "memory",
"MCP_ENABLED": False,
"ORCHESTRATOR_BUDGET": 9,
"SPECIALIST_BUDGET": 4,
}
values.update(overrides)
return Settings(_env_file=None, **values) # type: ignore[call-arg]
def assembled(tmp_path, runtime, workflow=TOY, **overrides):
return open_assembly(
settings(**overrides),
workflow,
root=ROOT,
runtime=runtime,
recorder=InMemoryRecorder(),
output_root=tmp_path,
)
async def test_an_agent_is_its_folder_its_settings_and_its_workflow(tmp_path):
runtime = FakeAgentRuntime(once_each())
async with assembled(tmp_path, runtime) as assembly:
result = await assembly.engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
assert assembly.engine.workflow is TOY
assert assembly.bundle.name == "toy"
assert assembly.roles["orchestrator"].budget == 9
assert assembly.roles["shopper"].budget == 4
assert assembly.mcp_gaps == {}
# With no Jev key the router follows its playbook on rules alone, and no
# Orchestrator model is run.
assert isinstance(assembly.decider, NoDecider)
assert "orchestrator" not in {run.role for run in runtime.runs}
async def test_a_role_is_given_the_workflows_own_tool_on_its_working_copy(tmp_path):
turns = once_each()
turns["shopper"] = [[Call("tally"), *submits(SHOPPING_LIST)]]
runtime = FakeAgentRuntime(turns)
async with assembled(tmp_path, runtime) as assembly:
await assembly.engine.start("A picnic", pre_approved=True)
by_role = {run.role: run for run in runtime.runs}
assert "tally" in by_role["shopper"].tools
assert by_role["shopper"].results[0] == ("tally", "0 items so far.")
# Only the role whose file names it.
assert "tally" not in by_role["checker"].tools
async def test_the_model_orchestrator_is_still_there_to_be_scripted(tmp_path):
runtime = FakeAgentRuntime(
{"orchestrator": [[Call("ask_user", {"questions": ["How many guests?"]})]]}
)
async with assembled(tmp_path, runtime, ORCHESTRATOR="model") as assembly:
result = await assembly.engine.start("A picnic")
assert result.prompt["questions"][0]["question"] == "How many guests?"
async def test_an_unknown_orchestrator_setting_is_refused(tmp_path):
with pytest.raises(ValueError, match="ORCHESTRATOR must be router or model"):
async with assembled(tmp_path, FakeAgentRuntime({}), ORCHESTRATOR="llm"):
pass
async def test_a_workflow_may_be_built_for_the_settings_and_the_bundle(tmp_path):
seen = {}
def build(given_settings, bundle):
seen.update(provider=bundle.name, budget=given_settings.SPECIALIST_BUDGET)
return dataclasses.replace(TOY, done="Packed.")
async with assembled(tmp_path, FakeAgentRuntime({}), build) as assembly:
assert assembly.engine.workflow.done == "Packed."
assert seen == {"provider": "toy", "budget": 4}
async def test_overlay_skills_come_from_the_agents_own_folder(tmp_path):
runtime = FakeAgentRuntime(once_each())
async with assembled(tmp_path, runtime) as assembly:
result = await assembly.engine.start(
"A picnic", pre_approved=True, overlay_skills=["house-rules"]
)
with pytest.raises(ValueError, match="Unknown overlay Skills: no-such-rules"):
await assembly.engine.start(
"A picnic", pre_approved=True, overlay_skills=["no-such-rules"]
)
assert list(result.state["overlay_skills"]) == ["house-rules"]
assert all(
run.task.context["overlay_skills"] == ["house-rules"] for run in runtime.runs
)

169
tests/toy/test_digest.py Normal file
View File

@@ -0,0 +1,169 @@
"""Sources digested into a spec at the start of an Execution."""
import dataclasses
import hashlib
import json
import pytest
import yaml
from execution_engine.scope import ESSENTIAL, Catalogue, FieldSpec
from execution_engine.workflows.workflow import Digested
from tests.toy.support import make_engine, once_each
from tests.toy.workflow import TOY
CATALOGUE = Catalogue(
schema_version=1,
kinds=("picnic",),
fields={
spec.path: spec
for spec in (
FieldSpec("metadata.customer_case", tier=ESSENTIAL),
FieldSpec("engagement.kind", type="enum", values=("picnic",)),
FieldSpec("intent.statement", type="text", tier=ESSENTIAL),
FieldSpec("guests.count", type="integer"),
FieldSpec("guests.allergies", type="list"),
FieldSpec("run.pre_approved", type="boolean"),
)
},
)
INVITATION = json.dumps(
{"party": "park", "wants": "A picnic for the team.", "coming": 9}
)
def digest(sources):
"""Reads `invitation.json`, a format this Workflow knows; nothing else."""
if "invitation.json" not in sources:
return Digested({})
try:
invitation = json.loads(sources["invitation.json"])
except ValueError as exc:
raise ValueError(f"invitation.json: not valid JSON ({exc})") from exc
return Digested(
spec={
"metadata": {"customer_case": invitation["party"]},
"engagement": {"kind": "picnic"},
"intent": {"statement": invitation["wants"]},
"guests": {"count": invitation["coming"]},
"run": {"pre_approved": True},
},
used=("invitation.json",),
notes={"guests.count": "invitation.json, `coming`"},
)
DIGESTING = dataclasses.replace(TOY, digest=digest)
def engine_for(tmp_path, workflow=DIGESTING):
return make_engine(
tmp_path, once_each(), workflow=workflow, scope_catalogue=CATALOGUE
)
def completed(tmp_path) -> dict:
path = tmp_path / "deliverables" / "cases" / "park" / "intent"
return yaml.safe_load((path / "001.scope.completed.yaml").read_text())
async def test_an_execution_starts_from_what_its_sources_say(tmp_path):
engine, _ = engine_for(tmp_path)
result = await engine.start("", sources={"invitation.json": INVITATION})
assert result.status == "completed"
assert result.state["customer_case"] == "park"
assert result.state["workflow_request"] == "A picnic for the team."
assert result.state["scope"]["context"]["stated"]["guests"] == {"count": 9}
async def test_a_digested_value_says_where_it_came_from(tmp_path):
engine, _ = engine_for(tmp_path)
await engine.start("", sources={"invitation.json": INVITATION})
spec = completed(tmp_path)
assert spec["provenance"]["guests.count"] == "digested"
assert spec["provenance"]["guests.allergies"] == "unknown"
assert spec["readiness"]["digested"] == 5
async def test_what_the_caller_writes_is_laid_over_what_was_digested(tmp_path):
engine, _ = engine_for(tmp_path)
result = await engine.start(
"",
sources={"invitation.json": INVITATION},
scope="guests:\n count: 12\n allergies: [nuts]\n",
)
spec = completed(tmp_path)
assert result.state["scope"]["context"]["stated"]["guests"] == {
"count": 12,
"allergies": ["nuts"],
}
assert spec["provenance"]["guests.count"] == "supplied"
assert spec["provenance"]["intent.statement"] == "digested"
async def test_the_sources_are_kept_as_supplied_with_their_hashes(tmp_path):
engine, _ = engine_for(tmp_path)
notes = "Bring a blanket."
result = await engine.start(
"", sources={"invitation.json": INVITATION, "notes/host.md": notes}
)
kept = tmp_path / "deliverables" / "cases" / "park" / "intent" / "001.sources"
assert (kept / "invitation.json").read_text() == INVITATION
assert (kept / "notes" / "host.md").read_text() == notes
assert result.state["sources"] == {
"invitation.json": {
"sha256": hashlib.sha256(INVITATION.encode()).hexdigest(),
"digested": True,
},
"notes/host.md": {
"sha256": hashlib.sha256(notes.encode()).hexdigest(),
"digested": False,
},
}
# What code did not read is a document for a Specialist to read.
assert result.state["source_documents"] == {"notes/host.md": notes}
execution = next(
(tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir()
)
record = json.loads((execution / "execution.json").read_text())
assert record["sources"]["invitation.json"]["digested"] is True
@pytest.mark.parametrize(
("sources", "problem"),
[
({"../invitation.json": INVITATION}, "source names must be relative paths"),
({"invitation.json": "{"}, "invitation.json: not valid JSON"),
({"invitation.json": 7}, "sources must be files by name"),
],
)
async def test_sources_that_cannot_be_read_refuse_the_start(tmp_path, sources, problem):
engine, runtime = engine_for(tmp_path)
with pytest.raises(ValueError, match=problem):
await engine.start("A picnic", sources=sources)
assert runtime.runs == []
assert not (tmp_path / "deliverables").exists()
async def test_a_workflow_that_digests_nothing_passes_every_source_on_to_be_read(
tmp_path,
):
engine, _ = engine_for(tmp_path, TOY)
result = await engine.start(
"A picnic", pre_approved=True, sources={"notes.md": "Bring a blanket."}
)
assert result.status == "completed"
assert result.state["scope"] is None
assert result.state["source_documents"] == {"notes.md": "Bring a blanket."}

416
tests/toy/test_execution.py Normal file
View File

@@ -0,0 +1,416 @@
"""Whole Executions of the toy Workflow: the engine on a subject it was not built from."""
import json
import pytest
from execution_engine.adk.fake_runtime import Call
from tests.toy.support import (
approve,
fails,
folder,
make_engine,
manifest,
once_each,
submits,
)
from tests.toy.workflow import (
CHECK,
CHECKING,
MENU,
MENU_FILE,
PACKING,
PLAN,
PLANNING,
SHOPPING_FILE,
SHOPPING_LIST,
)
THREE_DISHES = json.dumps(
[
{"id": "D-1", "name": "Sandwiches"},
{"id": "D-2", "name": "Lemonade"},
{"id": "D-3", "name": "Cake"},
]
)
FOR_THREE = json.dumps(
[
{"dish": "D-1", "item": "bread", "quantity": 2},
{"dish": "D-2", "item": "lemons", "quantity": 6},
{"dish": "D-3", "item": "flour", "quantity": 1},
]
)
THREE_VERDICTS = json.dumps(
{
"dry_run": {"verdict": "pass", "evidence": "Fits the basket."},
"checks": [
{"check": dish, "verdict": "pass", "evidence": "Enough."}
for dish in ("D-1", "D-2", "D-3")
],
"proposed_additions": [],
}
)
def actions(result) -> list[str]:
return [decision["action"] for decision in result.state["decisions"]]
async def test_a_pre_approved_execution_runs_to_the_end_and_writes_its_files(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
result = await engine.start(
"A picnic for six", pre_approved=True, customer_case="park", execution_id="e1"
)
assert result.status == "completed"
assert [run.role for run in runtime.runs] == [
"menu-writer",
"shopper",
"checker",
"packer",
]
out = folder(tmp_path, "park", "e1")
assert {path.name for path in out.iterdir()} == {
"deliverables",
"execution.json",
"menu.json",
"shopping.json",
"verdicts.json",
"picnic-plan.md",
}
record = manifest(tmp_path, "park", "e1")
assert record["final_document"] == "picnic-plan.md"
assert record["phases"] == {
PLANNING: "approved",
CHECKING: "approved",
PACKING: "approved",
}
# The final Deliverable is also the file at the top of the folder.
assert (out / "picnic-plan.md").read_text() == (
out / "deliverables" / "plan.md"
).read_text()
# What the Workflow records of its own, and nothing of any other subject.
assert record["dishes"] == ["D-1", "D-2"]
assert not {"solution_guide", "terraform_modules", "delivery_plan"} & set(record)
async def test_no_spec_is_needed_and_none_is_recorded(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
result = await engine.start("A picnic", pre_approved=True, execution_id="e1")
assert result.state["scope"] is None
assert manifest(tmp_path, "unassigned", "e1")["scope"] is None
assert all(run.task.context["scope"] is None for run in runtime.runs)
async def test_code_adds_its_view_to_a_submission(tmp_path):
engine, _ = make_engine(tmp_path, once_each())
result = await engine.start("A picnic", pre_approved=True)
shopping = result.state["deliverables"][SHOPPING_LIST]["content"]
assert shopping.rstrip().endswith("## Totals\n\n3 items for 2 dishes.")
async def test_each_specialist_is_given_what_it_works_from(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
await engine.start("A picnic", pre_approved=True)
by_role = {run.role: run.task.context for run in runtime.runs}
assert by_role["menu-writer"]["dependencies"] == {}
assert set(by_role["shopper"]["dependencies"]) == {MENU}
assert by_role["shopper"]["contract"] == {
"sections": ["What to buy"],
"attachments": [SHOPPING_FILE],
}
# The check works from the list; the menu arrives as its data file alone.
assert set(by_role["checker"]["dependencies"]) == {MENU, SHOPPING_LIST}
assert by_role["checker"]["dependencies"][MENU] == {
"attachments": {MENU_FILE: runtime.runs[0].outcome.attachments[MENU_FILE]}
}
# A section code wrote for a reader is not shown to the next Specialist, which
# has the same facts as data. The final Deliverable assembles it, so it is.
assert (
"## Totals" not in by_role["checker"]["dependencies"][SHOPPING_LIST]["content"]
)
assert set(by_role["packer"]["dependencies"]) == {MENU, SHOPPING_LIST, CHECK}
assert "## Totals" in by_role["packer"]["dependencies"][SHOPPING_LIST]["content"]
async def test_a_submission_that_breaks_the_chain_is_refused(tmp_path):
turns = once_each()
nothing_for_lemonade = json.dumps([{"dish": "D-1", "item": "bread"}])
turns["shopper"] = [
[
*submits(SHOPPING_LIST, **{SHOPPING_FILE: nothing_for_lemonade}),
*submits(SHOPPING_LIST),
]
]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
refused, accepted = runtime.runs[1].results
assert "nothing is bought for dish D-2" in refused[1]
assert accepted[1] == "Submitted shopping_list."
async def test_with_a_user_each_phase_waits_for_its_approval(tmp_path):
engine, runtime = make_engine(tmp_path, once_each())
first = await engine.start("A picnic", execution_id="e1")
assert first.status == "awaiting_user"
assert first.prompt["approval"]["phase"] == PLANNING
assert first.prompt["approval"]["deliverables"] == [MENU, SHOPPING_LIST]
# The summary is assembled from the Workflow's own facts.
assert "2 dishes on the menu." in first.prompt["approval"]["summary"]
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper"]
second = await engine.resume("e1", approve(PLANNING))
assert second.prompt["approval"]["phase"] == CHECKING
assert (
"Validation: 2 Checks passed, 0 failed" in second.prompt["approval"]["summary"]
)
third = await engine.resume("e1", approve(CHECKING))
assert third.prompt["approval"]["deliverables"] == [PLAN]
done = await engine.resume("e1", approve(PACKING))
assert done.status == "completed"
async def test_a_specialists_question_reaches_the_user_and_the_answer_comes_back(
tmp_path,
):
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
lambda task: (
submits(MENU) if task.context["answers"][-1]["reply"] == "None" else []
),
]
engine, _ = make_engine(tmp_path, turns)
asked = await engine.start("A picnic", execution_id="e1")
assert asked.prompt == {"questions": [{"from": MENU, "question": "Any allergies?"}]}
answered = await engine.resume("e1", {"reply": "None"})
assert answered.prompt["approval"]["phase"] == PLANNING
async def test_a_failed_check_sends_the_list_back_and_the_menu_stands(tmp_path):
turns = once_each()
turns["shopper"] = [
submits(SHOPPING_LIST),
lambda task: submits(SHOPPING_LIST) if task.context["feedback"] else [],
]
turns["checker"] = [fails(D_2="fail"), submits(CHECK)]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
# The Workflow says what a failed Check reopens: the list, not the menu.
assert [run.role for run in runtime.runs] == [
"menu-writer",
"shopper",
"checker",
"shopper",
"checker",
"packer",
]
assert "revise_code" in actions(result)
assert runtime.runs[3].task.context["feedback"] == ["D-2 failed: D-2 is short."]
ledger = result.state["ledger"]["deliverables"]
assert ledger[SHOPPING_LIST]["revisions"] == 1
assert ledger[MENU]["revisions"] == 0
async def test_a_check_that_fails_twice_running_sends_the_menu_back(tmp_path):
turns = once_each()
turns["menu-writer"] = [submits(MENU), submits(MENU)]
turns["shopper"] = [submits(SHOPPING_LIST)] * 3
turns["checker"] = [fails(D_2="fail"), fails(D_2="fail"), submits(CHECK)]
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
assert "revise_target_architecture" in actions(result)
assert result.state["ledger"]["deliverables"][MENU]["revisions"] == 1
assert [run.role for run in runtime.runs].count("menu-writer") == 2
async def test_a_user_who_asks_for_changes_at_an_approval_gets_a_revision(tmp_path):
turns = once_each()
turns["shopper"] = [submits(SHOPPING_LIST), submits(SHOPPING_LIST)]
engine, runtime = make_engine(tmp_path, turns)
await engine.start("A picnic", execution_id="e1")
again = await engine.resume(
"e1",
{
"approval": {
"phase": PLANNING,
"decision": "changes",
"feedback": "More lemons on the shopping list, please.",
}
},
)
# Named outright, so it is revised with no decision model to say which.
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper", "shopper"]
assert "More lemons" in runtime.runs[2].task.context["feedback"][0]
assert again.prompt["approval"]["phase"] == PLANNING
async def test_a_change_request_starts_from_its_baseline_and_says_what_changed(
tmp_path,
):
engine, _ = make_engine(tmp_path, once_each())
await engine.start(
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
)
turns = {
"menu-writer": [
lambda task: (
submits(MENU, **{MENU_FILE: THREE_DISHES})
if task.context["baseline"]["deliverable"]
else []
)
],
"shopper": [submits(SHOPPING_LIST, **{SHOPPING_FILE: FOR_THREE})],
"checker": [submits(CHECK, **{"verdicts.json": THREE_VERDICTS})],
# The final Deliverable is written afresh; the Baseline's is not handed back.
"packer": [
lambda task: (
submits(PLAN) if task.context["baseline"]["deliverable"] is None else []
)
],
}
changed, runtime = make_engine(tmp_path, turns)
result = await changed.start(
"Add a cake.",
pre_approved=True,
customer_case="park",
execution_id="second",
baseline="first",
)
assert result.status == "completed"
out = folder(tmp_path, "park", "second")
assert json.loads((out / "changes.json").read_text()) == {
"added": ["D-3"],
"removed": [],
}
assert manifest(tmp_path, "park", "second")["baseline"] == "first"
# The shopper is told what has changed upstream of its Deliverable, in data.
assert runtime.runs[1].task.context["baseline"]["changes"] == {
"added": ["D-3"],
"removed": [],
}
# The Baseline is never modified.
assert manifest(tmp_path, "park", "first")["dishes"] == ["D-1", "D-2"]
async def test_a_baseline_from_another_customer_case_is_refused(tmp_path):
engine, _ = make_engine(tmp_path, once_each())
await engine.start(
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
)
other, _ = make_engine(tmp_path, {})
with pytest.raises(ValueError, match="another Customer case"):
await other.start(
"Add a cake.", pre_approved=True, customer_case="beach", baseline="first"
)
async def test_an_option_takes_its_default_the_callers_value_or_its_baselines(
tmp_path,
):
engine_turns = once_each()
for role in engine_turns:
engine_turns[role] = engine_turns[role] * 3
engine, runtime = make_engine(tmp_path, engine_turns)
default = await engine.start("A picnic", pre_approved=True, customer_case="park")
chosen = await engine.start(
"A picnic",
pre_approved=True,
customer_case="park",
execution_id="six",
guests=6,
)
followed = await engine.start(
"The same again.", pre_approved=True, customer_case="park", baseline="six"
)
assert default.state["options"] == {"guests": 4}
assert chosen.state["options"] == {"guests": 6}
assert followed.state["options"] == {"guests": 6}
assert manifest(tmp_path, "park", "six")["options"] == {"guests": 6}
# Every Specialist is told, through the Workflow's own entry in its task.
assert {run.task.context["guests"] for run in runtime.runs[4:8]} == {6}
@pytest.mark.parametrize(
("options", "message"),
[
({"guests": 0}, "guests must be a whole number"),
({"guests": "many"}, "guests must be a whole number"),
({"terraform_modules": True}, "Unknown options: terraform_modules"),
],
)
async def test_an_option_the_workflow_does_not_allow_refuses_the_start(
tmp_path, options, message
):
engine, runtime = make_engine(tmp_path, once_each())
with pytest.raises(ValueError, match=message):
await engine.start("A picnic", pre_approved=True, **options)
assert runtime.runs == []
async def test_with_nobody_to_answer_a_specialist_that_cannot_finish_is_stopped(
tmp_path,
):
turns = once_each()
asks = [Call("needs_input", {"questions": ["Which park?"]})]
turns["menu-writer"] = [asks] * 10
engine, runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed_with_warnings"
assert [run.role for run in runtime.runs] == ["menu-writer"] * 3
assert result.state["warnings"] == [
(
"Stopped: menu-writer ended 3 runs running without submitting menu, with "
"nobody to answer it."
)
]
async def test_a_specialist_that_finishes_on_its_last_chance_is_not_stopped(tmp_path):
turns = once_each()
asks = [Call("needs_input", {"questions": ["Which park?"]})]
turns["menu-writer"] = [asks, asks, submits(MENU)]
engine, _runtime = make_engine(tmp_path, turns)
result = await engine.start("A picnic", pre_approved=True)
assert result.status == "completed"
assert result.state["unfinished_runs"] == {}

128
tests/toy/test_roles.py Normal file
View File

@@ -0,0 +1,128 @@
"""Role files, loaded for a Workflow and a provider bundle that are not the Reference's."""
import shutil
from pathlib import Path
import pytest
from execution_engine.adk.roles import RoleConfigError, load_roles
from execution_engine.providers import load_provider
from tests.toy.workflow import ROOT, TOY
def load(root: Path = ROOT, **options):
return load_roles(root / "agents", load_provider("toy", root), TOY, **options)
@pytest.fixture
def root(tmp_path: Path) -> Path:
"""A copy of the toy agent's folder, to break."""
shutil.copytree(ROOT, tmp_path / "root")
return tmp_path / "root"
def test_the_bundle_needs_no_output_template_and_no_scope():
bundle = load_provider("toy", ROOT)
assert bundle.output_template is None
assert bundle.scope_skill is None
assert set(bundle.skills) == {"picnic-planning"}
def test_a_workflows_roles_are_the_engines_two_and_one_per_deliverable():
roles = load()
assert set(roles) == {
"orchestrator",
"reply-writer",
"menu-writer",
"shopper",
"checker",
"packer",
}
def test_a_role_takes_its_skills_model_and_timeout_from_the_bundle():
roles = load(specialist_budget=7, orchestrator_budget=33)
assert roles["shopper"].skills == ("picnic-planning",)
assert roles["packer"].skills == ()
assert roles["shopper"].model == "scripted/none"
assert roles["checker"].model == "scripted/careful"
assert roles["checker"].model_timeout_seconds == 240
# A role with no budget of its own takes the setting.
assert roles["shopper"].budget == 7
assert roles["checker"].budget == 12
assert roles["orchestrator"].budget == 33
def test_a_role_file_names_its_tools_and_includes_its_partials():
roles = load()
assert roles["shopper"].tools == ("tally",)
assert roles["shopper"].instructions.startswith("You write the shopping list")
assert "## How you work" in roles["shopper"].instructions
assert "## How you work" not in roles["orchestrator"].instructions
def test_a_missing_role_file_is_refused(root):
(root / "agents" / "packer.md").unlink()
with pytest.raises(RoleConfigError, match="missing role files for packer"):
load(root)
def test_a_role_file_the_workflow_has_no_use_for_is_refused(root):
(root / "agents" / "pillar-advisor.md").write_text(
"---\nrole: pillar-advisor\n---\nAdvise.\n"
)
with pytest.raises(RoleConfigError, match="unexpected role files pillar-advisor"):
load(root)
def test_a_role_the_workflow_asks_for_must_have_its_file(root):
import dataclasses
wanting = dataclasses.replace(TOY, roles=("taster",), optional_roles=("driver",))
bundle = load_provider("toy", root)
with pytest.raises(RoleConfigError, match="missing role files for taster"):
load_roles(root / "agents", bundle, wanting)
for name in ("taster", "driver"):
(root / "agents" / f"{name}.md").write_text(f"---\nrole: {name}\n---\nHelp.\n")
assert {"taster", "driver"} <= set(load_roles(root / "agents", bundle, wanting))
def test_a_role_must_name_the_deliverable_its_workflow_gives_it(root):
path = root / "agents" / "shopper.md"
path.write_text(path.read_text().replace("shopping_list", "menu"))
with pytest.raises(RoleConfigError, match="'deliverable' must be 'shopping_list'"):
load(root)
def test_a_tool_the_bundle_does_not_offer_is_refused(root):
path = root / "agents" / "shopper.md"
path.write_text(path.read_text().replace("[tally]", "[terraform_validate]"))
with pytest.raises(RoleConfigError, match="terraform_validate are not offered"):
load(root)
def test_a_workflow_may_change_the_loaded_roles_from_the_bundle():
import dataclasses
def one_more_call(roles, bundle):
return {
name: dataclasses.replace(role, budget=role.budget + 1)
for name, role in roles.items()
}
adjusted = dataclasses.replace(TOY, adjust_roles=one_more_call)
roles = load_roles(ROOT / "agents", load_provider("toy", ROOT), adjusted)
assert roles["checker"].budget == 13

374
tests/toy/test_server.py Normal file
View File

@@ -0,0 +1,374 @@
"""An agent's server on the toy Workflow: the card, A2A tasks and the one-shot route."""
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
import pytest
from starlette.testclient import TestClient
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.roles import load_roles
from execution_engine.agent import Assembly
from execution_engine.config import Settings
from execution_engine.main import create_app
from execution_engine.providers import load_provider
from tests.toy.support import make_engine, once_each
from tests.toy.workflow import PLANNING, ROOT, TOY
SETTINGS = Settings( # type: ignore[call-arg]
_env_file=None, CLOUD_PROVIDER="toy", AGENT_NAME="picnic-agent"
)
def client_for(
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
) -> tuple[TestClient, FakeAgentRuntime]:
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, runtime = make_engine(tmp_path, turns, roles=roles, **overrides)
@asynccontextmanager
async def assembly(settings):
yield Assembly(engine, bundle, roles, {"larder": "not reachable"})
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
def send(
client: TestClient,
parts: list[dict],
task: dict | None = None,
metadata=None,
path: str = "/a2a",
):
message = {"messageId": str(uuid.uuid4()), "role": "ROLE_USER", "parts": parts}
if task:
message.update(taskId=task["id"], contextId=task["contextId"])
if metadata:
message["metadata"] = metadata
response = client.post(
path,
json={
"jsonrpc": "2.0",
"id": 1,
"method": "SendMessage",
"params": {"message": message},
},
headers={"A2A-Version": "1.0"},
)
assert response.status_code == 200, response.text
body = response.json()
assert "error" not in body, body
return body["result"]["task"]
def status_parts(task: dict) -> tuple[str, dict | None]:
parts = task["status"]["message"]["parts"]
text = "\n".join(part["text"] for part in parts if "text" in part)
data = next((part["data"] for part in parts if "data" in part), None)
return text, data
def test_the_card_describes_the_workflow_it_serves(tmp_path):
client, _ = client_for(tmp_path, {})
with client:
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
card = client.get(path).json()
skill = card["skills"][0]
assert card["name"] == "picnic-agent"
assert card["description"].startswith("Plans a picnic with you")
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
assert (skill["id"], skill["name"]) == ("picnic", "Picnic planning")
assert "planning: menu, shopping_list" in skill["description"]
assert "packing: plan" in skill["description"]
assert skill["tags"] == ["picnic", "toy"]
assert "architecture" not in str(card).lower()
def test_the_card_declares_the_fields_a_platform_step_sends(tmp_path):
# KAB's workflow builder draws a step's inputs from skills[0].inputSchema.
client, _ = client_for(tmp_path, {})
with client:
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
skill = client.get(path).json()["skills"][0]
schema = skill["inputSchema"]
assert skill["inputArrives"] == "as_fields"
assert set(schema["properties"]) == {
"request",
"customer_case",
"scope",
"pre_approved",
}
assert schema["properties"]["scope"]["uiFieldType"] == "textarea"
assert schema["properties"]["pre_approved"]["type"] == "boolean"
assert schema["required"] == ["customer_case"]
def test_a_request_in_the_metadata_is_the_executions_request(tmp_path):
# A platform step sends every input as metadata and, as the message text,
# all of them again as one JSON document. The request is the field.
client, _ = client_for(tmp_path, once_each())
with client:
task = send(
client,
[{"text": '{"request": "A picnic for nine", "pre_approved": true}'}],
metadata={
"request": "A picnic for nine",
"pre_approved": True,
"customer_case": "park",
},
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
record = (out / "execution.json").read_text()
assert "A picnic for nine" in record
assert '\\"pre_approved\\"' not in record
def test_a_caller_that_posts_to_the_agents_own_address_is_served(tmp_path):
# The KAB platform calls the URL an agent was registered with, not the
# interface its card names, so the same handler answers at the root.
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
*turns["menu-writer"],
]
client, _ = client_for(tmp_path, turns)
with client:
task = send(client, [{"text": "A picnic for six"}], path="/")
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
text, _ = status_parts(task)
assert "Any allergies?" in text
def test_health_names_the_provider_and_what_could_not_be_reached(tmp_path):
client, _ = client_for(tmp_path, {})
with client:
health = client.get("/health").json()
assert health["status"] == "healthy"
assert health["agent"] == "picnic-agent"
assert health["cloud_provider"] == "toy"
assert health["mcp_discovery_gaps"] == {"larder": "not reachable"}
def test_a_question_pauses_the_task_and_a_text_reply_resumes_it(tmp_path):
turns = once_each()
turns["menu-writer"] = [
[Call("needs_input", {"questions": ["Any allergies?"]})],
*turns["menu-writer"],
]
client, runtime = client_for(tmp_path, turns)
with client:
task = send(client, [{"text": "A picnic for six"}])
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
text, data = status_parts(task)
assert "Any allergies?" in text
assert data == {
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
}
task = send(client, [{"text": "No nuts"}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
def test_an_approval_counts_only_as_structured_data(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
task = send(client, [{"text": "A picnic"}])
text, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
assert "Please review the planning Phase" in text
assert '"decision": "approve"' in text
task = send(client, [{"text": "approve"}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == PLANNING
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
task = send(client, [{"data": approval}], task)
_, data = status_parts(task)
assert data["prompt"]["approval"]["phase"] == "checking"
def test_a_pre_approved_task_completes_with_its_files_as_artefacts(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
task = send(
client,
[{"text": "A picnic"}],
metadata={"pre_approved": True, "guests": 9, "customer_case": "park"},
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
text, data = status_parts(task)
assert text == "The picnic is planned."
assert data == {"status": "completed", "warnings": []}
names = {artifact["name"] for artifact in task["artifacts"]}
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
# An option the Workflow has is taken from the message's metadata.
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
assert '"guests": 9' in (out / "execution.json").read_text()
def test_an_engine_failure_fails_the_task(tmp_path):
# An Orchestrator on a model, with nothing scripted for it to do.
client, _ = client_for(tmp_path, {}, orchestrator=None)
with client:
task = send(client, [{"text": "A picnic"}])
assert task["status"]["state"] == "TASK_STATE_FAILED"
assert "ScriptExhausted" in status_parts(task)[0]
def test_generate_runs_pre_approved_and_replies_with_the_deliverables(tmp_path):
client, _ = client_for(tmp_path, once_each())
with client:
response = client.post("/generate", json={"request": "A picnic", "guests": 2})
body = response.json()
assert response.status_code == 200
assert body["status"] == "completed"
assert set(body["deliverables"]) == {"menu", "shopping_list", "check", "plan"}
assert body["warnings"] == []
@pytest.mark.parametrize(
("payload", "error"),
[
({}, "'request' or a 'scope' spec is required"),
({"request": "A picnic", "guests": 0}, "guests must be a whole number"),
(
{"request": "A picnic", "customer_case": "Not A Slug"},
"customer_case must be",
),
],
)
def test_generate_refuses_what_cannot_start(tmp_path, payload, error):
client, runtime = client_for(tmp_path, once_each())
with client:
response = client.post("/generate", json=payload)
assert response.status_code == 400
assert error in response.json()["error"]
assert runtime.runs == []
def test_an_app_needs_the_agents_folder_or_a_way_to_assemble_it():
with pytest.raises(ValueError, match="needs the agent's root"):
create_app(TOY, SETTINGS)
# --- messages that start nothing ---------------------------------------------------------
def client_with_intake(tmp_path, decider, turns=None):
from execution_engine.adk.runtime import RoleSpec
from execution_engine.nodes.intake import DeliveredAnswers, Intake
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, runtime = make_engine(tmp_path, turns or once_each(), roles=roles)
@asynccontextmanager
async def assembly(settings):
yield Assembly(
engine,
bundle,
roles,
{},
intake=Intake(decider, tmp_path),
answers=DeliveredAnswers(
runtime,
RoleSpec("reply-writer", "Write.", budget=3),
tmp_path,
TOY.final,
),
)
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
def test_a_question_about_the_agent_is_answered_in_the_workflows_own_words(tmp_path):
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
decider = FakeDecider(
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.05, 0.9)}
)
client, runtime = client_with_intake(tmp_path, decider)
with client:
task = send(client, [{"text": "What can you do?"}])
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert status_parts(task)[0] == TOY.about
assert runtime.runs == []
def test_a_question_about_delivered_work_is_answered_from_its_final_document(tmp_path):
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
seen: dict = {}
def writer(task):
seen.update(task.context)
return [Call("write_reply", {"text": "Lemonade, because it is summer."})]
decider = FakeDecider(
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.95, 0.9)}
)
turns = {**once_each(), "reply-writer": [writer]}
client, _ = client_with_intake(tmp_path, decider, turns)
with client:
send(
client,
[{"text": "A picnic"}],
metadata={"pre_approved": True, "customer_case": "park"},
)
task = send(
client, [{"text": "Why lemonade?"}], metadata={"customer_case": "park"}
)
assert status_parts(task)[0] == "Lemonade, because it is summer."
# Written from the plan, the Workflow's final Deliverable.
assert seen["delivered"]["final_document"].startswith("# plan")
def test_a_workflow_built_from_the_settings_and_bundle_is_the_one_assembled(
tmp_path, monkeypatch
):
# An agent whose Workflow depends on its settings and provider bundle gives a
# function to build it; the card still describes the Workflow it was given.
import execution_engine.main as server
seen: dict[str, Any] = {}
bundle = load_provider("toy", ROOT)
roles = load_roles(ROOT / "agents", bundle, TOY)
engine, _ = make_engine(tmp_path, {}, roles=roles)
@asynccontextmanager
async def fake_open_assembly(settings, workflow, *, root):
seen.update(workflow=workflow, root=root)
yield Assembly(engine, bundle, roles, {})
def build(settings, bundle):
return TOY
monkeypatch.setattr(server, "open_assembly", fake_open_assembly)
app = server.create_app(TOY, SETTINGS, root=ROOT, build=build)
with TestClient(app) as client:
assert client.get("/health").json()["status"] == "healthy"
assert (
client.get("/.well-known/agent-card.json").json()["name"] == "picnic-agent"
)
assert seen == {"workflow": build, "root": ROOT}

155
tests/toy/test_workflow.py Normal file
View File

@@ -0,0 +1,155 @@
"""A Workflow is checked when it is built, and holds its names as plain strings."""
import dataclasses
from enum import StrEnum
import pytest
from execution_engine.workflows.deliverables import DeliverableSpec
from execution_engine.workflows.workflow import (
ValidationPolicy,
Workflow,
WorkflowError,
)
from tests.toy.workflow import CHECK, MENU, PLAN, SHOPPING_LIST, TOY
def spec(kind: str, phase: str, role: str, *depends_on: str) -> DeliverableSpec:
return DeliverableSpec(kind, phase, role, depends_on=depends_on)
def build(
*specs: DeliverableSpec, phases=("one", "two"), final="b", **parts
) -> Workflow:
return Workflow(
name="t",
about="",
phases=phases,
deliverables={each.kind: each for each in specs},
final=final,
**parts,
)
A, B = spec("a", "one", "first"), spec("b", "two", "second", "a")
def test_a_workflow_that_holds_together_is_accepted():
workflow = build(A, B)
assert workflow.specialists() == {"first": "a", "second": "b"}
assert workflow.inputs_of("b") == ("a",)
assert workflow.dependents_of("a") == ("b",)
@pytest.mark.parametrize(
("specs", "parts", "problem"),
[
((A, spec("b", "two", "second", "c")), {}, "b: names unknown Deliverable c"),
((A, spec("b", "three", "second")), {}, "b: Phase three is not listed"),
((A, spec("b", "one", "second")), {}, "Phase two has no Deliverable"),
((A, spec("b", "two", "first")), {}, "b: first already produces a"),
(
(spec("a", "one", "first", "b"), spec("b", "two", "second")),
{},
"a: depends on b, in a later Phase",
),
((A, B), {"final": "z"}, "final Deliverable z is not declared"),
(
(A, B),
{"data_checks": {"z": ("z.json", lambda *_: [])}},
"unknown Deliverable z",
),
((A, B), {"spec_deliverable": "z"}, "unknown Deliverable z"),
(
(A, B),
{"validation": ValidationPolicy("b", "v.json", "a", "z")},
"unknown Deliverable z",
),
((A, B), {"phases": ("one", "two", "one")}, "a Phase is listed twice"),
],
)
def test_a_workflow_that_does_not_hold_together_is_refused(specs, parts, problem):
with pytest.raises(WorkflowError, match=problem):
build(*specs, **parts)
def test_a_spec_filed_under_another_kind_is_refused():
with pytest.raises(WorkflowError, match="a: its spec is for x"):
Workflow(
name="t",
about="",
phases=("one",),
deliverables={"a": spec("x", "one", "first")},
final="a",
)
def test_names_written_as_an_enum_are_held_as_plain_strings():
class Kind(StrEnum):
A = "a"
B = "b"
class Phase(StrEnum):
ONE = "one"
TWO = "two"
workflow = Workflow(
name="t",
about="",
phases=tuple(Phase),
deliverables={
Kind.A: DeliverableSpec(Kind.A, Phase.ONE, "first"),
Kind.B: DeliverableSpec(Kind.B, Phase.TWO, "second", depends_on=(Kind.A,)),
},
final=Kind.B,
data_checks={Kind.A: ("a.json", lambda *_: [])},
)
# They go into checkpointed state, which is plain data.
names = [
*workflow.phases,
*workflow.deliverables,
workflow.final,
*workflow.data_checks,
*workflow.deliverables["b"].depends_on,
workflow.deliverables["b"].phase,
]
assert all(type(name) is str for name in names)
assert workflow.kind(Kind.A) == "a"
assert type(workflow.kind(Kind.A)) is str
def test_a_name_that_is_not_the_workflows_is_refused():
with pytest.raises(
ValueError, match="'requirements' is not a Deliverable of picnic"
):
TOY.kind("requirements")
with pytest.raises(ValueError, match="'scoping' is not a Phase of picnic"):
TOY.phase("scoping")
def test_the_final_deliverable_reads_everything_and_is_shown_what_readers_are():
assert TOY.inputs_of(PLAN) == (MENU, SHOPPING_LIST, CHECK)
given = {
MENU: {"content": "# Menu\n", "attachments": {"menu.json": "[]"}},
SHOPPING_LIST: {
"content": "# List\n\n## Totals\n\n3 items.\n",
"attachments": {"shopping.json": "[]"},
},
}
for_checker = TOY.shown(CHECK, TOY.dependencies_for(CHECK, given))
assert for_checker[SHOPPING_LIST]["content"] == "# List\n"
# Not read in full: the data file alone.
assert for_checker[MENU] == {"attachments": {"menu.json": "[]"}}
def test_a_workflow_with_no_validation_never_reads_one():
plain = dataclasses.replace(TOY, validation=None)
assert plain.validation is None
assert plain.check_submission(MENU, "# Menu\n## Dishes\n", {"menu.json": "[]"}) == (
"menu.json: lists no dishes",
)

301
tests/toy/workflow.py Normal file
View File

@@ -0,0 +1,301 @@
"""A toy Workflow: planning a picnic.
It is here to show that the engine needs nothing of the subject it was built from
(ADR-0001). Four Deliverables in three Phases: a menu, a shopping list written from
it, a check that every dish can be made from the list, and the plan that puts the
three together. It has no spec, no graph
and no cloud, and it uses each kind of part a Workflow can supply at least once:
data checks along a chain, code that adds a view on submission, a validation that
reopens the list or the menu, an account of what a Change request changed, an
option a caller may set, a tool of its own, and its own facts at an Approval.
"""
from __future__ import annotations
import json
from collections.abc import Mapping, MutableMapping, Sequence
from pathlib import Path
from typing import Any
from execution_engine.adk.runtime import ToolResult, ToolSpec
from execution_engine.workflows.deliverables import DeliverableSpec, Upstream
from execution_engine.workflows.workflow import (
Assembling,
Bench,
CardText,
ChangePolicy,
LocalToolBuilder,
OptionSpec,
ValidationPolicy,
Workflow,
)
ROOT = Path(__file__).parent / "root" # the toy agent's own folder
PLANNING, CHECKING, PACKING = "planning", "checking", "packing"
MENU, SHOPPING_LIST, CHECK, PLAN = "menu", "shopping_list", "check", "plan"
MENU_FILE, SHOPPING_FILE, VERDICTS_FILE = "menu.json", "shopping.json", "verdicts.json"
CHANGES_FILE = "changes.json"
PLAN_FILE = "picnic-plan.md"
TOTALS_HEADING = "Totals"
GUESTS = "guests"
def _list(text: str, name: str) -> tuple[list[dict[str, Any]], list[str]]:
try:
data = json.loads(text)
except ValueError as exc:
return [], [f"{name}: not valid JSON ({exc})"]
if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
return [], [f"{name}: must be a JSON list of objects"]
return data, []
def dishes(deliverables: Upstream | None) -> list[dict[str, Any]]:
"""The dishes on the menu, from its data file; none when it is not there."""
text = ((deliverables or {}).get(MENU) or {}).get("attachments", {}).get(MENU_FILE)
return _list(text, MENU_FILE)[0] if text else []
def dish_ids(deliverables: Upstream | None) -> list[str]:
return [str(dish.get("id")) for dish in dishes(deliverables)]
def check_menu(attachments: Mapping[str, str], upstream: Upstream | None) -> list[str]:
items, problems = _list(attachments[MENU_FILE], MENU_FILE)
if problems:
return problems
if not items:
return [f"{MENU_FILE}: lists no dishes"]
ids = [item.get("id") for item in items]
problems += [
f"{MENU_FILE}[{index}]: needs id and name"
for index, item in enumerate(items)
if not item.get("id") or not item.get("name")
]
problems += [
f"{MENU_FILE}: dish {dish} is listed more than once"
for dish in sorted({str(i) for i in ids if i and ids.count(i) > 1})
]
return problems
def check_shopping(
attachments: Mapping[str, str], upstream: Upstream | None
) -> list[str]:
items, problems = _list(attachments[SHOPPING_FILE], SHOPPING_FILE)
if problems:
return problems
on_menu = dish_ids(upstream)
bought_for = {str(item.get("dish")) for item in items}
problems += [
f"{SHOPPING_FILE}: nothing is bought for dish {dish}"
for dish in on_menu
if dish not in bought_for
]
if upstream is not None:
problems += [
f"{SHOPPING_FILE}: dish {dish} is not on the menu"
for dish in sorted(bought_for - set(on_menu))
]
return problems
def check_verdicts(
attachments: Mapping[str, str], upstream: Upstream | None
) -> list[str]:
try:
data = json.loads(attachments[VERDICTS_FILE])
except ValueError as exc:
return [f"{VERDICTS_FILE}: not valid JSON ({exc})"]
given = {
str(item.get("check"))
for item in (data.get("checks") or [] if isinstance(data, dict) else [])
if isinstance(item, dict)
}
return [
f"{VERDICTS_FILE}: dish {dish} has no verdict"
for dish in dish_ids(upstream)
if dish not in given
]
def with_totals(
bench: Bench, content: str, attachments: dict[str, str]
) -> tuple[str, dict[str, str]]:
"""How much is bought, counted from the list: it cannot disagree with it."""
items, _ = _list(attachments[SHOPPING_FILE], SHOPPING_FILE)
content = content.split(f"\n## {TOTALS_HEADING}")[0].rstrip()
content += (
f"\n\n## {TOTALS_HEADING}\n\n{len(items)} items for "
f"{len(dish_ids(bench.upstream))} dishes.\n"
)
return content, attachments
def change_summary(before: Upstream | None, after: Upstream) -> dict[str, Any]:
was, now = set(dish_ids(before)), set(dish_ids(after))
return {"added": sorted(now - was), "removed": sorted(was - now)}
def changes_in_words(summary: Mapping[str, Any]) -> str:
return "\n".join(
[f"Dish {dish} was added." for dish in summary["added"]]
+ [f"Dish {dish} was removed." for dish in summary["removed"]]
)
def approval_facts(phase: str, deliverables: Upstream) -> dict[str, Any]:
return {"dishes": len(dishes(deliverables))}
def summary_lines(phase: str, facts: Mapping[str, Any]) -> list[str]:
return [f"{facts['dishes']} dishes on the menu."]
def task_entries(
state: Mapping[str, Any], kind: str
) -> tuple[dict[str, Any], dict[str, Any]]:
# How many people the picnic is for: every Specialist is told.
return {}, {GUESTS: (state.get("options") or {}).get(GUESTS)}
def outputs(
state: Mapping[str, Any], deliverables: Upstream, files: dict[str, str]
) -> dict[str, Any]:
return {"dishes": dish_ids(deliverables)}
def _guests(value: Any) -> str | None:
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
return "guests must be a whole number of people, at least 1"
return None
def local_tools(given: Assembling) -> LocalToolBuilder:
"""`tally`: how many items the working copy of the shopping list holds."""
def build(working_copy: MutableMapping[str, str] | None) -> Sequence[ToolSpec]:
async def tally() -> ToolResult:
text = (working_copy or {}).get(SHOPPING_FILE, "[]")
return ToolResult(f"{len(_list(text, SHOPPING_FILE)[0])} items so far.")
return [
ToolSpec(
"tally",
"Count the items on the shopping list so far.",
{"type": "object", "properties": {}},
tally,
)
]
return build
SAMPLES = {
MENU_FILE: json.dumps(
[{"id": "D-1", "name": "Sandwiches"}, {"id": "D-2", "name": "Lemonade"}]
),
SHOPPING_FILE: json.dumps(
[
{"dish": "D-1", "item": "bread", "quantity": 2},
{"dish": "D-1", "item": "cheese", "quantity": 1},
{"dish": "D-2", "item": "lemons", "quantity": 6},
]
),
VERDICTS_FILE: json.dumps(
{
"dry_run": {"verdict": "pass", "evidence": "Everything fits the basket."},
"checks": [
{"check": "D-1", "verdict": "pass", "evidence": "Bread and cheese."},
{"check": "D-2", "verdict": "pass", "evidence": "Six lemons."},
],
"proposed_additions": [],
}
),
}
TOY = Workflow(
name="picnic",
about="I plan picnics: a menu, a shopping list and a check of one against the other.",
phases=(PLANNING, CHECKING, PACKING),
deliverables={
spec.kind: spec
for spec in (
DeliverableSpec(
MENU,
PLANNING,
"menu-writer",
sections=("Dishes",),
attachments=(MENU_FILE,),
),
DeliverableSpec(
SHOPPING_LIST,
PLANNING,
"shopper",
depends_on=(MENU,),
sections=("What to buy",),
attachments=(SHOPPING_FILE,),
# The list is built up in a working copy, a few lines at a time.
working_copy=SHOPPING_LIST,
),
DeliverableSpec(
CHECK,
CHECKING,
"checker",
depends_on=(SHOPPING_LIST,),
sections=("Findings",),
attachments=(VERDICTS_FILE,),
),
DeliverableSpec(
PLAN,
PACKING,
"packer",
depends_on=(CHECK,),
sections=("The plan",),
# The plan is assembled from every Deliverable's prose.
reads_in_full=(MENU, SHOPPING_LIST, CHECK),
),
)
},
final=PLAN,
final_file=PLAN_FILE,
data_checks={
MENU: (MENU_FILE, check_menu),
SHOPPING_LIST: (SHOPPING_FILE, check_shopping),
CHECK: (VERDICTS_FILE, check_verdicts),
},
for_readers={SHOPPING_LIST: TOTALS_HEADING},
validation=ValidationPolicy(
kind=CHECK,
data_file=VERDICTS_FILE,
implementation=SHOPPING_LIST,
design=MENU,
check_ids=dish_ids,
known_ids=lambda deliverables: set(dish_ids(deliverables)),
),
on_submit={SHOPPING_LIST: with_totals},
approval_facts=approval_facts,
summary_lines=summary_lines,
agreed=lambda deliverables: json.dumps(dishes(deliverables)),
changes=ChangePolicy(
file=CHANGES_FILE,
summary=change_summary,
is_empty=lambda summary: not (summary["added"] or summary["removed"]),
in_words=changes_in_words,
),
subject="a picnic plan",
task_entries=task_entries,
outputs=outputs,
options={GUESTS: OptionSpec(default=4, check=_guests)},
local_tools=local_tools,
card=CardText(
description="Plans a picnic with you: a menu, a shopping list and a check.",
skill_id="picnic",
skill_name="Picnic planning",
tags=("picnic",),
examples=("Plan a picnic for six in the park.",),
),
done="The picnic is planned.",
samples=SAMPLES,
)