feat: working orhcestration engine
This commit is contained in:
0
tests/toy/__init__.py
Normal file
0
tests/toy/__init__.py
Normal file
7
tests/toy/root/agents/checker.md
Normal file
7
tests/toy/root/agents/checker.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
role: checker
|
||||
deliverable: check
|
||||
budget: 12
|
||||
includes: [protocol]
|
||||
---
|
||||
You check that every dish can be made from the shopping list.
|
||||
6
tests/toy/root/agents/menu-writer.md
Normal file
6
tests/toy/root/agents/menu-writer.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
role: menu-writer
|
||||
deliverable: menu
|
||||
includes: [protocol]
|
||||
---
|
||||
You write the menu: the dishes, each with an id.
|
||||
4
tests/toy/root/agents/orchestrator.md
Normal file
4
tests/toy/root/agents/orchestrator.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
role: orchestrator
|
||||
---
|
||||
You coordinate a picnic plan. You produce nothing yourself.
|
||||
6
tests/toy/root/agents/packer.md
Normal file
6
tests/toy/root/agents/packer.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
role: packer
|
||||
deliverable: plan
|
||||
includes: [protocol]
|
||||
---
|
||||
You put the menu, the shopping list and the check together as the picnic plan.
|
||||
3
tests/toy/root/agents/partials/protocol.md
Normal file
3
tests/toy/root/agents/partials/protocol.md
Normal file
@@ -0,0 +1,3 @@
|
||||
## How you work
|
||||
|
||||
You produce exactly one Deliverable, named in your task. Submit it with `submit_deliverable` when it meets its `contract`, or ask with `needs_input`.
|
||||
4
tests/toy/root/agents/reply-writer.md
Normal file
4
tests/toy/root/agents/reply-writer.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
role: reply-writer
|
||||
---
|
||||
You write what the user reads, from the facts you are given.
|
||||
7
tests/toy/root/agents/shopper.md
Normal file
7
tests/toy/root/agents/shopper.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
role: shopper
|
||||
deliverable: shopping_list
|
||||
tools: [tally]
|
||||
includes: [protocol]
|
||||
---
|
||||
You write the shopping list: what to buy for every dish on the menu.
|
||||
19
tests/toy/root/providers/toy.yaml
Normal file
19
tests/toy/root/providers/toy.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
# A provider bundle for the engine's own tests. Nothing here is a real provider.
|
||||
name: toy
|
||||
|
||||
skills_dirs:
|
||||
- skills
|
||||
|
||||
models:
|
||||
default: scripted/none
|
||||
roles:
|
||||
checker: scripted/careful
|
||||
timeout_seconds:
|
||||
checker: 240
|
||||
|
||||
role_skills:
|
||||
menu-writer: [picnic-planning]
|
||||
shopper: [picnic-planning]
|
||||
|
||||
tools:
|
||||
general: [tally]
|
||||
10
tests/toy/root/skills/overlays/house-rules/SKILL.md
Normal file
10
tests/toy/root/skills/overlays/house-rules/SKILL.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
name: house-rules
|
||||
description: >-
|
||||
A stand-in overlay Skill for the engine's own tests: one household's rules for a
|
||||
picnic. Use on any Deliverable of an Execution that names this overlay.
|
||||
---
|
||||
|
||||
# House rules
|
||||
|
||||
Nothing here is a rule. It exists so a test can name an overlay Skill.
|
||||
10
tests/toy/root/skills/picnic-planning/SKILL.md
Normal file
10
tests/toy/root/skills/picnic-planning/SKILL.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
name: picnic-planning
|
||||
description: >-
|
||||
A stand-in Skill for the engine's own tests: how a picnic is planned. Use when
|
||||
writing the menu or the shopping list of a picnic plan.
|
||||
---
|
||||
|
||||
# Picnic planning
|
||||
|
||||
Nothing here is guidance. It exists so the toy Workflow's roles have a Skill.
|
||||
96
tests/toy/support.py
Normal file
96
tests/toy/support.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""What the toy Workflow's tests share: its roles, an engine, and scripted turns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from execution_engine.adk.runtime import NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.toy.workflow import CHECK, SAMPLES, TOY, VERDICTS_FILE
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in TOY.deliverables.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def submits(kind: str, **files: str) -> list[Call]:
|
||||
"""One turn that submits a passing Deliverable, with `files` in place of its
|
||||
sample attachments."""
|
||||
call = sample_submission(TOY, kind)
|
||||
return [
|
||||
Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {**call.args["attachments"], **files}},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def verdicts(**by_dish: str) -> str:
|
||||
"""verdicts.json with the sample's verdicts, and `by_dish` in place of them:
|
||||
`verdicts(D_2="fail")`."""
|
||||
data = json.loads(SAMPLES[VERDICTS_FILE])
|
||||
for check in data["checks"]:
|
||||
verdict = by_dish.get(check["check"].replace("-", "_"))
|
||||
if verdict:
|
||||
check.update(verdict=verdict, evidence=f"{check['check']} is short.")
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def fails(**by_dish: str) -> list[Call]:
|
||||
return submits(CHECK, **{VERDICTS_FILE: verdicts(**by_dish)})
|
||||
|
||||
|
||||
def once_each() -> dict[str, list[Any]]:
|
||||
"""Every Specialist submits a passing Deliverable, once."""
|
||||
return {spec.specialist: [submits(kind)] for kind, spec in TOY.deliverables.items()}
|
||||
|
||||
|
||||
def make_engine(
|
||||
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
|
||||
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
||||
"""An engine on the toy Workflow whose Orchestrator routes by its playbook's
|
||||
rules: no decision model, and no script for the Orchestrator."""
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
options: dict[str, Any] = {
|
||||
"workflow": TOY,
|
||||
"runtime": runtime,
|
||||
"roles": ROLES,
|
||||
"recorder": InMemoryRecorder(),
|
||||
"checkpointer": InMemorySaver(),
|
||||
"output_root": tmp_path,
|
||||
"orchestrator": Router(NoDecider(), None, Thresholds()),
|
||||
}
|
||||
options.update(overrides)
|
||||
return ExecutionEngine(**options), runtime
|
||||
|
||||
|
||||
def approve(phase: str) -> dict[str, Any]:
|
||||
return {"approval": {"phase": phase, "decision": "approve"}}
|
||||
|
||||
|
||||
def folder(tmp_path: Path, customer_case: str, execution_id: str) -> Path:
|
||||
return (
|
||||
tmp_path
|
||||
/ "deliverables"
|
||||
/ "cases"
|
||||
/ customer_case
|
||||
/ "executions"
|
||||
/ execution_id
|
||||
)
|
||||
|
||||
|
||||
def manifest(tmp_path: Path, customer_case: str, execution_id: str) -> dict[str, Any]:
|
||||
path = folder(tmp_path, customer_case, execution_id) / "execution.json"
|
||||
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
118
tests/toy/test_assembly.py
Normal file
118
tests/toy/test_assembly.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""Assembling an agent from settings, its own folder and a Workflow."""
|
||||
|
||||
import dataclasses
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import NoDecider
|
||||
from execution_engine.agent import open_assembly
|
||||
from execution_engine.config import Settings
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.toy.support import once_each, submits
|
||||
from tests.toy.workflow import ROOT, SHOPPING_LIST, TOY
|
||||
|
||||
|
||||
def settings(**overrides: Any) -> Settings:
|
||||
values: dict[str, Any] = {
|
||||
"CLOUD_PROVIDER": "toy",
|
||||
"CHECKPOINTER": "memory",
|
||||
"MCP_ENABLED": False,
|
||||
"ORCHESTRATOR_BUDGET": 9,
|
||||
"SPECIALIST_BUDGET": 4,
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(_env_file=None, **values) # type: ignore[call-arg]
|
||||
|
||||
|
||||
def assembled(tmp_path, runtime, workflow=TOY, **overrides):
|
||||
return open_assembly(
|
||||
settings(**overrides),
|
||||
workflow,
|
||||
root=ROOT,
|
||||
runtime=runtime,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
async def test_an_agent_is_its_folder_its_settings_and_its_workflow(tmp_path):
|
||||
runtime = FakeAgentRuntime(once_each())
|
||||
|
||||
async with assembled(tmp_path, runtime) as assembly:
|
||||
result = await assembly.engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert assembly.engine.workflow is TOY
|
||||
assert assembly.bundle.name == "toy"
|
||||
assert assembly.roles["orchestrator"].budget == 9
|
||||
assert assembly.roles["shopper"].budget == 4
|
||||
assert assembly.mcp_gaps == {}
|
||||
# With no Jev key the router follows its playbook on rules alone, and no
|
||||
# Orchestrator model is run.
|
||||
assert isinstance(assembly.decider, NoDecider)
|
||||
assert "orchestrator" not in {run.role for run in runtime.runs}
|
||||
|
||||
|
||||
async def test_a_role_is_given_the_workflows_own_tool_on_its_working_copy(tmp_path):
|
||||
turns = once_each()
|
||||
turns["shopper"] = [[Call("tally"), *submits(SHOPPING_LIST)]]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
|
||||
async with assembled(tmp_path, runtime) as assembly:
|
||||
await assembly.engine.start("A picnic", pre_approved=True)
|
||||
|
||||
by_role = {run.role: run for run in runtime.runs}
|
||||
assert "tally" in by_role["shopper"].tools
|
||||
assert by_role["shopper"].results[0] == ("tally", "0 items so far.")
|
||||
# Only the role whose file names it.
|
||||
assert "tally" not in by_role["checker"].tools
|
||||
|
||||
|
||||
async def test_the_model_orchestrator_is_still_there_to_be_scripted(tmp_path):
|
||||
runtime = FakeAgentRuntime(
|
||||
{"orchestrator": [[Call("ask_user", {"questions": ["How many guests?"]})]]}
|
||||
)
|
||||
|
||||
async with assembled(tmp_path, runtime, ORCHESTRATOR="model") as assembly:
|
||||
result = await assembly.engine.start("A picnic")
|
||||
|
||||
assert result.prompt["questions"][0]["question"] == "How many guests?"
|
||||
|
||||
|
||||
async def test_an_unknown_orchestrator_setting_is_refused(tmp_path):
|
||||
with pytest.raises(ValueError, match="ORCHESTRATOR must be router or model"):
|
||||
async with assembled(tmp_path, FakeAgentRuntime({}), ORCHESTRATOR="llm"):
|
||||
pass
|
||||
|
||||
|
||||
async def test_a_workflow_may_be_built_for_the_settings_and_the_bundle(tmp_path):
|
||||
seen = {}
|
||||
|
||||
def build(given_settings, bundle):
|
||||
seen.update(provider=bundle.name, budget=given_settings.SPECIALIST_BUDGET)
|
||||
return dataclasses.replace(TOY, done="Packed.")
|
||||
|
||||
async with assembled(tmp_path, FakeAgentRuntime({}), build) as assembly:
|
||||
assert assembly.engine.workflow.done == "Packed."
|
||||
|
||||
assert seen == {"provider": "toy", "budget": 4}
|
||||
|
||||
|
||||
async def test_overlay_skills_come_from_the_agents_own_folder(tmp_path):
|
||||
runtime = FakeAgentRuntime(once_each())
|
||||
|
||||
async with assembled(tmp_path, runtime) as assembly:
|
||||
result = await assembly.engine.start(
|
||||
"A picnic", pre_approved=True, overlay_skills=["house-rules"]
|
||||
)
|
||||
with pytest.raises(ValueError, match="Unknown overlay Skills: no-such-rules"):
|
||||
await assembly.engine.start(
|
||||
"A picnic", pre_approved=True, overlay_skills=["no-such-rules"]
|
||||
)
|
||||
|
||||
assert list(result.state["overlay_skills"]) == ["house-rules"]
|
||||
assert all(
|
||||
run.task.context["overlay_skills"] == ["house-rules"] for run in runtime.runs
|
||||
)
|
||||
169
tests/toy/test_digest.py
Normal file
169
tests/toy/test_digest.py
Normal file
@@ -0,0 +1,169 @@
|
||||
"""Sources digested into a spec at the start of an Execution."""
|
||||
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from execution_engine.scope import ESSENTIAL, Catalogue, FieldSpec
|
||||
from execution_engine.workflows.workflow import Digested
|
||||
from tests.toy.support import make_engine, once_each
|
||||
from tests.toy.workflow import TOY
|
||||
|
||||
CATALOGUE = Catalogue(
|
||||
schema_version=1,
|
||||
kinds=("picnic",),
|
||||
fields={
|
||||
spec.path: spec
|
||||
for spec in (
|
||||
FieldSpec("metadata.customer_case", tier=ESSENTIAL),
|
||||
FieldSpec("engagement.kind", type="enum", values=("picnic",)),
|
||||
FieldSpec("intent.statement", type="text", tier=ESSENTIAL),
|
||||
FieldSpec("guests.count", type="integer"),
|
||||
FieldSpec("guests.allergies", type="list"),
|
||||
FieldSpec("run.pre_approved", type="boolean"),
|
||||
)
|
||||
},
|
||||
)
|
||||
INVITATION = json.dumps(
|
||||
{"party": "park", "wants": "A picnic for the team.", "coming": 9}
|
||||
)
|
||||
|
||||
|
||||
def digest(sources):
|
||||
"""Reads `invitation.json`, a format this Workflow knows; nothing else."""
|
||||
if "invitation.json" not in sources:
|
||||
return Digested({})
|
||||
try:
|
||||
invitation = json.loads(sources["invitation.json"])
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"invitation.json: not valid JSON ({exc})") from exc
|
||||
return Digested(
|
||||
spec={
|
||||
"metadata": {"customer_case": invitation["party"]},
|
||||
"engagement": {"kind": "picnic"},
|
||||
"intent": {"statement": invitation["wants"]},
|
||||
"guests": {"count": invitation["coming"]},
|
||||
"run": {"pre_approved": True},
|
||||
},
|
||||
used=("invitation.json",),
|
||||
notes={"guests.count": "invitation.json, `coming`"},
|
||||
)
|
||||
|
||||
|
||||
DIGESTING = dataclasses.replace(TOY, digest=digest)
|
||||
|
||||
|
||||
def engine_for(tmp_path, workflow=DIGESTING):
|
||||
return make_engine(
|
||||
tmp_path, once_each(), workflow=workflow, scope_catalogue=CATALOGUE
|
||||
)
|
||||
|
||||
|
||||
def completed(tmp_path) -> dict:
|
||||
path = tmp_path / "deliverables" / "cases" / "park" / "intent"
|
||||
return yaml.safe_load((path / "001.scope.completed.yaml").read_text())
|
||||
|
||||
|
||||
async def test_an_execution_starts_from_what_its_sources_say(tmp_path):
|
||||
engine, _ = engine_for(tmp_path)
|
||||
|
||||
result = await engine.start("", sources={"invitation.json": INVITATION})
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.state["customer_case"] == "park"
|
||||
assert result.state["workflow_request"] == "A picnic for the team."
|
||||
assert result.state["scope"]["context"]["stated"]["guests"] == {"count": 9}
|
||||
|
||||
|
||||
async def test_a_digested_value_says_where_it_came_from(tmp_path):
|
||||
engine, _ = engine_for(tmp_path)
|
||||
|
||||
await engine.start("", sources={"invitation.json": INVITATION})
|
||||
|
||||
spec = completed(tmp_path)
|
||||
assert spec["provenance"]["guests.count"] == "digested"
|
||||
assert spec["provenance"]["guests.allergies"] == "unknown"
|
||||
assert spec["readiness"]["digested"] == 5
|
||||
|
||||
|
||||
async def test_what_the_caller_writes_is_laid_over_what_was_digested(tmp_path):
|
||||
engine, _ = engine_for(tmp_path)
|
||||
|
||||
result = await engine.start(
|
||||
"",
|
||||
sources={"invitation.json": INVITATION},
|
||||
scope="guests:\n count: 12\n allergies: [nuts]\n",
|
||||
)
|
||||
|
||||
spec = completed(tmp_path)
|
||||
assert result.state["scope"]["context"]["stated"]["guests"] == {
|
||||
"count": 12,
|
||||
"allergies": ["nuts"],
|
||||
}
|
||||
assert spec["provenance"]["guests.count"] == "supplied"
|
||||
assert spec["provenance"]["intent.statement"] == "digested"
|
||||
|
||||
|
||||
async def test_the_sources_are_kept_as_supplied_with_their_hashes(tmp_path):
|
||||
engine, _ = engine_for(tmp_path)
|
||||
notes = "Bring a blanket."
|
||||
|
||||
result = await engine.start(
|
||||
"", sources={"invitation.json": INVITATION, "notes/host.md": notes}
|
||||
)
|
||||
|
||||
kept = tmp_path / "deliverables" / "cases" / "park" / "intent" / "001.sources"
|
||||
assert (kept / "invitation.json").read_text() == INVITATION
|
||||
assert (kept / "notes" / "host.md").read_text() == notes
|
||||
assert result.state["sources"] == {
|
||||
"invitation.json": {
|
||||
"sha256": hashlib.sha256(INVITATION.encode()).hexdigest(),
|
||||
"digested": True,
|
||||
},
|
||||
"notes/host.md": {
|
||||
"sha256": hashlib.sha256(notes.encode()).hexdigest(),
|
||||
"digested": False,
|
||||
},
|
||||
}
|
||||
# What code did not read is a document for a Specialist to read.
|
||||
assert result.state["source_documents"] == {"notes/host.md": notes}
|
||||
execution = next(
|
||||
(tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir()
|
||||
)
|
||||
record = json.loads((execution / "execution.json").read_text())
|
||||
assert record["sources"]["invitation.json"]["digested"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sources", "problem"),
|
||||
[
|
||||
({"../invitation.json": INVITATION}, "source names must be relative paths"),
|
||||
({"invitation.json": "{"}, "invitation.json: not valid JSON"),
|
||||
({"invitation.json": 7}, "sources must be files by name"),
|
||||
],
|
||||
)
|
||||
async def test_sources_that_cannot_be_read_refuse_the_start(tmp_path, sources, problem):
|
||||
engine, runtime = engine_for(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match=problem):
|
||||
await engine.start("A picnic", sources=sources)
|
||||
|
||||
assert runtime.runs == []
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
|
||||
|
||||
async def test_a_workflow_that_digests_nothing_passes_every_source_on_to_be_read(
|
||||
tmp_path,
|
||||
):
|
||||
engine, _ = engine_for(tmp_path, TOY)
|
||||
|
||||
result = await engine.start(
|
||||
"A picnic", pre_approved=True, sources={"notes.md": "Bring a blanket."}
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.state["scope"] is None
|
||||
assert result.state["source_documents"] == {"notes.md": "Bring a blanket."}
|
||||
416
tests/toy/test_execution.py
Normal file
416
tests/toy/test_execution.py
Normal file
@@ -0,0 +1,416 @@
|
||||
"""Whole Executions of the toy Workflow: the engine on a subject it was not built from."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call
|
||||
from tests.toy.support import (
|
||||
approve,
|
||||
fails,
|
||||
folder,
|
||||
make_engine,
|
||||
manifest,
|
||||
once_each,
|
||||
submits,
|
||||
)
|
||||
from tests.toy.workflow import (
|
||||
CHECK,
|
||||
CHECKING,
|
||||
MENU,
|
||||
MENU_FILE,
|
||||
PACKING,
|
||||
PLAN,
|
||||
PLANNING,
|
||||
SHOPPING_FILE,
|
||||
SHOPPING_LIST,
|
||||
)
|
||||
|
||||
THREE_DISHES = json.dumps(
|
||||
[
|
||||
{"id": "D-1", "name": "Sandwiches"},
|
||||
{"id": "D-2", "name": "Lemonade"},
|
||||
{"id": "D-3", "name": "Cake"},
|
||||
]
|
||||
)
|
||||
FOR_THREE = json.dumps(
|
||||
[
|
||||
{"dish": "D-1", "item": "bread", "quantity": 2},
|
||||
{"dish": "D-2", "item": "lemons", "quantity": 6},
|
||||
{"dish": "D-3", "item": "flour", "quantity": 1},
|
||||
]
|
||||
)
|
||||
THREE_VERDICTS = json.dumps(
|
||||
{
|
||||
"dry_run": {"verdict": "pass", "evidence": "Fits the basket."},
|
||||
"checks": [
|
||||
{"check": dish, "verdict": "pass", "evidence": "Enough."}
|
||||
for dish in ("D-1", "D-2", "D-3")
|
||||
],
|
||||
"proposed_additions": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def actions(result) -> list[str]:
|
||||
return [decision["action"] for decision in result.state["decisions"]]
|
||||
|
||||
|
||||
async def test_a_pre_approved_execution_runs_to_the_end_and_writes_its_files(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start(
|
||||
"A picnic for six", pre_approved=True, customer_case="park", execution_id="e1"
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"menu-writer",
|
||||
"shopper",
|
||||
"checker",
|
||||
"packer",
|
||||
]
|
||||
out = folder(tmp_path, "park", "e1")
|
||||
assert {path.name for path in out.iterdir()} == {
|
||||
"deliverables",
|
||||
"execution.json",
|
||||
"menu.json",
|
||||
"shopping.json",
|
||||
"verdicts.json",
|
||||
"picnic-plan.md",
|
||||
}
|
||||
record = manifest(tmp_path, "park", "e1")
|
||||
assert record["final_document"] == "picnic-plan.md"
|
||||
assert record["phases"] == {
|
||||
PLANNING: "approved",
|
||||
CHECKING: "approved",
|
||||
PACKING: "approved",
|
||||
}
|
||||
# The final Deliverable is also the file at the top of the folder.
|
||||
assert (out / "picnic-plan.md").read_text() == (
|
||||
out / "deliverables" / "plan.md"
|
||||
).read_text()
|
||||
# What the Workflow records of its own, and nothing of any other subject.
|
||||
assert record["dishes"] == ["D-1", "D-2"]
|
||||
assert not {"solution_guide", "terraform_modules", "delivery_plan"} & set(record)
|
||||
|
||||
|
||||
async def test_no_spec_is_needed_and_none_is_recorded(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True, execution_id="e1")
|
||||
|
||||
assert result.state["scope"] is None
|
||||
assert manifest(tmp_path, "unassigned", "e1")["scope"] is None
|
||||
assert all(run.task.context["scope"] is None for run in runtime.runs)
|
||||
|
||||
|
||||
async def test_code_adds_its_view_to_a_submission(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
shopping = result.state["deliverables"][SHOPPING_LIST]["content"]
|
||||
assert shopping.rstrip().endswith("## Totals\n\n3 items for 2 dishes.")
|
||||
|
||||
|
||||
async def test_each_specialist_is_given_what_it_works_from(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
by_role = {run.role: run.task.context for run in runtime.runs}
|
||||
assert by_role["menu-writer"]["dependencies"] == {}
|
||||
assert set(by_role["shopper"]["dependencies"]) == {MENU}
|
||||
assert by_role["shopper"]["contract"] == {
|
||||
"sections": ["What to buy"],
|
||||
"attachments": [SHOPPING_FILE],
|
||||
}
|
||||
# The check works from the list; the menu arrives as its data file alone.
|
||||
assert set(by_role["checker"]["dependencies"]) == {MENU, SHOPPING_LIST}
|
||||
assert by_role["checker"]["dependencies"][MENU] == {
|
||||
"attachments": {MENU_FILE: runtime.runs[0].outcome.attachments[MENU_FILE]}
|
||||
}
|
||||
# A section code wrote for a reader is not shown to the next Specialist, which
|
||||
# has the same facts as data. The final Deliverable assembles it, so it is.
|
||||
assert (
|
||||
"## Totals" not in by_role["checker"]["dependencies"][SHOPPING_LIST]["content"]
|
||||
)
|
||||
assert set(by_role["packer"]["dependencies"]) == {MENU, SHOPPING_LIST, CHECK}
|
||||
assert "## Totals" in by_role["packer"]["dependencies"][SHOPPING_LIST]["content"]
|
||||
|
||||
|
||||
async def test_a_submission_that_breaks_the_chain_is_refused(tmp_path):
|
||||
turns = once_each()
|
||||
nothing_for_lemonade = json.dumps([{"dish": "D-1", "item": "bread"}])
|
||||
turns["shopper"] = [
|
||||
[
|
||||
*submits(SHOPPING_LIST, **{SHOPPING_FILE: nothing_for_lemonade}),
|
||||
*submits(SHOPPING_LIST),
|
||||
]
|
||||
]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
refused, accepted = runtime.runs[1].results
|
||||
assert "nothing is bought for dish D-2" in refused[1]
|
||||
assert accepted[1] == "Submitted shopping_list."
|
||||
|
||||
|
||||
async def test_with_a_user_each_phase_waits_for_its_approval(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
first = await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
assert first.status == "awaiting_user"
|
||||
assert first.prompt["approval"]["phase"] == PLANNING
|
||||
assert first.prompt["approval"]["deliverables"] == [MENU, SHOPPING_LIST]
|
||||
# The summary is assembled from the Workflow's own facts.
|
||||
assert "2 dishes on the menu." in first.prompt["approval"]["summary"]
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper"]
|
||||
|
||||
second = await engine.resume("e1", approve(PLANNING))
|
||||
|
||||
assert second.prompt["approval"]["phase"] == CHECKING
|
||||
assert (
|
||||
"Validation: 2 Checks passed, 0 failed" in second.prompt["approval"]["summary"]
|
||||
)
|
||||
|
||||
third = await engine.resume("e1", approve(CHECKING))
|
||||
|
||||
assert third.prompt["approval"]["deliverables"] == [PLAN]
|
||||
|
||||
done = await engine.resume("e1", approve(PACKING))
|
||||
|
||||
assert done.status == "completed"
|
||||
|
||||
|
||||
async def test_a_specialists_question_reaches_the_user_and_the_answer_comes_back(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [
|
||||
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
||||
lambda task: (
|
||||
submits(MENU) if task.context["answers"][-1]["reply"] == "None" else []
|
||||
),
|
||||
]
|
||||
engine, _ = make_engine(tmp_path, turns)
|
||||
|
||||
asked = await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
assert asked.prompt == {"questions": [{"from": MENU, "question": "Any allergies?"}]}
|
||||
|
||||
answered = await engine.resume("e1", {"reply": "None"})
|
||||
|
||||
assert answered.prompt["approval"]["phase"] == PLANNING
|
||||
|
||||
|
||||
async def test_a_failed_check_sends_the_list_back_and_the_menu_stands(tmp_path):
|
||||
turns = once_each()
|
||||
turns["shopper"] = [
|
||||
submits(SHOPPING_LIST),
|
||||
lambda task: submits(SHOPPING_LIST) if task.context["feedback"] else [],
|
||||
]
|
||||
turns["checker"] = [fails(D_2="fail"), submits(CHECK)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
# The Workflow says what a failed Check reopens: the list, not the menu.
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"menu-writer",
|
||||
"shopper",
|
||||
"checker",
|
||||
"shopper",
|
||||
"checker",
|
||||
"packer",
|
||||
]
|
||||
assert "revise_code" in actions(result)
|
||||
assert runtime.runs[3].task.context["feedback"] == ["D-2 failed: D-2 is short."]
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert ledger[SHOPPING_LIST]["revisions"] == 1
|
||||
assert ledger[MENU]["revisions"] == 0
|
||||
|
||||
|
||||
async def test_a_check_that_fails_twice_running_sends_the_menu_back(tmp_path):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [submits(MENU), submits(MENU)]
|
||||
turns["shopper"] = [submits(SHOPPING_LIST)] * 3
|
||||
turns["checker"] = [fails(D_2="fail"), fails(D_2="fail"), submits(CHECK)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "revise_target_architecture" in actions(result)
|
||||
assert result.state["ledger"]["deliverables"][MENU]["revisions"] == 1
|
||||
assert [run.role for run in runtime.runs].count("menu-writer") == 2
|
||||
|
||||
|
||||
async def test_a_user_who_asks_for_changes_at_an_approval_gets_a_revision(tmp_path):
|
||||
turns = once_each()
|
||||
turns["shopper"] = [submits(SHOPPING_LIST), submits(SHOPPING_LIST)]
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
await engine.start("A picnic", execution_id="e1")
|
||||
|
||||
again = await engine.resume(
|
||||
"e1",
|
||||
{
|
||||
"approval": {
|
||||
"phase": PLANNING,
|
||||
"decision": "changes",
|
||||
"feedback": "More lemons on the shopping list, please.",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
# Named outright, so it is revised with no decision model to say which.
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer", "shopper", "shopper"]
|
||||
assert "More lemons" in runtime.runs[2].task.context["feedback"][0]
|
||||
assert again.prompt["approval"]["phase"] == PLANNING
|
||||
|
||||
|
||||
async def test_a_change_request_starts_from_its_baseline_and_says_what_changed(
|
||||
tmp_path,
|
||||
):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
await engine.start(
|
||||
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
|
||||
)
|
||||
turns = {
|
||||
"menu-writer": [
|
||||
lambda task: (
|
||||
submits(MENU, **{MENU_FILE: THREE_DISHES})
|
||||
if task.context["baseline"]["deliverable"]
|
||||
else []
|
||||
)
|
||||
],
|
||||
"shopper": [submits(SHOPPING_LIST, **{SHOPPING_FILE: FOR_THREE})],
|
||||
"checker": [submits(CHECK, **{"verdicts.json": THREE_VERDICTS})],
|
||||
# The final Deliverable is written afresh; the Baseline's is not handed back.
|
||||
"packer": [
|
||||
lambda task: (
|
||||
submits(PLAN) if task.context["baseline"]["deliverable"] is None else []
|
||||
)
|
||||
],
|
||||
}
|
||||
changed, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await changed.start(
|
||||
"Add a cake.",
|
||||
pre_approved=True,
|
||||
customer_case="park",
|
||||
execution_id="second",
|
||||
baseline="first",
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
out = folder(tmp_path, "park", "second")
|
||||
assert json.loads((out / "changes.json").read_text()) == {
|
||||
"added": ["D-3"],
|
||||
"removed": [],
|
||||
}
|
||||
assert manifest(tmp_path, "park", "second")["baseline"] == "first"
|
||||
# The shopper is told what has changed upstream of its Deliverable, in data.
|
||||
assert runtime.runs[1].task.context["baseline"]["changes"] == {
|
||||
"added": ["D-3"],
|
||||
"removed": [],
|
||||
}
|
||||
# The Baseline is never modified.
|
||||
assert manifest(tmp_path, "park", "first")["dishes"] == ["D-1", "D-2"]
|
||||
|
||||
|
||||
async def test_a_baseline_from_another_customer_case_is_refused(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, once_each())
|
||||
await engine.start(
|
||||
"A picnic", pre_approved=True, customer_case="park", execution_id="first"
|
||||
)
|
||||
other, _ = make_engine(tmp_path, {})
|
||||
|
||||
with pytest.raises(ValueError, match="another Customer case"):
|
||||
await other.start(
|
||||
"Add a cake.", pre_approved=True, customer_case="beach", baseline="first"
|
||||
)
|
||||
|
||||
|
||||
async def test_an_option_takes_its_default_the_callers_value_or_its_baselines(
|
||||
tmp_path,
|
||||
):
|
||||
engine_turns = once_each()
|
||||
for role in engine_turns:
|
||||
engine_turns[role] = engine_turns[role] * 3
|
||||
engine, runtime = make_engine(tmp_path, engine_turns)
|
||||
|
||||
default = await engine.start("A picnic", pre_approved=True, customer_case="park")
|
||||
chosen = await engine.start(
|
||||
"A picnic",
|
||||
pre_approved=True,
|
||||
customer_case="park",
|
||||
execution_id="six",
|
||||
guests=6,
|
||||
)
|
||||
followed = await engine.start(
|
||||
"The same again.", pre_approved=True, customer_case="park", baseline="six"
|
||||
)
|
||||
|
||||
assert default.state["options"] == {"guests": 4}
|
||||
assert chosen.state["options"] == {"guests": 6}
|
||||
assert followed.state["options"] == {"guests": 6}
|
||||
assert manifest(tmp_path, "park", "six")["options"] == {"guests": 6}
|
||||
# Every Specialist is told, through the Workflow's own entry in its task.
|
||||
assert {run.task.context["guests"] for run in runtime.runs[4:8]} == {6}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("options", "message"),
|
||||
[
|
||||
({"guests": 0}, "guests must be a whole number"),
|
||||
({"guests": "many"}, "guests must be a whole number"),
|
||||
({"terraform_modules": True}, "Unknown options: terraform_modules"),
|
||||
],
|
||||
)
|
||||
async def test_an_option_the_workflow_does_not_allow_refuses_the_start(
|
||||
tmp_path, options, message
|
||||
):
|
||||
engine, runtime = make_engine(tmp_path, once_each())
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
await engine.start("A picnic", pre_approved=True, **options)
|
||||
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_with_nobody_to_answer_a_specialist_that_cannot_finish_is_stopped(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once_each()
|
||||
asks = [Call("needs_input", {"questions": ["Which park?"]})]
|
||||
turns["menu-writer"] = [asks] * 10
|
||||
engine, runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert [run.role for run in runtime.runs] == ["menu-writer"] * 3
|
||||
assert result.state["warnings"] == [
|
||||
(
|
||||
"Stopped: menu-writer ended 3 runs running without submitting menu, with "
|
||||
"nobody to answer it."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_a_specialist_that_finishes_on_its_last_chance_is_not_stopped(tmp_path):
|
||||
turns = once_each()
|
||||
asks = [Call("needs_input", {"questions": ["Which park?"]})]
|
||||
turns["menu-writer"] = [asks, asks, submits(MENU)]
|
||||
engine, _runtime = make_engine(tmp_path, turns)
|
||||
|
||||
result = await engine.start("A picnic", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.state["unfinished_runs"] == {}
|
||||
128
tests/toy/test_roles.py
Normal file
128
tests/toy/test_roles.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""Role files, loaded for a Workflow and a provider bundle that are not the Reference's."""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.adk.roles import RoleConfigError, load_roles
|
||||
from execution_engine.providers import load_provider
|
||||
from tests.toy.workflow import ROOT, TOY
|
||||
|
||||
|
||||
def load(root: Path = ROOT, **options):
|
||||
return load_roles(root / "agents", load_provider("toy", root), TOY, **options)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def root(tmp_path: Path) -> Path:
|
||||
"""A copy of the toy agent's folder, to break."""
|
||||
shutil.copytree(ROOT, tmp_path / "root")
|
||||
return tmp_path / "root"
|
||||
|
||||
|
||||
def test_the_bundle_needs_no_output_template_and_no_scope():
|
||||
bundle = load_provider("toy", ROOT)
|
||||
|
||||
assert bundle.output_template is None
|
||||
assert bundle.scope_skill is None
|
||||
assert set(bundle.skills) == {"picnic-planning"}
|
||||
|
||||
|
||||
def test_a_workflows_roles_are_the_engines_two_and_one_per_deliverable():
|
||||
roles = load()
|
||||
|
||||
assert set(roles) == {
|
||||
"orchestrator",
|
||||
"reply-writer",
|
||||
"menu-writer",
|
||||
"shopper",
|
||||
"checker",
|
||||
"packer",
|
||||
}
|
||||
|
||||
|
||||
def test_a_role_takes_its_skills_model_and_timeout_from_the_bundle():
|
||||
roles = load(specialist_budget=7, orchestrator_budget=33)
|
||||
|
||||
assert roles["shopper"].skills == ("picnic-planning",)
|
||||
assert roles["packer"].skills == ()
|
||||
assert roles["shopper"].model == "scripted/none"
|
||||
assert roles["checker"].model == "scripted/careful"
|
||||
assert roles["checker"].model_timeout_seconds == 240
|
||||
# A role with no budget of its own takes the setting.
|
||||
assert roles["shopper"].budget == 7
|
||||
assert roles["checker"].budget == 12
|
||||
assert roles["orchestrator"].budget == 33
|
||||
|
||||
|
||||
def test_a_role_file_names_its_tools_and_includes_its_partials():
|
||||
roles = load()
|
||||
|
||||
assert roles["shopper"].tools == ("tally",)
|
||||
assert roles["shopper"].instructions.startswith("You write the shopping list")
|
||||
assert "## How you work" in roles["shopper"].instructions
|
||||
assert "## How you work" not in roles["orchestrator"].instructions
|
||||
|
||||
|
||||
def test_a_missing_role_file_is_refused(root):
|
||||
(root / "agents" / "packer.md").unlink()
|
||||
|
||||
with pytest.raises(RoleConfigError, match="missing role files for packer"):
|
||||
load(root)
|
||||
|
||||
|
||||
def test_a_role_file_the_workflow_has_no_use_for_is_refused(root):
|
||||
(root / "agents" / "pillar-advisor.md").write_text(
|
||||
"---\nrole: pillar-advisor\n---\nAdvise.\n"
|
||||
)
|
||||
|
||||
with pytest.raises(RoleConfigError, match="unexpected role files pillar-advisor"):
|
||||
load(root)
|
||||
|
||||
|
||||
def test_a_role_the_workflow_asks_for_must_have_its_file(root):
|
||||
import dataclasses
|
||||
|
||||
wanting = dataclasses.replace(TOY, roles=("taster",), optional_roles=("driver",))
|
||||
bundle = load_provider("toy", root)
|
||||
|
||||
with pytest.raises(RoleConfigError, match="missing role files for taster"):
|
||||
load_roles(root / "agents", bundle, wanting)
|
||||
|
||||
for name in ("taster", "driver"):
|
||||
(root / "agents" / f"{name}.md").write_text(f"---\nrole: {name}\n---\nHelp.\n")
|
||||
|
||||
assert {"taster", "driver"} <= set(load_roles(root / "agents", bundle, wanting))
|
||||
|
||||
|
||||
def test_a_role_must_name_the_deliverable_its_workflow_gives_it(root):
|
||||
path = root / "agents" / "shopper.md"
|
||||
path.write_text(path.read_text().replace("shopping_list", "menu"))
|
||||
|
||||
with pytest.raises(RoleConfigError, match="'deliverable' must be 'shopping_list'"):
|
||||
load(root)
|
||||
|
||||
|
||||
def test_a_tool_the_bundle_does_not_offer_is_refused(root):
|
||||
path = root / "agents" / "shopper.md"
|
||||
path.write_text(path.read_text().replace("[tally]", "[terraform_validate]"))
|
||||
|
||||
with pytest.raises(RoleConfigError, match="terraform_validate are not offered"):
|
||||
load(root)
|
||||
|
||||
|
||||
def test_a_workflow_may_change_the_loaded_roles_from_the_bundle():
|
||||
import dataclasses
|
||||
|
||||
def one_more_call(roles, bundle):
|
||||
return {
|
||||
name: dataclasses.replace(role, budget=role.budget + 1)
|
||||
for name, role in roles.items()
|
||||
}
|
||||
|
||||
adjusted = dataclasses.replace(TOY, adjust_roles=one_more_call)
|
||||
|
||||
roles = load_roles(ROOT / "agents", load_provider("toy", ROOT), adjusted)
|
||||
|
||||
assert roles["checker"].budget == 13
|
||||
374
tests/toy/test_server.py
Normal file
374
tests/toy/test_server.py
Normal file
@@ -0,0 +1,374 @@
|
||||
"""An agent's server on the toy Workflow: the card, A2A tasks and the one-shot route."""
|
||||
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.roles import load_roles
|
||||
from execution_engine.agent import Assembly
|
||||
from execution_engine.config import Settings
|
||||
from execution_engine.main import create_app
|
||||
from execution_engine.providers import load_provider
|
||||
from tests.toy.support import make_engine, once_each
|
||||
from tests.toy.workflow import PLANNING, ROOT, TOY
|
||||
|
||||
SETTINGS = Settings( # type: ignore[call-arg]
|
||||
_env_file=None, CLOUD_PROVIDER="toy", AGENT_NAME="picnic-agent"
|
||||
)
|
||||
|
||||
|
||||
def client_for(
|
||||
tmp_path: Path, turns: dict[str, list[Any]], **overrides: Any
|
||||
) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
bundle = load_provider("toy", ROOT)
|
||||
roles = load_roles(ROOT / "agents", bundle, TOY)
|
||||
engine, runtime = make_engine(tmp_path, turns, roles=roles, **overrides)
|
||||
|
||||
@asynccontextmanager
|
||||
async def assembly(settings):
|
||||
yield Assembly(engine, bundle, roles, {"larder": "not reachable"})
|
||||
|
||||
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
|
||||
|
||||
|
||||
def send(
|
||||
client: TestClient,
|
||||
parts: list[dict],
|
||||
task: dict | None = None,
|
||||
metadata=None,
|
||||
path: str = "/a2a",
|
||||
):
|
||||
message = {"messageId": str(uuid.uuid4()), "role": "ROLE_USER", "parts": parts}
|
||||
if task:
|
||||
message.update(taskId=task["id"], contextId=task["contextId"])
|
||||
if metadata:
|
||||
message["metadata"] = metadata
|
||||
response = client.post(
|
||||
path,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "SendMessage",
|
||||
"params": {"message": message},
|
||||
},
|
||||
headers={"A2A-Version": "1.0"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert "error" not in body, body
|
||||
return body["result"]["task"]
|
||||
|
||||
|
||||
def status_parts(task: dict) -> tuple[str, dict | None]:
|
||||
parts = task["status"]["message"]["parts"]
|
||||
text = "\n".join(part["text"] for part in parts if "text" in part)
|
||||
data = next((part["data"] for part in parts if "data" in part), None)
|
||||
return text, data
|
||||
|
||||
|
||||
def test_the_card_describes_the_workflow_it_serves(tmp_path):
|
||||
client, _ = client_for(tmp_path, {})
|
||||
with client:
|
||||
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
||||
card = client.get(path).json()
|
||||
skill = card["skills"][0]
|
||||
assert card["name"] == "picnic-agent"
|
||||
assert card["description"].startswith("Plans a picnic with you")
|
||||
assert card["supportedInterfaces"][0]["url"].endswith("/a2a")
|
||||
assert (skill["id"], skill["name"]) == ("picnic", "Picnic planning")
|
||||
assert "planning: menu, shopping_list" in skill["description"]
|
||||
assert "packing: plan" in skill["description"]
|
||||
assert skill["tags"] == ["picnic", "toy"]
|
||||
assert "architecture" not in str(card).lower()
|
||||
|
||||
|
||||
def test_the_card_declares_the_fields_a_platform_step_sends(tmp_path):
|
||||
# KAB's workflow builder draws a step's inputs from skills[0].inputSchema.
|
||||
client, _ = client_for(tmp_path, {})
|
||||
with client:
|
||||
for path in ("/.well-known/agent-card.json", "/.well-known/agent.json"):
|
||||
skill = client.get(path).json()["skills"][0]
|
||||
schema = skill["inputSchema"]
|
||||
assert skill["inputArrives"] == "as_fields"
|
||||
assert set(schema["properties"]) == {
|
||||
"request",
|
||||
"customer_case",
|
||||
"scope",
|
||||
"pre_approved",
|
||||
}
|
||||
assert schema["properties"]["scope"]["uiFieldType"] == "textarea"
|
||||
assert schema["properties"]["pre_approved"]["type"] == "boolean"
|
||||
assert schema["required"] == ["customer_case"]
|
||||
|
||||
|
||||
def test_a_request_in_the_metadata_is_the_executions_request(tmp_path):
|
||||
# A platform step sends every input as metadata and, as the message text,
|
||||
# all of them again as one JSON document. The request is the field.
|
||||
client, _ = client_for(tmp_path, once_each())
|
||||
with client:
|
||||
task = send(
|
||||
client,
|
||||
[{"text": '{"request": "A picnic for nine", "pre_approved": true}'}],
|
||||
metadata={
|
||||
"request": "A picnic for nine",
|
||||
"pre_approved": True,
|
||||
"customer_case": "park",
|
||||
},
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
|
||||
record = (out / "execution.json").read_text()
|
||||
assert "A picnic for nine" in record
|
||||
assert '\\"pre_approved\\"' not in record
|
||||
|
||||
|
||||
def test_a_caller_that_posts_to_the_agents_own_address_is_served(tmp_path):
|
||||
# The KAB platform calls the URL an agent was registered with, not the
|
||||
# interface its card names, so the same handler answers at the root.
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [
|
||||
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
||||
*turns["menu-writer"],
|
||||
]
|
||||
client, _ = client_for(tmp_path, turns)
|
||||
with client:
|
||||
task = send(client, [{"text": "A picnic for six"}], path="/")
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
text, _ = status_parts(task)
|
||||
assert "Any allergies?" in text
|
||||
|
||||
|
||||
def test_health_names_the_provider_and_what_could_not_be_reached(tmp_path):
|
||||
client, _ = client_for(tmp_path, {})
|
||||
with client:
|
||||
health = client.get("/health").json()
|
||||
|
||||
assert health["status"] == "healthy"
|
||||
assert health["agent"] == "picnic-agent"
|
||||
assert health["cloud_provider"] == "toy"
|
||||
assert health["mcp_discovery_gaps"] == {"larder": "not reachable"}
|
||||
|
||||
|
||||
def test_a_question_pauses_the_task_and_a_text_reply_resumes_it(tmp_path):
|
||||
turns = once_each()
|
||||
turns["menu-writer"] = [
|
||||
[Call("needs_input", {"questions": ["Any allergies?"]})],
|
||||
*turns["menu-writer"],
|
||||
]
|
||||
client, runtime = client_for(tmp_path, turns)
|
||||
with client:
|
||||
task = send(client, [{"text": "A picnic for six"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
text, data = status_parts(task)
|
||||
assert "Any allergies?" in text
|
||||
assert data == {
|
||||
"prompt": {"questions": [{"from": "menu", "question": "Any allergies?"}]}
|
||||
}
|
||||
|
||||
task = send(client, [{"text": "No nuts"}], task)
|
||||
|
||||
_, data = status_parts(task)
|
||||
assert data["prompt"]["approval"]["phase"] == PLANNING
|
||||
assert runtime.runs[1].task.context["answers"][-1]["reply"] == "No nuts"
|
||||
|
||||
|
||||
def test_an_approval_counts_only_as_structured_data(tmp_path):
|
||||
client, _ = client_for(tmp_path, once_each())
|
||||
with client:
|
||||
task = send(client, [{"text": "A picnic"}])
|
||||
text, data = status_parts(task)
|
||||
assert data["prompt"]["approval"]["phase"] == PLANNING
|
||||
assert "Please review the planning Phase" in text
|
||||
assert '"decision": "approve"' in text
|
||||
|
||||
task = send(client, [{"text": "approve"}], task)
|
||||
_, data = status_parts(task)
|
||||
assert data["prompt"]["approval"]["phase"] == PLANNING
|
||||
|
||||
approval = {"approval": {"phase": PLANNING, "decision": "approve"}}
|
||||
task = send(client, [{"data": approval}], task)
|
||||
_, data = status_parts(task)
|
||||
assert data["prompt"]["approval"]["phase"] == "checking"
|
||||
|
||||
|
||||
def test_a_pre_approved_task_completes_with_its_files_as_artefacts(tmp_path):
|
||||
client, _ = client_for(tmp_path, once_each())
|
||||
with client:
|
||||
task = send(
|
||||
client,
|
||||
[{"text": "A picnic"}],
|
||||
metadata={"pre_approved": True, "guests": 9, "customer_case": "park"},
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
text, data = status_parts(task)
|
||||
assert text == "The picnic is planned."
|
||||
assert data == {"status": "completed", "warnings": []}
|
||||
names = {artifact["name"] for artifact in task["artifacts"]}
|
||||
assert names == {"picnic-plan.md", "menu.json", "shopping.json", "verdicts.json"}
|
||||
# An option the Workflow has is taken from the message's metadata.
|
||||
out = next((tmp_path / "deliverables" / "cases" / "park" / "executions").iterdir())
|
||||
assert '"guests": 9' in (out / "execution.json").read_text()
|
||||
|
||||
|
||||
def test_an_engine_failure_fails_the_task(tmp_path):
|
||||
# An Orchestrator on a model, with nothing scripted for it to do.
|
||||
client, _ = client_for(tmp_path, {}, orchestrator=None)
|
||||
with client:
|
||||
task = send(client, [{"text": "A picnic"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_FAILED"
|
||||
assert "ScriptExhausted" in status_parts(task)[0]
|
||||
|
||||
|
||||
def test_generate_runs_pre_approved_and_replies_with_the_deliverables(tmp_path):
|
||||
client, _ = client_for(tmp_path, once_each())
|
||||
with client:
|
||||
response = client.post("/generate", json={"request": "A picnic", "guests": 2})
|
||||
|
||||
body = response.json()
|
||||
assert response.status_code == 200
|
||||
assert body["status"] == "completed"
|
||||
assert set(body["deliverables"]) == {"menu", "shopping_list", "check", "plan"}
|
||||
assert body["warnings"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "error"),
|
||||
[
|
||||
({}, "'request' or a 'scope' spec is required"),
|
||||
({"request": "A picnic", "guests": 0}, "guests must be a whole number"),
|
||||
(
|
||||
{"request": "A picnic", "customer_case": "Not A Slug"},
|
||||
"customer_case must be",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_generate_refuses_what_cannot_start(tmp_path, payload, error):
|
||||
client, runtime = client_for(tmp_path, once_each())
|
||||
with client:
|
||||
response = client.post("/generate", json=payload)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert error in response.json()["error"]
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
def test_an_app_needs_the_agents_folder_or_a_way_to_assemble_it():
|
||||
with pytest.raises(ValueError, match="needs the agent's root"):
|
||||
create_app(TOY, SETTINGS)
|
||||
|
||||
|
||||
# --- messages that start nothing ---------------------------------------------------------
|
||||
|
||||
|
||||
def client_with_intake(tmp_path, decider, turns=None):
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.nodes.intake import DeliveredAnswers, Intake
|
||||
|
||||
bundle = load_provider("toy", ROOT)
|
||||
roles = load_roles(ROOT / "agents", bundle, TOY)
|
||||
engine, runtime = make_engine(tmp_path, turns or once_each(), roles=roles)
|
||||
|
||||
@asynccontextmanager
|
||||
async def assembly(settings):
|
||||
yield Assembly(
|
||||
engine,
|
||||
bundle,
|
||||
roles,
|
||||
{},
|
||||
intake=Intake(decider, tmp_path),
|
||||
answers=DeliveredAnswers(
|
||||
runtime,
|
||||
RoleSpec("reply-writer", "Write.", budget=3),
|
||||
tmp_path,
|
||||
TOY.final,
|
||||
),
|
||||
)
|
||||
|
||||
return TestClient(create_app(TOY, SETTINGS, assembly)), runtime
|
||||
|
||||
|
||||
def test_a_question_about_the_agent_is_answered_in_the_workflows_own_words(tmp_path):
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer
|
||||
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.05, 0.9)}
|
||||
)
|
||||
client, runtime = client_with_intake(tmp_path, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "What can you do?"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert status_parts(task)[0] == TOY.about
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
def test_a_question_about_delivered_work_is_answered_from_its_final_document(tmp_path):
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
def writer(task):
|
||||
seen.update(task.context)
|
||||
return [Call("write_reply", {"text": "Lemonade, because it is summer."})]
|
||||
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer("question", 0.95), "about_delivered": Answer(0.95, 0.9)}
|
||||
)
|
||||
turns = {**once_each(), "reply-writer": [writer]}
|
||||
client, _ = client_with_intake(tmp_path, decider, turns)
|
||||
with client:
|
||||
send(
|
||||
client,
|
||||
[{"text": "A picnic"}],
|
||||
metadata={"pre_approved": True, "customer_case": "park"},
|
||||
)
|
||||
task = send(
|
||||
client, [{"text": "Why lemonade?"}], metadata={"customer_case": "park"}
|
||||
)
|
||||
|
||||
assert status_parts(task)[0] == "Lemonade, because it is summer."
|
||||
# Written from the plan, the Workflow's final Deliverable.
|
||||
assert seen["delivered"]["final_document"].startswith("# plan")
|
||||
|
||||
|
||||
def test_a_workflow_built_from_the_settings_and_bundle_is_the_one_assembled(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
# An agent whose Workflow depends on its settings and provider bundle gives a
|
||||
# function to build it; the card still describes the Workflow it was given.
|
||||
import execution_engine.main as server
|
||||
|
||||
seen: dict[str, Any] = {}
|
||||
bundle = load_provider("toy", ROOT)
|
||||
roles = load_roles(ROOT / "agents", bundle, TOY)
|
||||
engine, _ = make_engine(tmp_path, {}, roles=roles)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_open_assembly(settings, workflow, *, root):
|
||||
seen.update(workflow=workflow, root=root)
|
||||
yield Assembly(engine, bundle, roles, {})
|
||||
|
||||
def build(settings, bundle):
|
||||
return TOY
|
||||
|
||||
monkeypatch.setattr(server, "open_assembly", fake_open_assembly)
|
||||
app = server.create_app(TOY, SETTINGS, root=ROOT, build=build)
|
||||
with TestClient(app) as client:
|
||||
assert client.get("/health").json()["status"] == "healthy"
|
||||
assert (
|
||||
client.get("/.well-known/agent-card.json").json()["name"] == "picnic-agent"
|
||||
)
|
||||
|
||||
assert seen == {"workflow": build, "root": ROOT}
|
||||
155
tests/toy/test_workflow.py
Normal file
155
tests/toy/test_workflow.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""A Workflow is checked when it is built, and holds its names as plain strings."""
|
||||
|
||||
import dataclasses
|
||||
from enum import StrEnum
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.workflows.deliverables import DeliverableSpec
|
||||
from execution_engine.workflows.workflow import (
|
||||
ValidationPolicy,
|
||||
Workflow,
|
||||
WorkflowError,
|
||||
)
|
||||
from tests.toy.workflow import CHECK, MENU, PLAN, SHOPPING_LIST, TOY
|
||||
|
||||
|
||||
def spec(kind: str, phase: str, role: str, *depends_on: str) -> DeliverableSpec:
|
||||
return DeliverableSpec(kind, phase, role, depends_on=depends_on)
|
||||
|
||||
|
||||
def build(
|
||||
*specs: DeliverableSpec, phases=("one", "two"), final="b", **parts
|
||||
) -> Workflow:
|
||||
return Workflow(
|
||||
name="t",
|
||||
about="",
|
||||
phases=phases,
|
||||
deliverables={each.kind: each for each in specs},
|
||||
final=final,
|
||||
**parts,
|
||||
)
|
||||
|
||||
|
||||
A, B = spec("a", "one", "first"), spec("b", "two", "second", "a")
|
||||
|
||||
|
||||
def test_a_workflow_that_holds_together_is_accepted():
|
||||
workflow = build(A, B)
|
||||
|
||||
assert workflow.specialists() == {"first": "a", "second": "b"}
|
||||
assert workflow.inputs_of("b") == ("a",)
|
||||
assert workflow.dependents_of("a") == ("b",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("specs", "parts", "problem"),
|
||||
[
|
||||
((A, spec("b", "two", "second", "c")), {}, "b: names unknown Deliverable c"),
|
||||
((A, spec("b", "three", "second")), {}, "b: Phase three is not listed"),
|
||||
((A, spec("b", "one", "second")), {}, "Phase two has no Deliverable"),
|
||||
((A, spec("b", "two", "first")), {}, "b: first already produces a"),
|
||||
(
|
||||
(spec("a", "one", "first", "b"), spec("b", "two", "second")),
|
||||
{},
|
||||
"a: depends on b, in a later Phase",
|
||||
),
|
||||
((A, B), {"final": "z"}, "final Deliverable z is not declared"),
|
||||
(
|
||||
(A, B),
|
||||
{"data_checks": {"z": ("z.json", lambda *_: [])}},
|
||||
"unknown Deliverable z",
|
||||
),
|
||||
((A, B), {"spec_deliverable": "z"}, "unknown Deliverable z"),
|
||||
(
|
||||
(A, B),
|
||||
{"validation": ValidationPolicy("b", "v.json", "a", "z")},
|
||||
"unknown Deliverable z",
|
||||
),
|
||||
((A, B), {"phases": ("one", "two", "one")}, "a Phase is listed twice"),
|
||||
],
|
||||
)
|
||||
def test_a_workflow_that_does_not_hold_together_is_refused(specs, parts, problem):
|
||||
with pytest.raises(WorkflowError, match=problem):
|
||||
build(*specs, **parts)
|
||||
|
||||
|
||||
def test_a_spec_filed_under_another_kind_is_refused():
|
||||
with pytest.raises(WorkflowError, match="a: its spec is for x"):
|
||||
Workflow(
|
||||
name="t",
|
||||
about="",
|
||||
phases=("one",),
|
||||
deliverables={"a": spec("x", "one", "first")},
|
||||
final="a",
|
||||
)
|
||||
|
||||
|
||||
def test_names_written_as_an_enum_are_held_as_plain_strings():
|
||||
class Kind(StrEnum):
|
||||
A = "a"
|
||||
B = "b"
|
||||
|
||||
class Phase(StrEnum):
|
||||
ONE = "one"
|
||||
TWO = "two"
|
||||
|
||||
workflow = Workflow(
|
||||
name="t",
|
||||
about="",
|
||||
phases=tuple(Phase),
|
||||
deliverables={
|
||||
Kind.A: DeliverableSpec(Kind.A, Phase.ONE, "first"),
|
||||
Kind.B: DeliverableSpec(Kind.B, Phase.TWO, "second", depends_on=(Kind.A,)),
|
||||
},
|
||||
final=Kind.B,
|
||||
data_checks={Kind.A: ("a.json", lambda *_: [])},
|
||||
)
|
||||
|
||||
# They go into checkpointed state, which is plain data.
|
||||
names = [
|
||||
*workflow.phases,
|
||||
*workflow.deliverables,
|
||||
workflow.final,
|
||||
*workflow.data_checks,
|
||||
*workflow.deliverables["b"].depends_on,
|
||||
workflow.deliverables["b"].phase,
|
||||
]
|
||||
assert all(type(name) is str for name in names)
|
||||
assert workflow.kind(Kind.A) == "a"
|
||||
assert type(workflow.kind(Kind.A)) is str
|
||||
|
||||
|
||||
def test_a_name_that_is_not_the_workflows_is_refused():
|
||||
with pytest.raises(
|
||||
ValueError, match="'requirements' is not a Deliverable of picnic"
|
||||
):
|
||||
TOY.kind("requirements")
|
||||
with pytest.raises(ValueError, match="'scoping' is not a Phase of picnic"):
|
||||
TOY.phase("scoping")
|
||||
|
||||
|
||||
def test_the_final_deliverable_reads_everything_and_is_shown_what_readers_are():
|
||||
assert TOY.inputs_of(PLAN) == (MENU, SHOPPING_LIST, CHECK)
|
||||
given = {
|
||||
MENU: {"content": "# Menu\n", "attachments": {"menu.json": "[]"}},
|
||||
SHOPPING_LIST: {
|
||||
"content": "# List\n\n## Totals\n\n3 items.\n",
|
||||
"attachments": {"shopping.json": "[]"},
|
||||
},
|
||||
}
|
||||
|
||||
for_checker = TOY.shown(CHECK, TOY.dependencies_for(CHECK, given))
|
||||
|
||||
assert for_checker[SHOPPING_LIST]["content"] == "# List\n"
|
||||
# Not read in full: the data file alone.
|
||||
assert for_checker[MENU] == {"attachments": {"menu.json": "[]"}}
|
||||
|
||||
|
||||
def test_a_workflow_with_no_validation_never_reads_one():
|
||||
plain = dataclasses.replace(TOY, validation=None)
|
||||
|
||||
assert plain.validation is None
|
||||
assert plain.check_submission(MENU, "# Menu\n## Dishes\n", {"menu.json": "[]"}) == (
|
||||
"menu.json: lists no dishes",
|
||||
)
|
||||
301
tests/toy/workflow.py
Normal file
301
tests/toy/workflow.py
Normal file
@@ -0,0 +1,301 @@
|
||||
"""A toy Workflow: planning a picnic.
|
||||
|
||||
It is here to show that the engine needs nothing of the subject it was built from
|
||||
(ADR-0001). Four Deliverables in three Phases: a menu, a shopping list written from
|
||||
it, a check that every dish can be made from the list, and the plan that puts the
|
||||
three together. It has no spec, no graph
|
||||
and no cloud, and it uses each kind of part a Workflow can supply at least once:
|
||||
data checks along a chain, code that adds a view on submission, a validation that
|
||||
reopens the list or the menu, an account of what a Change request changed, an
|
||||
option a caller may set, a tool of its own, and its own facts at an Approval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.adk.runtime import ToolResult, ToolSpec
|
||||
from execution_engine.workflows.deliverables import DeliverableSpec, Upstream
|
||||
from execution_engine.workflows.workflow import (
|
||||
Assembling,
|
||||
Bench,
|
||||
CardText,
|
||||
ChangePolicy,
|
||||
LocalToolBuilder,
|
||||
OptionSpec,
|
||||
ValidationPolicy,
|
||||
Workflow,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).parent / "root" # the toy agent's own folder
|
||||
|
||||
PLANNING, CHECKING, PACKING = "planning", "checking", "packing"
|
||||
MENU, SHOPPING_LIST, CHECK, PLAN = "menu", "shopping_list", "check", "plan"
|
||||
MENU_FILE, SHOPPING_FILE, VERDICTS_FILE = "menu.json", "shopping.json", "verdicts.json"
|
||||
CHANGES_FILE = "changes.json"
|
||||
PLAN_FILE = "picnic-plan.md"
|
||||
TOTALS_HEADING = "Totals"
|
||||
GUESTS = "guests"
|
||||
|
||||
|
||||
def _list(text: str, name: str) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError as exc:
|
||||
return [], [f"{name}: not valid JSON ({exc})"]
|
||||
if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
|
||||
return [], [f"{name}: must be a JSON list of objects"]
|
||||
return data, []
|
||||
|
||||
|
||||
def dishes(deliverables: Upstream | None) -> list[dict[str, Any]]:
|
||||
"""The dishes on the menu, from its data file; none when it is not there."""
|
||||
text = ((deliverables or {}).get(MENU) or {}).get("attachments", {}).get(MENU_FILE)
|
||||
return _list(text, MENU_FILE)[0] if text else []
|
||||
|
||||
|
||||
def dish_ids(deliverables: Upstream | None) -> list[str]:
|
||||
return [str(dish.get("id")) for dish in dishes(deliverables)]
|
||||
|
||||
|
||||
def check_menu(attachments: Mapping[str, str], upstream: Upstream | None) -> list[str]:
|
||||
items, problems = _list(attachments[MENU_FILE], MENU_FILE)
|
||||
if problems:
|
||||
return problems
|
||||
if not items:
|
||||
return [f"{MENU_FILE}: lists no dishes"]
|
||||
ids = [item.get("id") for item in items]
|
||||
problems += [
|
||||
f"{MENU_FILE}[{index}]: needs id and name"
|
||||
for index, item in enumerate(items)
|
||||
if not item.get("id") or not item.get("name")
|
||||
]
|
||||
problems += [
|
||||
f"{MENU_FILE}: dish {dish} is listed more than once"
|
||||
for dish in sorted({str(i) for i in ids if i and ids.count(i) > 1})
|
||||
]
|
||||
return problems
|
||||
|
||||
|
||||
def check_shopping(
|
||||
attachments: Mapping[str, str], upstream: Upstream | None
|
||||
) -> list[str]:
|
||||
items, problems = _list(attachments[SHOPPING_FILE], SHOPPING_FILE)
|
||||
if problems:
|
||||
return problems
|
||||
on_menu = dish_ids(upstream)
|
||||
bought_for = {str(item.get("dish")) for item in items}
|
||||
problems += [
|
||||
f"{SHOPPING_FILE}: nothing is bought for dish {dish}"
|
||||
for dish in on_menu
|
||||
if dish not in bought_for
|
||||
]
|
||||
if upstream is not None:
|
||||
problems += [
|
||||
f"{SHOPPING_FILE}: dish {dish} is not on the menu"
|
||||
for dish in sorted(bought_for - set(on_menu))
|
||||
]
|
||||
return problems
|
||||
|
||||
|
||||
def check_verdicts(
|
||||
attachments: Mapping[str, str], upstream: Upstream | None
|
||||
) -> list[str]:
|
||||
try:
|
||||
data = json.loads(attachments[VERDICTS_FILE])
|
||||
except ValueError as exc:
|
||||
return [f"{VERDICTS_FILE}: not valid JSON ({exc})"]
|
||||
given = {
|
||||
str(item.get("check"))
|
||||
for item in (data.get("checks") or [] if isinstance(data, dict) else [])
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
return [
|
||||
f"{VERDICTS_FILE}: dish {dish} has no verdict"
|
||||
for dish in dish_ids(upstream)
|
||||
if dish not in given
|
||||
]
|
||||
|
||||
|
||||
def with_totals(
|
||||
bench: Bench, content: str, attachments: dict[str, str]
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
"""How much is bought, counted from the list: it cannot disagree with it."""
|
||||
items, _ = _list(attachments[SHOPPING_FILE], SHOPPING_FILE)
|
||||
content = content.split(f"\n## {TOTALS_HEADING}")[0].rstrip()
|
||||
content += (
|
||||
f"\n\n## {TOTALS_HEADING}\n\n{len(items)} items for "
|
||||
f"{len(dish_ids(bench.upstream))} dishes.\n"
|
||||
)
|
||||
return content, attachments
|
||||
|
||||
|
||||
def change_summary(before: Upstream | None, after: Upstream) -> dict[str, Any]:
|
||||
was, now = set(dish_ids(before)), set(dish_ids(after))
|
||||
return {"added": sorted(now - was), "removed": sorted(was - now)}
|
||||
|
||||
|
||||
def changes_in_words(summary: Mapping[str, Any]) -> str:
|
||||
return "\n".join(
|
||||
[f"Dish {dish} was added." for dish in summary["added"]]
|
||||
+ [f"Dish {dish} was removed." for dish in summary["removed"]]
|
||||
)
|
||||
|
||||
|
||||
def approval_facts(phase: str, deliverables: Upstream) -> dict[str, Any]:
|
||||
return {"dishes": len(dishes(deliverables))}
|
||||
|
||||
|
||||
def summary_lines(phase: str, facts: Mapping[str, Any]) -> list[str]:
|
||||
return [f"{facts['dishes']} dishes on the menu."]
|
||||
|
||||
|
||||
def task_entries(
|
||||
state: Mapping[str, Any], kind: str
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
# How many people the picnic is for: every Specialist is told.
|
||||
return {}, {GUESTS: (state.get("options") or {}).get(GUESTS)}
|
||||
|
||||
|
||||
def outputs(
|
||||
state: Mapping[str, Any], deliverables: Upstream, files: dict[str, str]
|
||||
) -> dict[str, Any]:
|
||||
return {"dishes": dish_ids(deliverables)}
|
||||
|
||||
|
||||
def _guests(value: Any) -> str | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
return "guests must be a whole number of people, at least 1"
|
||||
return None
|
||||
|
||||
|
||||
def local_tools(given: Assembling) -> LocalToolBuilder:
|
||||
"""`tally`: how many items the working copy of the shopping list holds."""
|
||||
|
||||
def build(working_copy: MutableMapping[str, str] | None) -> Sequence[ToolSpec]:
|
||||
async def tally() -> ToolResult:
|
||||
text = (working_copy or {}).get(SHOPPING_FILE, "[]")
|
||||
return ToolResult(f"{len(_list(text, SHOPPING_FILE)[0])} items so far.")
|
||||
|
||||
return [
|
||||
ToolSpec(
|
||||
"tally",
|
||||
"Count the items on the shopping list so far.",
|
||||
{"type": "object", "properties": {}},
|
||||
tally,
|
||||
)
|
||||
]
|
||||
|
||||
return build
|
||||
|
||||
|
||||
SAMPLES = {
|
||||
MENU_FILE: json.dumps(
|
||||
[{"id": "D-1", "name": "Sandwiches"}, {"id": "D-2", "name": "Lemonade"}]
|
||||
),
|
||||
SHOPPING_FILE: json.dumps(
|
||||
[
|
||||
{"dish": "D-1", "item": "bread", "quantity": 2},
|
||||
{"dish": "D-1", "item": "cheese", "quantity": 1},
|
||||
{"dish": "D-2", "item": "lemons", "quantity": 6},
|
||||
]
|
||||
),
|
||||
VERDICTS_FILE: json.dumps(
|
||||
{
|
||||
"dry_run": {"verdict": "pass", "evidence": "Everything fits the basket."},
|
||||
"checks": [
|
||||
{"check": "D-1", "verdict": "pass", "evidence": "Bread and cheese."},
|
||||
{"check": "D-2", "verdict": "pass", "evidence": "Six lemons."},
|
||||
],
|
||||
"proposed_additions": [],
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
TOY = Workflow(
|
||||
name="picnic",
|
||||
about="I plan picnics: a menu, a shopping list and a check of one against the other.",
|
||||
phases=(PLANNING, CHECKING, PACKING),
|
||||
deliverables={
|
||||
spec.kind: spec
|
||||
for spec in (
|
||||
DeliverableSpec(
|
||||
MENU,
|
||||
PLANNING,
|
||||
"menu-writer",
|
||||
sections=("Dishes",),
|
||||
attachments=(MENU_FILE,),
|
||||
),
|
||||
DeliverableSpec(
|
||||
SHOPPING_LIST,
|
||||
PLANNING,
|
||||
"shopper",
|
||||
depends_on=(MENU,),
|
||||
sections=("What to buy",),
|
||||
attachments=(SHOPPING_FILE,),
|
||||
# The list is built up in a working copy, a few lines at a time.
|
||||
working_copy=SHOPPING_LIST,
|
||||
),
|
||||
DeliverableSpec(
|
||||
CHECK,
|
||||
CHECKING,
|
||||
"checker",
|
||||
depends_on=(SHOPPING_LIST,),
|
||||
sections=("Findings",),
|
||||
attachments=(VERDICTS_FILE,),
|
||||
),
|
||||
DeliverableSpec(
|
||||
PLAN,
|
||||
PACKING,
|
||||
"packer",
|
||||
depends_on=(CHECK,),
|
||||
sections=("The plan",),
|
||||
# The plan is assembled from every Deliverable's prose.
|
||||
reads_in_full=(MENU, SHOPPING_LIST, CHECK),
|
||||
),
|
||||
)
|
||||
},
|
||||
final=PLAN,
|
||||
final_file=PLAN_FILE,
|
||||
data_checks={
|
||||
MENU: (MENU_FILE, check_menu),
|
||||
SHOPPING_LIST: (SHOPPING_FILE, check_shopping),
|
||||
CHECK: (VERDICTS_FILE, check_verdicts),
|
||||
},
|
||||
for_readers={SHOPPING_LIST: TOTALS_HEADING},
|
||||
validation=ValidationPolicy(
|
||||
kind=CHECK,
|
||||
data_file=VERDICTS_FILE,
|
||||
implementation=SHOPPING_LIST,
|
||||
design=MENU,
|
||||
check_ids=dish_ids,
|
||||
known_ids=lambda deliverables: set(dish_ids(deliverables)),
|
||||
),
|
||||
on_submit={SHOPPING_LIST: with_totals},
|
||||
approval_facts=approval_facts,
|
||||
summary_lines=summary_lines,
|
||||
agreed=lambda deliverables: json.dumps(dishes(deliverables)),
|
||||
changes=ChangePolicy(
|
||||
file=CHANGES_FILE,
|
||||
summary=change_summary,
|
||||
is_empty=lambda summary: not (summary["added"] or summary["removed"]),
|
||||
in_words=changes_in_words,
|
||||
),
|
||||
subject="a picnic plan",
|
||||
task_entries=task_entries,
|
||||
outputs=outputs,
|
||||
options={GUESTS: OptionSpec(default=4, check=_guests)},
|
||||
local_tools=local_tools,
|
||||
card=CardText(
|
||||
description="Plans a picnic with you: a menu, a shopping list and a check.",
|
||||
skill_id="picnic",
|
||||
skill_name="Picnic planning",
|
||||
tags=("picnic",),
|
||||
examples=("Plan a picnic for six in the park.",),
|
||||
),
|
||||
done="The picnic is planned.",
|
||||
samples=SAMPLES,
|
||||
)
|
||||
Reference in New Issue
Block a user