1083 lines
39 KiB
Python
1083 lines
39 KiB
Python
"""The Execution engine: LangGraph state, Specialist fan-out, interrupts and recording.
|
|
|
|
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
|
|
|
from execution_engine.adk.fake_runtime import Call
|
|
from execution_engine.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
sample_submission,
|
|
)
|
|
from tests.scoped import ScopedEngine
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime
|
|
|
|
K = DeliverableKind
|
|
P = PhaseId
|
|
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, f"Produce {spec.kind}.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
# --- scripting helpers ---------------------------------------------------------------
|
|
|
|
|
|
def delegate(*kinds: DeliverableKind) -> Call:
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
|
|
def request_approval(phase: PhaseId) -> Call:
|
|
return Call("request_approval", {"phase": phase.value, "summary": f"{phase} done"})
|
|
|
|
|
|
def ask_user(*questions: str) -> Call:
|
|
return Call("ask_user", {"questions": list(questions)})
|
|
|
|
|
|
def reopen(kind: DeliverableKind, reason: str) -> Call:
|
|
return Call("reopen", {"kind": kind.value, "reason": reason})
|
|
|
|
|
|
FINISH = Call("finish")
|
|
|
|
|
|
submission = sample_submission
|
|
|
|
|
|
def submits(kind: str) -> list[Call]:
|
|
return [submission(kind)]
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine:
|
|
options = {
|
|
"runtime": runtime,
|
|
"roles": ROLES,
|
|
"recorder": InMemoryRecorder(),
|
|
"checkpointer": InMemorySaver(),
|
|
"output_root": tmp_path,
|
|
}
|
|
options.update(overrides)
|
|
return ScopedEngine(**options)
|
|
|
|
|
|
def happy_path_turns(**extra_specialist_turns) -> dict:
|
|
turns = {
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
[
|
|
request_approval(P.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[
|
|
request_approval(P.SOLUTION_IMPLEMENTATION),
|
|
delegate(K.VALIDATION_RESULTS),
|
|
],
|
|
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
|
[request_approval(P.PACKAGING), FINISH],
|
|
],
|
|
**{spec.specialist: [submits(kind)] for kind, spec in DEPENDENCY_GRAPH.items()},
|
|
}
|
|
turns.update(extra_specialist_turns)
|
|
return turns
|
|
|
|
|
|
# --- happy path and parallel start -----------------------------------------------------
|
|
|
|
|
|
async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path):
|
|
runtime = FakeAgentRuntime(happy_path_turns())
|
|
recorder = InMemoryRecorder()
|
|
engine = make_engine(tmp_path, runtime, recorder=recorder)
|
|
|
|
result = await engine.start("Event-driven ingestion", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert result.prompt is None
|
|
ledger = result.state["ledger"]
|
|
assert set(ledger["phases"].values()) == {"approved"}
|
|
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
|
|
|
|
out = (
|
|
tmp_path
|
|
/ "deliverables"
|
|
/ "cases"
|
|
/ "unassigned"
|
|
/ "executions"
|
|
/ result.execution_id
|
|
)
|
|
assert (out / "solution-architecture-guide.md").is_file()
|
|
for kind in K:
|
|
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
|
|
assert (out / "terraform" / "main.tf").is_file()
|
|
assert (out / "architecture.mmd").is_file()
|
|
|
|
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
|
|
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
|
|
recorded_types = [e.type for e in recorder.events[result.execution_id]]
|
|
assert recorded_types.count("approved") == 6 # scoping, then the five after it
|
|
|
|
|
|
async def test_independent_specialists_run_together_between_orchestrator_turns(
|
|
tmp_path,
|
|
):
|
|
runtime = FakeAgentRuntime(happy_path_turns())
|
|
|
|
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
|
|
|
roles = [run.role for run in runtime.runs]
|
|
sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5]
|
|
assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == {
|
|
specialist(K.ARCHITECTURE_DESCRIPTION),
|
|
specialist(K.DEPLOYMENT_GUIDANCE),
|
|
}
|
|
assert roles[sixth_turn + 3] == "orchestrator"
|
|
|
|
|
|
async def test_specialist_receives_its_dependencies_content(tmp_path):
|
|
runtime = FakeAgentRuntime(happy_path_turns())
|
|
|
|
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
|
|
|
describer_run = next(
|
|
r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION)
|
|
)
|
|
dependencies = describer_run.task.context["dependencies"]
|
|
# The scope reaches every Specialist as `scope`, so it is not shown again here.
|
|
assert describer_run.task.context["scope"] is not None
|
|
assert list(dependencies) == [
|
|
"requirements",
|
|
"technical_decomposition",
|
|
"product_selection",
|
|
"design_recommendations",
|
|
"target_architecture",
|
|
]
|
|
# The engine drew the diagram from the graph when the Target architecture was submitted.
|
|
assert dependencies["target_architecture"]["attachments"][
|
|
"architecture.mmd"
|
|
].startswith("flowchart")
|
|
assert describer_run.task.deliverable == "architecture_description"
|
|
|
|
|
|
# --- refusals reach the agent ------------------------------------------------------------
|
|
|
|
|
|
async def test_orchestrator_cannot_delegate_work_whose_dependencies_are_unmet(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.PRODUCT_SELECTION), delegate(K.REQUIREMENTS)],
|
|
[ask_user("Anything else?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
}
|
|
)
|
|
|
|
result = await make_engine(tmp_path, runtime).start("Ingestion")
|
|
|
|
first_turn = runtime.runs[0]
|
|
assert first_turn.results[0][0] == "delegate"
|
|
assert "Refused" in first_turn.results[0][1]
|
|
assert "product_selection" in first_turn.results[0][1]
|
|
assert result.status == "awaiting_user"
|
|
|
|
|
|
async def test_incomplete_submission_is_returned_to_the_specialist(tmp_path):
|
|
incomplete = Call(
|
|
"submit_deliverable", {"content": "## Functional requirements\nx"}
|
|
)
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]],
|
|
specialist(K.REQUIREMENTS): [[incomplete, submission(K.REQUIREMENTS)]],
|
|
}
|
|
)
|
|
|
|
await make_engine(tmp_path, runtime).start("Ingestion")
|
|
|
|
analyst = runtime.runs[1]
|
|
assert "Non-functional requirements" in analyst.results[0][1]
|
|
assert analyst.results[1] == ("submit_deliverable", "Submitted requirements.")
|
|
|
|
|
|
# --- interactive approvals ----------------------------------------------------------------
|
|
|
|
|
|
async def test_approval_pauses_the_execution_until_the_user_decides(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
[delegate(K.PRODUCT_SELECTION)],
|
|
[ask_user("Pause here")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
|
|
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
|
|
paused = await engine.start("Ingestion")
|
|
|
|
assert paused.status == "awaiting_user"
|
|
assert paused.prompt["approval"]["phase"] == "requirements_discovery"
|
|
assert set(paused.prompt["approval"]["deliverables"]) == {
|
|
"requirements",
|
|
"technical_decomposition",
|
|
}
|
|
assert (
|
|
paused.state["ledger"]["phases"]["requirements_discovery"]
|
|
== "awaiting_approval"
|
|
)
|
|
|
|
resumed = await engine.resume(
|
|
paused.execution_id,
|
|
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
|
)
|
|
|
|
assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved"
|
|
assert (
|
|
resumed.state["ledger"]["deliverables"]["product_selection"]["status"]
|
|
== "submitted"
|
|
)
|
|
assert resumed.prompt == {
|
|
"questions": [{"from": "orchestrator", "question": "Pause here"}]
|
|
}
|
|
|
|
|
|
async def test_free_text_reply_never_grants_an_approval(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
paused = await engine.start("Ingestion")
|
|
|
|
again = await engine.resume(paused.execution_id, {"reply": "approve, looks good"})
|
|
|
|
assert (
|
|
again.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval"
|
|
)
|
|
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
|
assert again.state["conversation"][-2:] == [
|
|
{"from": "user", "text": "approve, looks good"},
|
|
{
|
|
"from": "orchestrator",
|
|
"text": "Please approve requirements_discovery: requirements_discovery done",
|
|
},
|
|
]
|
|
|
|
|
|
async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
[
|
|
reopen(K.REQUIREMENTS, "Add the RTO target"),
|
|
delegate(K.REQUIREMENTS),
|
|
],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
],
|
|
specialist(K.REQUIREMENTS): [
|
|
submits(K.REQUIREMENTS),
|
|
submits(K.REQUIREMENTS),
|
|
],
|
|
specialist(K.TECHNICAL_DECOMPOSITION): [
|
|
submits(K.TECHNICAL_DECOMPOSITION),
|
|
submits(K.TECHNICAL_DECOMPOSITION),
|
|
],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
paused = await engine.start("Ingestion")
|
|
|
|
second = await engine.resume(
|
|
paused.execution_id,
|
|
{
|
|
"approval": {
|
|
"phase": "requirements_discovery",
|
|
"decision": "changes",
|
|
"feedback": "Add the RTO target",
|
|
}
|
|
},
|
|
)
|
|
|
|
assert second.prompt["approval"]["phase"] == "requirements_discovery"
|
|
ledger = second.state["ledger"]
|
|
assert ledger["deliverables"]["requirements"]["revisions"] == 1
|
|
second_analyst_run = [
|
|
r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)
|
|
][1]
|
|
assert second_analyst_run.task.context["feedback"] == ["Add the RTO target"]
|
|
assert "changes_requested" in [e["type"] for e in ledger["events"]]
|
|
|
|
|
|
# --- one voice ---------------------------------------------------------------------------------
|
|
|
|
|
|
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
|
|
tmp_path,
|
|
):
|
|
def describer_turn(task):
|
|
if task.context["answers"]:
|
|
return submits(K.ARCHITECTURE_DESCRIPTION)
|
|
return [Call("needs_input", {"questions": ["Who reads this description?"]})]
|
|
|
|
def iac_turn(task):
|
|
if task.context["answers"]:
|
|
return submits(K.DEPLOYMENT_GUIDANCE)
|
|
return [Call("needs_input", {"questions": ["Which region?"]})]
|
|
|
|
both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
[delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
[request_approval(P.SOLUTION_ARCHITECTURE)],
|
|
[both],
|
|
[ask_user()],
|
|
[both],
|
|
[ask_user("Continue?")],
|
|
],
|
|
**{
|
|
specialist(kind): [submits(kind)]
|
|
for kind in (
|
|
K.REQUIREMENTS,
|
|
K.TECHNICAL_DECOMPOSITION,
|
|
K.PRODUCT_SELECTION,
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
K.TARGET_ARCHITECTURE,
|
|
)
|
|
},
|
|
specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn],
|
|
specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
approval = await engine.start("Ingestion")
|
|
design = await engine.resume(
|
|
approval.execution_id,
|
|
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
|
)
|
|
first = await engine.resume(
|
|
design.execution_id,
|
|
{"approval": {"phase": "solution_architecture", "decision": "approve"}},
|
|
)
|
|
|
|
assert first.prompt == {
|
|
"questions": [
|
|
{
|
|
"from": "architecture_description",
|
|
"question": "Who reads this description?",
|
|
},
|
|
{"from": "deployment_guidance", "question": "Which region?"},
|
|
]
|
|
}
|
|
|
|
second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"})
|
|
|
|
deliverables = second.state["ledger"]["deliverables"]
|
|
assert deliverables["architecture_description"]["status"] == "submitted"
|
|
assert deliverables["deployment_guidance"]["status"] == "submitted"
|
|
iac_retry = [
|
|
r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
|
][1]
|
|
assert iac_retry.task.context["answers"] == [
|
|
{"questions": ["Which region?"], "reply": "Engineers, in London"}
|
|
]
|
|
|
|
|
|
# --- revision after validation ------------------------------------------------------------
|
|
|
|
|
|
async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp_path):
|
|
turns = happy_path_turns()
|
|
turns["orchestrator"] = [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
[
|
|
request_approval(P.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[
|
|
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
|
|
delegate(K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
|
[request_approval(P.PACKAGING), FINISH],
|
|
]
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
|
submits(K.DEPLOYMENT_GUIDANCE),
|
|
submits(K.DEPLOYMENT_GUIDANCE),
|
|
]
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[submission(K.VALIDATION_RESULTS, "FAILED: missing IAM binding")],
|
|
submits(K.VALIDATION_RESULTS),
|
|
]
|
|
runtime = FakeAgentRuntime(turns)
|
|
|
|
result = await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
events = result.state["ledger"]["events"]
|
|
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
|
|
assert withdrawn == ["solution_implementation"]
|
|
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
|
|
assert requests.count("solution_implementation") == 2
|
|
assert requests.count("solution_architecture") == 1
|
|
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
|
|
assert iac_runs[1].task.context["feedback"] == [
|
|
"terraform plan: missing IAM binding"
|
|
]
|
|
|
|
|
|
async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[reopen(K.REQUIREMENTS, "not good enough")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
}
|
|
)
|
|
|
|
result = await make_engine(tmp_path, runtime, max_revisions=0).start(
|
|
"Ingestion", pre_approved=True
|
|
)
|
|
|
|
assert result.status == "completed_with_warnings"
|
|
assert "revision limit" in result.state["warnings"][0]
|
|
# What was produced is kept, marked as unfinished, and the request is on record.
|
|
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
|
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
|
assert intent["status"] == "completed_with_warnings"
|
|
assert intent["execution"] == f"executions/{result.execution_id}"
|
|
out_dir = case / intent["execution"]
|
|
manifest = json.loads((out_dir / "execution.json").read_text())
|
|
assert manifest["status"] == "completed_with_warnings"
|
|
assert "revision limit" in manifest["warnings"][0]
|
|
assert manifest["solution_guide"] is None
|
|
assert not (out_dir / "solution-architecture-guide.md").exists()
|
|
assert (out_dir / "deliverables" / "requirements.md").is_file()
|
|
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
|
|
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
|
|
assert manifest["phases"]["packaging"] != "approved"
|
|
|
|
|
|
# --- budgets -------------------------------------------------------------------------------------
|
|
|
|
|
|
async def test_orchestrator_budget_exhaustion_hands_control_to_the_user(tmp_path):
|
|
roles = {**ROLES, "orchestrator": RoleSpec("orchestrator", "", budget=2)}
|
|
wandering = [Call("ready_work"), Call("ready_work"), Call("ready_work")]
|
|
runtime = FakeAgentRuntime({"orchestrator": [wandering]})
|
|
|
|
result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion")
|
|
|
|
assert result.status == "awaiting_user"
|
|
question = result.prompt["questions"][0]
|
|
assert question["from"] == "orchestrator"
|
|
assert "2 tool calls" in question["question"]
|
|
|
|
|
|
async def test_specialist_budget_exhaustion_is_relayed_as_a_question(tmp_path):
|
|
roles = {
|
|
**ROLES,
|
|
specialist(K.REQUIREMENTS): RoleSpec("requirements-analyst", "", budget=1),
|
|
}
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]],
|
|
specialist(K.REQUIREMENTS): [
|
|
[
|
|
Call("submit_deliverable", {"content": "## Current state\nx"}),
|
|
submission(K.REQUIREMENTS),
|
|
]
|
|
],
|
|
}
|
|
)
|
|
|
|
result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion")
|
|
|
|
assert result.prompt["questions"][0]["from"] == "requirements"
|
|
assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
|
|
|
|
|
|
# --- cloud read permission -------------------------------------------------------------------
|
|
|
|
|
|
def cloud_read_factory(calls: list[str]):
|
|
async def list_projects() -> ToolResult:
|
|
calls.append("list_projects")
|
|
return ToolResult("projects: demo-project")
|
|
|
|
tool = ToolSpec(
|
|
"list_projects",
|
|
"List projects.",
|
|
{"type": "object", "properties": {}},
|
|
list_projects,
|
|
tier="cloud_read",
|
|
)
|
|
|
|
def factory(role: RoleSpec, deliverable: str | None, working: dict):
|
|
return [tool] if role.name == "requirements-analyst" else []
|
|
|
|
return factory
|
|
|
|
|
|
async def test_cloud_read_asks_permission_once_then_runs_the_tool(tmp_path):
|
|
calls: list[str] = []
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user("Next?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [
|
|
[Call("list_projects"), submission(K.REQUIREMENTS)],
|
|
[Call("list_projects"), submission(K.REQUIREMENTS)],
|
|
],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls))
|
|
|
|
asked = await engine.start("Ingestion")
|
|
|
|
assert asked.prompt == {
|
|
"permission": {"tier": "cloud_read", "requested_by": ["requirements"]}
|
|
}
|
|
assert calls == []
|
|
|
|
granted = await engine.resume(
|
|
asked.execution_id, {"permission": {"cloud_read": True}}
|
|
)
|
|
|
|
assert calls == ["list_projects"]
|
|
assert granted.state["allow_cloud_read"] is True
|
|
assert (
|
|
granted.state["ledger"]["deliverables"]["requirements"]["status"] == "submitted"
|
|
)
|
|
|
|
|
|
async def test_denied_cloud_read_leaves_the_agent_without_those_tools(tmp_path):
|
|
"""Their definitions were half of what the requirements analyst was sent, for
|
|
calls that could only be refused. It is told, and has nothing to call."""
|
|
calls: list[str] = []
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]],
|
|
specialist(K.REQUIREMENTS): [[submission(K.REQUIREMENTS)]],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls))
|
|
|
|
await engine.start("Ingestion", allow_cloud_read=False)
|
|
|
|
analyst = runtime.runs[1]
|
|
assert "list_projects" not in analyst.tools
|
|
assert analyst.task.context["cloud_read"] is False
|
|
assert calls == []
|
|
|
|
|
|
# --- isolation and durability -------------------------------------------------------------------
|
|
|
|
|
|
async def test_executions_in_one_session_share_no_state(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user("?")],
|
|
[ask_user("fresh?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
|
|
first = await engine.start("First", session_id="s-1")
|
|
second = await engine.start("Second", session_id="s-1")
|
|
|
|
assert first.execution_id != second.execution_id
|
|
assert second.state["deliverables"] == {}
|
|
assert second.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
|
|
assert second.state["workflow_request"] == "Second"
|
|
assert second.state["session_id"] == "s-1"
|
|
|
|
|
|
async def test_paused_execution_resumes_from_a_durable_checkpoint(tmp_path):
|
|
turns = {
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
|
[ask_user("Resumed")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
|
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
|
|
}
|
|
runtime = FakeAgentRuntime(turns)
|
|
db_path = str(tmp_path / "checkpoints.db")
|
|
|
|
async with AsyncSqliteSaver.from_conn_string(db_path) as saver:
|
|
paused = await make_engine(tmp_path, runtime, checkpointer=saver).start(
|
|
"Ingestion"
|
|
)
|
|
assert paused.prompt["approval"]["phase"] == "requirements_discovery"
|
|
|
|
async with AsyncSqliteSaver.from_conn_string(db_path) as saver:
|
|
resumed = await make_engine(tmp_path, runtime, checkpointer=saver).resume(
|
|
paused.execution_id,
|
|
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
|
)
|
|
|
|
assert resumed.prompt == {
|
|
"questions": [{"from": "orchestrator", "question": "Resumed"}]
|
|
}
|
|
assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved"
|
|
|
|
|
|
# --- identity and failures ------------------------------------------------------------------
|
|
|
|
|
|
async def test_caller_can_choose_the_execution_id_once(tmp_path):
|
|
runtime = FakeAgentRuntime({"orchestrator": [[ask_user("Hello?")]]})
|
|
engine = make_engine(tmp_path, runtime)
|
|
|
|
first = await engine.start("Ingestion", execution_id="task-123")
|
|
|
|
assert first.execution_id == "task-123"
|
|
with pytest.raises(ValueError, match="task-123"):
|
|
await engine.start("Again", execution_id="task-123")
|
|
|
|
|
|
async def test_a_failing_specialist_is_relayed_to_the_user_as_a_question(tmp_path):
|
|
class Failing(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
if role.name == specialist(K.REQUIREMENTS):
|
|
raise RuntimeError("model API unavailable")
|
|
return await super().run(role, task, tools)
|
|
|
|
runtime = Failing({"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]]})
|
|
|
|
result = await make_engine(tmp_path, runtime).start("Ingestion")
|
|
|
|
[question] = result.prompt["questions"]
|
|
assert question["from"] == "requirements"
|
|
assert "model API unavailable" in question["question"]
|
|
assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
|
|
|
|
|
|
async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[ask_user("What is the RTO?", "Which region?")],
|
|
[ask_user("Anything else?")],
|
|
]
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime, max_orchestrator_turns=3)
|
|
|
|
result = await engine.start(
|
|
"Ingestion",
|
|
pre_approved=True,
|
|
scripted_answers=[(r"\bRTO\b", "One hour."), ("region", "europe-west2")],
|
|
)
|
|
|
|
first_reply = result.state["conversation"][2]["text"]
|
|
assert "Q: What is the RTO?\nA: One hour." in first_reply
|
|
assert "Q: Which region?\nA: europe-west2" in first_reply
|
|
assert "No user is available" not in first_reply
|
|
second_reply = result.state["conversation"][4]["text"]
|
|
assert second_reply.startswith("No user is available")
|
|
|
|
|
|
# --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) -------
|
|
|
|
|
|
async def test_specialist_context_carries_its_contract(tmp_path):
|
|
runtime = FakeAgentRuntime(happy_path_turns())
|
|
|
|
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
|
|
|
run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS))
|
|
assert run.task.context["contract"] == {
|
|
"sections": ["Deployment dry-run", "Architecture and policy analysis"],
|
|
"attachments": ["verdicts.json"],
|
|
}
|
|
assert run.task.context["draft"] is None
|
|
|
|
|
|
async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path):
|
|
draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args
|
|
|
|
def confirm_then_submit_what_came_back(task):
|
|
returned = task.context["draft"]
|
|
return [Call("submit_deliverable", returned)]
|
|
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user()],
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user("Anything else?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [
|
|
[
|
|
Call(
|
|
"needs_input",
|
|
{
|
|
"questions": ["Do you approve this draft?"],
|
|
"draft": draft["content"],
|
|
"attachments": [
|
|
{"path": path, "content": content}
|
|
for path, content in draft["attachments"].items()
|
|
],
|
|
},
|
|
)
|
|
],
|
|
confirm_then_submit_what_came_back,
|
|
],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
|
|
paused = await engine.start("Ingestion")
|
|
assert paused.state["drafts"]["requirements"]["content"] == draft["content"]
|
|
|
|
after = await engine.resume(paused.execution_id, {"reply": "Approved."})
|
|
|
|
second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1]
|
|
assert second_run.task.context["draft"] == {
|
|
"content": draft["content"],
|
|
"attachments": draft["attachments"],
|
|
}
|
|
assert second_run.task.context["answers"][-1]["reply"] == "Approved."
|
|
assert after.state["deliverables"]["requirements"]["content"] == draft["content"]
|
|
assert after.state["drafts"] == {}
|
|
|
|
|
|
# --- cap on parallel Specialists (iteration 4, step 5) ---------------------------------
|
|
|
|
|
|
async def test_parallel_specialists_never_exceed_the_cap(tmp_path):
|
|
import asyncio
|
|
|
|
from execution_engine.adk.runtime import RunLimiter
|
|
|
|
in_flight = 0
|
|
peak = 0
|
|
|
|
class SlowRuntime(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
nonlocal in_flight, peak
|
|
if role.name == "orchestrator":
|
|
return await super().run(role, task, tools)
|
|
in_flight += 1
|
|
peak = max(peak, in_flight)
|
|
await asyncio.sleep(0.01)
|
|
try:
|
|
return await super().run(role, task, tools)
|
|
finally:
|
|
in_flight -= 1
|
|
|
|
runtime = SlowRuntime(happy_path_turns())
|
|
engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1))
|
|
|
|
result = await engine.start("Ingestion", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert peak == 1
|
|
|
|
|
|
async def test_without_a_cap_independent_specialists_overlap(tmp_path):
|
|
import asyncio
|
|
|
|
in_flight = 0
|
|
peak = 0
|
|
|
|
class SlowRuntime(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
nonlocal in_flight, peak
|
|
if role.name == "orchestrator":
|
|
return await super().run(role, task, tools)
|
|
in_flight += 1
|
|
peak = max(peak, in_flight)
|
|
await asyncio.sleep(0.01)
|
|
try:
|
|
return await super().run(role, task, tools)
|
|
finally:
|
|
in_flight -= 1
|
|
|
|
runtime = SlowRuntime(happy_path_turns())
|
|
|
|
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
|
|
|
assert peak == 2
|
|
|
|
|
|
# --- Customer cases group Executions ---------------------------------------------------
|
|
|
|
|
|
async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
|
recorder = InMemoryRecorder()
|
|
engine = make_engine(
|
|
tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder
|
|
)
|
|
|
|
result = await engine.start(
|
|
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
|
)
|
|
|
|
case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
|
out = case_dir / "executions" / result.execution_id
|
|
assert (out / "solution-architecture-guide.md").is_file()
|
|
assert result.state["customer_case"] == "acme-webhooks"
|
|
|
|
manifest = json.loads((out / "execution.json").read_text(encoding="utf-8"))
|
|
assert manifest["customer_case"] == "acme-webhooks"
|
|
assert manifest["workflow_request"] == "Ingestion"
|
|
assert manifest["intent_version"] == 1
|
|
assert manifest["solution_guide"] == "solution-architecture-guide.md"
|
|
assert manifest["deliverables"]["deployment_guidance"] == {
|
|
"document": "deliverables/deployment_guidance.md",
|
|
"attachments": [
|
|
"implementation.json",
|
|
"terraform/main.tf",
|
|
"terraform/variables.tf",
|
|
],
|
|
"status": "submitted",
|
|
}
|
|
assert manifest["status"] == "completed" and manifest["warnings"] == []
|
|
assert set(manifest["phases"].values()) == {"approved"}
|
|
for entry in manifest["deliverables"].values():
|
|
assert (out / entry["document"]).is_file()
|
|
assert all((out / name).is_file() for name in entry["attachments"])
|
|
assert (
|
|
recorder.statuses[result.execution_id][-1]["details"]["customer_case"]
|
|
== "acme-webhooks"
|
|
)
|
|
lessons = (
|
|
tmp_path
|
|
/ "lessons"
|
|
/ "candidates"
|
|
/ "acme-webhooks"
|
|
/ f"{result.execution_id}.json"
|
|
)
|
|
assert lessons.is_file()
|
|
assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks"
|
|
|
|
|
|
async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path):
|
|
first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
|
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
|
)
|
|
second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
|
"Ingestion, revised", pre_approved=True, customer_case="acme-webhooks"
|
|
)
|
|
|
|
case = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
|
assert {path.name for path in (case / "executions").iterdir()} == {
|
|
first.execution_id,
|
|
second.execution_id,
|
|
}
|
|
history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8"))
|
|
assert history["customer_case"] == "acme-webhooks"
|
|
assert [
|
|
(i["version"], i["workflow_request"], i["status"], i["execution"])
|
|
for i in history["intents"]
|
|
] == [
|
|
(1, "Ingestion", "completed", f"executions/{first.execution_id}"),
|
|
(2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"),
|
|
]
|
|
assert "Ingestion, revised" in (case / "intent" / "002.md").read_text()
|
|
assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2
|
|
# A Customer case groups Executions; it never feeds one into the next.
|
|
assert second.state["conversation"][0]["text"] == "Ingestion, revised"
|
|
assert "deliverables" in second.state and first.state != second.state
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7]
|
|
)
|
|
async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad):
|
|
engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns()))
|
|
|
|
with pytest.raises(ValueError, match="customer_case"):
|
|
await engine.start("Ingestion", pre_approved=True, customer_case=bad)
|
|
|
|
assert not (tmp_path / "deliverables").exists()
|
|
|
|
|
|
async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path):
|
|
result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
|
"Ingestion", pre_approved=True
|
|
)
|
|
|
|
assert result.state["customer_case"] == "unassigned"
|
|
assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"]
|
|
assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [
|
|
"unassigned"
|
|
]
|
|
|
|
|
|
async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
|
runtime = FakeAgentRuntime(
|
|
{"orchestrator": [[ask_user("Which region should this run in?")]]}
|
|
)
|
|
engine = make_engine(tmp_path, runtime)
|
|
|
|
paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks")
|
|
|
|
assert paused.status == "awaiting_user"
|
|
intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent"
|
|
statement = (intent / "001.md").read_text(encoding="utf-8")
|
|
assert "Ingest signed webhooks" in statement
|
|
assert paused.execution_id in statement
|
|
[entry] = json.loads((intent / "history.json").read_text())["intents"]
|
|
assert entry["status"] == "in_progress"
|
|
assert entry["statement"] == "intent/001.md"
|
|
assert entry["execution_id"] == paused.execution_id
|
|
|
|
|
|
# --- a user sees a Deliverable before it is first submitted -----------------------------
|
|
|
|
|
|
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
|
|
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user()],
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user("Anything else?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [
|
|
# It tries to submit straight away, is refused, and asks.
|
|
[
|
|
sample_submission(K.REQUIREMENTS),
|
|
Call(
|
|
"needs_input",
|
|
{"questions": ["Please approve the draft."], "draft": "# r"},
|
|
),
|
|
],
|
|
submits(K.REQUIREMENTS),
|
|
],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
|
|
|
first = await engine.start("Ingestion")
|
|
refusal = runtime.runs[1].results[0][1]
|
|
await engine.resume(first.execution_id, {"reply": "Approved."})
|
|
|
|
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
|
|
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
|
|
# Once they have replied, the submission is accepted.
|
|
assert runtime.runs[-2].results[0][1].startswith("Submitted")
|
|
|
|
|
|
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
|
|
runtime = FakeAgentRuntime(happy_path_turns())
|
|
|
|
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
|
|
"Ingestion", pre_approved=True
|
|
)
|
|
|
|
assert result.status == "completed"
|
|
|
|
|
|
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
|
|
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
|
|
the Orchestrator and then submitted. The user had answered a question about the
|
|
Deliverable but had never seen it."""
|
|
draft = Call(
|
|
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
|
|
)
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user()], # relays "the requirements analyst stopped"
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user()], # relays the draft
|
|
[delegate(K.REQUIREMENTS)],
|
|
[ask_user("Anything else?")],
|
|
],
|
|
specialist(K.REQUIREMENTS): [
|
|
[], # stops without ending its turn
|
|
[sample_submission(K.REQUIREMENTS), draft],
|
|
submits(K.REQUIREMENTS),
|
|
],
|
|
}
|
|
)
|
|
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
|
|
|
first = await engine.start("Ingestion")
|
|
assert "stopped" in first.prompt["questions"][0]["question"]
|
|
second = await engine.resume(first.execution_id, {"reply": "Try again."})
|
|
await engine.resume(first.execution_id, {"reply": "Approved."})
|
|
|
|
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
|
|
assert (
|
|
analyst[1]
|
|
.results[0][1]
|
|
.startswith("Not submitted. The user has not seen this Deliverable")
|
|
)
|
|
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
|
|
assert analyst[2].results[0][1].startswith("Submitted")
|