"""The Execution engine: LangGraph state, Specialist fan-out, interrupts and recording. Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime. """ import json from pathlib import Path import pytest from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from execution_engine.adk.fake_runtime import Call from execution_engine.adk.runtime import RoleSpec, ToolResult, ToolSpec from execution_engine.agent import ExecutionEngine from execution_engine.recorder import InMemoryRecorder from tests.reference_workflow.workflow import ( DEPENDENCY_GRAPH, DeliverableKind, PhaseId, sample_submission, ) from tests.scoped import ScopedEngine from tests.scoped import ScopedRuntime as FakeAgentRuntime K = DeliverableKind P = PhaseId ROLES = { "orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, f"Produce {spec.kind}.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } # --- scripting helpers --------------------------------------------------------------- def delegate(*kinds: DeliverableKind) -> Call: return Call("delegate", {"kinds": [kind.value for kind in kinds]}) def request_approval(phase: PhaseId) -> Call: return Call("request_approval", {"phase": phase.value, "summary": f"{phase} done"}) def ask_user(*questions: str) -> Call: return Call("ask_user", {"questions": list(questions)}) def reopen(kind: DeliverableKind, reason: str) -> Call: return Call("reopen", {"kind": kind.value, "reason": reason}) FINISH = Call("finish") submission = sample_submission def submits(kind: str) -> list[Call]: return [submission(kind)] def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine: options = { "runtime": runtime, "roles": ROLES, "recorder": InMemoryRecorder(), "checkpointer": InMemorySaver(), "output_root": tmp_path, } options.update(overrides) return ScopedEngine(**options) def happy_path_turns(**extra_specialist_turns) -> dict: turns = { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], [ request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE), ], [ request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS), ], [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], [request_approval(P.PACKAGING), FINISH], ], **{spec.specialist: [submits(kind)] for kind, spec in DEPENDENCY_GRAPH.items()}, } turns.update(extra_specialist_turns) return turns # --- happy path and parallel start ----------------------------------------------------- async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) recorder = InMemoryRecorder() engine = make_engine(tmp_path, runtime, recorder=recorder) result = await engine.start("Event-driven ingestion", pre_approved=True) assert result.status == "completed" assert result.prompt is None ledger = result.state["ledger"] assert set(ledger["phases"].values()) == {"approved"} assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"} out = ( tmp_path / "deliverables" / "cases" / "unassigned" / "executions" / result.execution_id ) assert (out / "solution-architecture-guide.md").is_file() for kind in K: assert (out / "deliverables" / f"{kind}.md").is_file(), kind assert (out / "terraform" / "main.tf").is_file() assert (out / "architecture.mmd").is_file() assert recorder.statuses[result.execution_id][-1]["status"] == "completed" assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K) recorded_types = [e.type for e in recorder.events[result.execution_id]] assert recorded_types.count("approved") == 6 # scoping, then the five after it async def test_independent_specialists_run_together_between_orchestrator_turns( tmp_path, ): runtime = FakeAgentRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) roles = [run.role for run in runtime.runs] sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5] assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == { specialist(K.ARCHITECTURE_DESCRIPTION), specialist(K.DEPLOYMENT_GUIDANCE), } assert roles[sixth_turn + 3] == "orchestrator" async def test_specialist_receives_its_dependencies_content(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) describer_run = next( r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION) ) dependencies = describer_run.task.context["dependencies"] # The scope reaches every Specialist as `scope`, so it is not shown again here. assert describer_run.task.context["scope"] is not None assert list(dependencies) == [ "requirements", "technical_decomposition", "product_selection", "design_recommendations", "target_architecture", ] # The engine drew the diagram from the graph when the Target architecture was submitted. assert dependencies["target_architecture"]["attachments"][ "architecture.mmd" ].startswith("flowchart") assert describer_run.task.deliverable == "architecture_description" # --- refusals reach the agent ------------------------------------------------------------ async def test_orchestrator_cannot_delegate_work_whose_dependencies_are_unmet(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.PRODUCT_SELECTION), delegate(K.REQUIREMENTS)], [ask_user("Anything else?")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) result = await make_engine(tmp_path, runtime).start("Ingestion") first_turn = runtime.runs[0] assert first_turn.results[0][0] == "delegate" assert "Refused" in first_turn.results[0][1] assert "product_selection" in first_turn.results[0][1] assert result.status == "awaiting_user" async def test_incomplete_submission_is_returned_to_the_specialist(tmp_path): incomplete = Call( "submit_deliverable", {"content": "## Functional requirements\nx"} ) runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], specialist(K.REQUIREMENTS): [[incomplete, submission(K.REQUIREMENTS)]], } ) await make_engine(tmp_path, runtime).start("Ingestion") analyst = runtime.runs[1] assert "Non-functional requirements" in analyst.results[0][1] assert analyst.results[1] == ("submit_deliverable", "Submitted requirements.") # --- interactive approvals ---------------------------------------------------------------- async def test_approval_pauses_the_execution_until_the_user_decides(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [delegate(K.PRODUCT_SELECTION)], [ask_user("Pause here")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") assert paused.status == "awaiting_user" assert paused.prompt["approval"]["phase"] == "requirements_discovery" assert set(paused.prompt["approval"]["deliverables"]) == { "requirements", "technical_decomposition", } assert ( paused.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval" ) resumed = await engine.resume( paused.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved" assert ( resumed.state["ledger"]["deliverables"]["product_selection"]["status"] == "submitted" ) assert resumed.prompt == { "questions": [{"from": "orchestrator", "question": "Pause here"}] } async def test_free_text_reply_never_grants_an_approval(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [request_approval(P.REQUIREMENTS_DISCOVERY)], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") again = await engine.resume(paused.execution_id, {"reply": "approve, looks good"}) assert ( again.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval" ) assert again.prompt["approval"]["phase"] == "requirements_discovery" assert again.state["conversation"][-2:] == [ {"from": "user", "text": "approve, looks good"}, { "from": "orchestrator", "text": "Please approve requirements_discovery: requirements_discovery done", }, ] async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [ reopen(K.REQUIREMENTS, "Add the RTO target"), delegate(K.REQUIREMENTS), ], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], ], specialist(K.REQUIREMENTS): [ submits(K.REQUIREMENTS), submits(K.REQUIREMENTS), ], specialist(K.TECHNICAL_DECOMPOSITION): [ submits(K.TECHNICAL_DECOMPOSITION), submits(K.TECHNICAL_DECOMPOSITION), ], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") second = await engine.resume( paused.execution_id, { "approval": { "phase": "requirements_discovery", "decision": "changes", "feedback": "Add the RTO target", } }, ) assert second.prompt["approval"]["phase"] == "requirements_discovery" ledger = second.state["ledger"] assert ledger["deliverables"]["requirements"]["revisions"] == 1 second_analyst_run = [ r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS) ][1] assert second_analyst_run.task.context["feedback"] == ["Add the RTO target"] assert "changes_requested" in [e["type"] for e in ledger["events"]] # --- one voice --------------------------------------------------------------------------------- async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt( tmp_path, ): def describer_turn(task): if task.context["answers"]: return submits(K.ARCHITECTURE_DESCRIPTION) return [Call("needs_input", {"questions": ["Who reads this description?"]})] def iac_turn(task): if task.context["answers"]: return submits(K.DEPLOYMENT_GUIDANCE) return [Call("needs_input", {"questions": ["Which region?"]})] both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE) runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], [request_approval(P.SOLUTION_ARCHITECTURE)], [both], [ask_user()], [both], [ask_user("Continue?")], ], **{ specialist(kind): [submits(kind)] for kind in ( K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE, ) }, specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn], specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn], } ) engine = make_engine(tmp_path, runtime) approval = await engine.start("Ingestion") design = await engine.resume( approval.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) first = await engine.resume( design.execution_id, {"approval": {"phase": "solution_architecture", "decision": "approve"}}, ) assert first.prompt == { "questions": [ { "from": "architecture_description", "question": "Who reads this description?", }, {"from": "deployment_guidance", "question": "Which region?"}, ] } second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"}) deliverables = second.state["ledger"]["deliverables"] assert deliverables["architecture_description"]["status"] == "submitted" assert deliverables["deployment_guidance"]["status"] == "submitted" iac_retry = [ r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE) ][1] assert iac_retry.task.context["answers"] == [ {"questions": ["Which region?"], "reply": "Engineers, in London"} ] # --- revision after validation ------------------------------------------------------------ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp_path): turns = happy_path_turns() turns["orchestrator"] = [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], [ request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE), ], [request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)], [ reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"), delegate(K.DEPLOYMENT_GUIDANCE), ], [request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)], [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], [request_approval(P.PACKAGING), FINISH], ] turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [ submits(K.DEPLOYMENT_GUIDANCE), submits(K.DEPLOYMENT_GUIDANCE), ] turns[specialist(K.VALIDATION_RESULTS)] = [ [submission(K.VALIDATION_RESULTS, "FAILED: missing IAM binding")], submits(K.VALIDATION_RESULTS), ] runtime = FakeAgentRuntime(turns) result = await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) assert result.status == "completed" events = result.state["ledger"]["events"] withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"] assert withdrawn == ["solution_implementation"] requests = [e["subject"] for e in events if e["type"] == "approval_requested"] assert requests.count("solution_implementation") == 2 assert requests.count("solution_architecture") == 1 iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)] assert iac_runs[1].task.context["feedback"] == [ "terraform plan: missing IAM binding" ] async def test_pre_approved_escalation_ends_with_warnings(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [reopen(K.REQUIREMENTS, "not good enough")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) result = await make_engine(tmp_path, runtime, max_revisions=0).start( "Ingestion", pre_approved=True ) assert result.status == "completed_with_warnings" assert "revision limit" in result.state["warnings"][0] # What was produced is kept, marked as unfinished, and the request is on record. case = tmp_path / "deliverables" / "cases" / "unassigned" [intent] = json.loads((case / "intent" / "history.json").read_text())["intents"] assert intent["status"] == "completed_with_warnings" assert intent["execution"] == f"executions/{result.execution_id}" out_dir = case / intent["execution"] manifest = json.loads((out_dir / "execution.json").read_text()) assert manifest["status"] == "completed_with_warnings" assert "revision limit" in manifest["warnings"][0] assert manifest["solution_guide"] is None assert not (out_dir / "solution-architecture-guide.md").exists() assert (out_dir / "deliverables" / "requirements.md").is_file() # Nothing unapproved reads as final: each Deliverable and Phase has its status. assert manifest["deliverables"]["requirements"]["status"] == "submitted" assert manifest["phases"]["packaging"] != "approved" # --- budgets ------------------------------------------------------------------------------------- async def test_orchestrator_budget_exhaustion_hands_control_to_the_user(tmp_path): roles = {**ROLES, "orchestrator": RoleSpec("orchestrator", "", budget=2)} wandering = [Call("ready_work"), Call("ready_work"), Call("ready_work")] runtime = FakeAgentRuntime({"orchestrator": [wandering]}) result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") assert result.status == "awaiting_user" question = result.prompt["questions"][0] assert question["from"] == "orchestrator" assert "2 tool calls" in question["question"] async def test_specialist_budget_exhaustion_is_relayed_as_a_question(tmp_path): roles = { **ROLES, specialist(K.REQUIREMENTS): RoleSpec("requirements-analyst", "", budget=1), } runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]], specialist(K.REQUIREMENTS): [ [ Call("submit_deliverable", {"content": "## Current state\nx"}), submission(K.REQUIREMENTS), ] ], } ) result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") assert result.prompt["questions"][0]["from"] == "requirements" assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" # --- cloud read permission ------------------------------------------------------------------- def cloud_read_factory(calls: list[str]): async def list_projects() -> ToolResult: calls.append("list_projects") return ToolResult("projects: demo-project") tool = ToolSpec( "list_projects", "List projects.", {"type": "object", "properties": {}}, list_projects, tier="cloud_read", ) def factory(role: RoleSpec, deliverable: str | None, working: dict): return [tool] if role.name == "requirements-analyst" else [] return factory async def test_cloud_read_asks_permission_once_then_runs_the_tool(tmp_path): calls: list[str] = [] runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.REQUIREMENTS)], [ask_user("Next?")], ], specialist(K.REQUIREMENTS): [ [Call("list_projects"), submission(K.REQUIREMENTS)], [Call("list_projects"), submission(K.REQUIREMENTS)], ], } ) engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) asked = await engine.start("Ingestion") assert asked.prompt == { "permission": {"tier": "cloud_read", "requested_by": ["requirements"]} } assert calls == [] granted = await engine.resume( asked.execution_id, {"permission": {"cloud_read": True}} ) assert calls == ["list_projects"] assert granted.state["allow_cloud_read"] is True assert ( granted.state["ledger"]["deliverables"]["requirements"]["status"] == "submitted" ) async def test_denied_cloud_read_leaves_the_agent_without_those_tools(tmp_path): """Their definitions were half of what the requirements analyst was sent, for calls that could only be refused. It is told, and has nothing to call.""" calls: list[str] = [] runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], specialist(K.REQUIREMENTS): [[submission(K.REQUIREMENTS)]], } ) engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) await engine.start("Ingestion", allow_cloud_read=False) analyst = runtime.runs[1] assert "list_projects" not in analyst.tools assert analyst.task.context["cloud_read"] is False assert calls == [] # --- isolation and durability ------------------------------------------------------------------- async def test_executions_in_one_session_share_no_state(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [ask_user("?")], [ask_user("fresh?")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) engine = make_engine(tmp_path, runtime) first = await engine.start("First", session_id="s-1") second = await engine.start("Second", session_id="s-1") assert first.execution_id != second.execution_id assert second.state["deliverables"] == {} assert second.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" assert second.state["workflow_request"] == "Second" assert second.state["session_id"] == "s-1" async def test_paused_execution_resumes_from_a_durable_checkpoint(tmp_path): turns = { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [ask_user("Resumed")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], } runtime = FakeAgentRuntime(turns) db_path = str(tmp_path / "checkpoints.db") async with AsyncSqliteSaver.from_conn_string(db_path) as saver: paused = await make_engine(tmp_path, runtime, checkpointer=saver).start( "Ingestion" ) assert paused.prompt["approval"]["phase"] == "requirements_discovery" async with AsyncSqliteSaver.from_conn_string(db_path) as saver: resumed = await make_engine(tmp_path, runtime, checkpointer=saver).resume( paused.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert resumed.prompt == { "questions": [{"from": "orchestrator", "question": "Resumed"}] } assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved" # --- identity and failures ------------------------------------------------------------------ async def test_caller_can_choose_the_execution_id_once(tmp_path): runtime = FakeAgentRuntime({"orchestrator": [[ask_user("Hello?")]]}) engine = make_engine(tmp_path, runtime) first = await engine.start("Ingestion", execution_id="task-123") assert first.execution_id == "task-123" with pytest.raises(ValueError, match="task-123"): await engine.start("Again", execution_id="task-123") async def test_a_failing_specialist_is_relayed_to_the_user_as_a_question(tmp_path): class Failing(FakeAgentRuntime): async def run(self, role, task, tools): if role.name == specialist(K.REQUIREMENTS): raise RuntimeError("model API unavailable") return await super().run(role, task, tools) runtime = Failing({"orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]]}) result = await make_engine(tmp_path, runtime).start("Ingestion") [question] = result.prompt["questions"] assert question["from"] == "requirements" assert "model API unavailable" in question["question"] assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [ask_user("What is the RTO?", "Which region?")], [ask_user("Anything else?")], ] } ) engine = make_engine(tmp_path, runtime, max_orchestrator_turns=3) result = await engine.start( "Ingestion", pre_approved=True, scripted_answers=[(r"\bRTO\b", "One hour."), ("region", "europe-west2")], ) first_reply = result.state["conversation"][2]["text"] assert "Q: What is the RTO?\nA: One hour." in first_reply assert "Q: Which region?\nA: europe-west2" in first_reply assert "No user is available" not in first_reply second_reply = result.state["conversation"][4]["text"] assert second_reply.startswith("No user is available") # --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) ------- async def test_specialist_context_carries_its_contract(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS)) assert run.task.context["contract"] == { "sections": ["Deployment dry-run", "Architecture and policy analysis"], "attachments": ["verdicts.json"], } assert run.task.context["draft"] is None async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path): draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args def confirm_then_submit_what_came_back(task): returned = task.context["draft"] return [Call("submit_deliverable", returned)] runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [ask_user()], [delegate(K.REQUIREMENTS)], [ask_user("Anything else?")], ], specialist(K.REQUIREMENTS): [ [ Call( "needs_input", { "questions": ["Do you approve this draft?"], "draft": draft["content"], "attachments": [ {"path": path, "content": content} for path, content in draft["attachments"].items() ], }, ) ], confirm_then_submit_what_came_back, ], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") assert paused.state["drafts"]["requirements"]["content"] == draft["content"] after = await engine.resume(paused.execution_id, {"reply": "Approved."}) second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1] assert second_run.task.context["draft"] == { "content": draft["content"], "attachments": draft["attachments"], } assert second_run.task.context["answers"][-1]["reply"] == "Approved." assert after.state["deliverables"]["requirements"]["content"] == draft["content"] assert after.state["drafts"] == {} # --- cap on parallel Specialists (iteration 4, step 5) --------------------------------- async def test_parallel_specialists_never_exceed_the_cap(tmp_path): import asyncio from execution_engine.adk.runtime import RunLimiter in_flight = 0 peak = 0 class SlowRuntime(FakeAgentRuntime): async def run(self, role, task, tools): nonlocal in_flight, peak if role.name == "orchestrator": return await super().run(role, task, tools) in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0.01) try: return await super().run(role, task, tools) finally: in_flight -= 1 runtime = SlowRuntime(happy_path_turns()) engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1)) result = await engine.start("Ingestion", pre_approved=True) assert result.status == "completed" assert peak == 1 async def test_without_a_cap_independent_specialists_overlap(tmp_path): import asyncio in_flight = 0 peak = 0 class SlowRuntime(FakeAgentRuntime): async def run(self, role, task, tools): nonlocal in_flight, peak if role.name == "orchestrator": return await super().run(role, task, tools) in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0.01) try: return await super().run(role, task, tools) finally: in_flight -= 1 runtime = SlowRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) assert peak == 2 # --- Customer cases group Executions --------------------------------------------------- async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path): recorder = InMemoryRecorder() engine = make_engine( tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder ) result = await engine.start( "Ingestion", pre_approved=True, customer_case="acme-webhooks" ) case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks" out = case_dir / "executions" / result.execution_id assert (out / "solution-architecture-guide.md").is_file() assert result.state["customer_case"] == "acme-webhooks" manifest = json.loads((out / "execution.json").read_text(encoding="utf-8")) assert manifest["customer_case"] == "acme-webhooks" assert manifest["workflow_request"] == "Ingestion" assert manifest["intent_version"] == 1 assert manifest["solution_guide"] == "solution-architecture-guide.md" assert manifest["deliverables"]["deployment_guidance"] == { "document": "deliverables/deployment_guidance.md", "attachments": [ "implementation.json", "terraform/main.tf", "terraform/variables.tf", ], "status": "submitted", } assert manifest["status"] == "completed" and manifest["warnings"] == [] assert set(manifest["phases"].values()) == {"approved"} for entry in manifest["deliverables"].values(): assert (out / entry["document"]).is_file() assert all((out / name).is_file() for name in entry["attachments"]) assert ( recorder.statuses[result.execution_id][-1]["details"]["customer_case"] == "acme-webhooks" ) lessons = ( tmp_path / "lessons" / "candidates" / "acme-webhooks" / f"{result.execution_id}.json" ) assert lessons.is_file() assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks" async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path): first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start( "Ingestion", pre_approved=True, customer_case="acme-webhooks" ) second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start( "Ingestion, revised", pre_approved=True, customer_case="acme-webhooks" ) case = tmp_path / "deliverables" / "cases" / "acme-webhooks" assert {path.name for path in (case / "executions").iterdir()} == { first.execution_id, second.execution_id, } history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8")) assert history["customer_case"] == "acme-webhooks" assert [ (i["version"], i["workflow_request"], i["status"], i["execution"]) for i in history["intents"] ] == [ (1, "Ingestion", "completed", f"executions/{first.execution_id}"), (2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"), ] assert "Ingestion, revised" in (case / "intent" / "002.md").read_text() assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2 # A Customer case groups Executions; it never feeds one into the next. assert second.state["conversation"][0]["text"] == "Ingestion, revised" assert "deliverables" in second.state and first.state != second.state @pytest.mark.parametrize( "bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7] ) async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad): engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())) with pytest.raises(ValueError, match="customer_case"): await engine.start("Ingestion", pre_approved=True, customer_case=bad) assert not (tmp_path / "deliverables").exists() async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path): result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start( "Ingestion", pre_approved=True ) assert result.state["customer_case"] == "unassigned" assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"] assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [ "unassigned" ] async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path): runtime = FakeAgentRuntime( {"orchestrator": [[ask_user("Which region should this run in?")]]} ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks") assert paused.status == "awaiting_user" intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent" statement = (intent / "001.md").read_text(encoding="utf-8") assert "Ingest signed webhooks" in statement assert paused.execution_id in statement [entry] = json.loads((intent / "history.json").read_text())["intents"] assert entry["status"] == "in_progress" assert entry["statement"] == "intent/001.md" assert entry["execution_id"] == paused.execution_id # --- a user sees a Deliverable before it is first submitted ----------------------------- async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path): """Evaluation case 04: a Specialist that ran out of budget submitted without asking.""" runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [ask_user()], [delegate(K.REQUIREMENTS)], [ask_user("Anything else?")], ], specialist(K.REQUIREMENTS): [ # It tries to submit straight away, is refused, and asks. [ sample_submission(K.REQUIREMENTS), Call( "needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}, ), ], submits(K.REQUIREMENTS), ], } ) engine = make_engine(tmp_path, runtime, confirm_drafts=True) first = await engine.start("Ingestion") refusal = runtime.runs[1].results[0][1] await engine.resume(first.execution_id, {"reply": "Approved."}) assert refusal.startswith("Not submitted. The user has not seen this Deliverable") assert first.prompt["questions"][0]["question"] == "Please approve the draft." # Once they have replied, the submission is accepted. assert runtime.runs[-2].results[0][1].startswith("Submitted") async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) result = await make_engine(tmp_path, runtime, confirm_drafts=True).start( "Ingestion", pre_approved=True ) assert result.status == "completed" async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path): """Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by the Orchestrator and then submitted. The user had answered a question about the Deliverable but had never seen it.""" draft = Call( "needs_input", {"questions": ["Please approve the draft."], "draft": "# r"} ) runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [ask_user()], # relays "the requirements analyst stopped" [delegate(K.REQUIREMENTS)], [ask_user()], # relays the draft [delegate(K.REQUIREMENTS)], [ask_user("Anything else?")], ], specialist(K.REQUIREMENTS): [ [], # stops without ending its turn [sample_submission(K.REQUIREMENTS), draft], submits(K.REQUIREMENTS), ], } ) engine = make_engine(tmp_path, runtime, confirm_drafts=True) first = await engine.start("Ingestion") assert "stopped" in first.prompt["questions"][0]["question"] second = await engine.resume(first.execution_id, {"reply": "Try again."}) await engine.resume(first.execution_id, {"reply": "Approved."}) analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)] assert ( analyst[1] .results[0][1] .startswith("Not submitted. The user has not seen this Deliverable") ) assert second.prompt["questions"][0]["question"] == "Please approve the draft." assert analyst[2].results[0][1].startswith("Submitted")