"""The Execution engine: LangGraph state, Specialist fan-out, interrupts and recording. Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime. """ from pathlib import Path from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from app.agents.fake_runtime import Call, FakeAgentRuntime from app.agents.runtime import RoleSpec, ToolResult, ToolSpec from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId from app.execution.engine import ExecutionEngine from app.execution.recorder import InMemoryRecorder K = DeliverableKind P = PhaseId ROLES = { "orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, f"Produce {spec.kind}.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } # --- scripting helpers --------------------------------------------------------------- def delegate(*kinds: DeliverableKind) -> Call: return Call("delegate", {"kinds": [kind.value for kind in kinds]}) def request_approval(phase: PhaseId) -> Call: return Call("request_approval", {"phase": phase.value, "summary": f"{phase} done"}) def ask_user(*questions: str) -> Call: return Call("ask_user", {"questions": list(questions)}) def reopen(kind: DeliverableKind, reason: str) -> Call: return Call("reopen", {"kind": kind.value, "reason": reason}) FINISH = Call("finish") def submission(kind: DeliverableKind, note: str = "") -> Call: spec = DEPENDENCY_GRAPH[kind] body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections) content = f"# {kind}\n{note or 'Summary.'}\n{body}\n" attachments = {name: f"# {name}\n" for name in spec.attachments} return Call("submit_deliverable", {"content": content, "attachments": attachments}) def submits(kind: DeliverableKind) -> list[Call]: return [submission(kind)] def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine: options = { "runtime": runtime, "roles": ROLES, "recorder": InMemoryRecorder(), "checkpointer": InMemorySaver(), "output_root": tmp_path, } options.update(overrides) return ExecutionEngine(**options) def happy_path_turns(**extra_specialist_turns) -> dict: turns = { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], [delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)], [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], [request_approval(P.PACKAGING), FINISH], ], **{spec.specialist: [submits(kind)] for kind, spec in DEPENDENCY_GRAPH.items()}, } turns.update(extra_specialist_turns) return turns # --- happy path and parallel start ----------------------------------------------------- async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) recorder = InMemoryRecorder() engine = make_engine(tmp_path, runtime, recorder=recorder) result = await engine.start("Event-driven ingestion", pre_approved=True) assert result.status == "completed" assert result.prompt is None ledger = result.state["ledger"] assert set(ledger["phases"].values()) == {"approved"} assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"} out = tmp_path / "deliverables" / "executions" / result.execution_id assert (out / "solution-architecture-guide.md").is_file() for kind in K: assert (out / "deliverables" / f"{kind}.md").is_file(), kind assert (out / "terraform" / "main.tf").is_file() assert (out / "architecture.mmd").is_file() assert recorder.statuses[result.execution_id][-1]["status"] == "completed" assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K) recorded_types = [e.type for e in recorder.events[result.execution_id]] assert recorded_types.count("approved") == 4 async def test_independent_specialists_run_together_between_orchestrator_turns( tmp_path, ): runtime = FakeAgentRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) roles = [run.role for run in runtime.runs] fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3] assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == { specialist(K.ARCHITECTURE_DIAGRAM), specialist(K.DESIGN_RECOMMENDATIONS), } assert roles[fourth_turn + 3] == "orchestrator" async def test_specialist_receives_its_dependencies_content(tmp_path): runtime = FakeAgentRuntime(happy_path_turns()) await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) describer_run = next( r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION) ) dependencies = describer_run.task.context["dependencies"] assert list(dependencies) == [ "requirements", "technical_decomposition", "product_selection", "architecture_diagram", ] assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"] assert describer_run.task.deliverable == "architecture_description" # --- refusals reach the agent ------------------------------------------------------------ async def test_orchestrator_cannot_delegate_work_whose_dependencies_are_unmet(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.PRODUCT_SELECTION), delegate(K.REQUIREMENTS)], [ask_user("Anything else?")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) result = await make_engine(tmp_path, runtime).start("Ingestion") first_turn = runtime.runs[0] assert first_turn.results[0][0] == "delegate" assert "Refused" in first_turn.results[0][1] assert "product_selection" in first_turn.results[0][1] assert result.status == "awaiting_user" async def test_incomplete_submission_is_returned_to_the_specialist(tmp_path): incomplete = Call( "submit_deliverable", {"content": "## Functional requirements\nx"} ) runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], specialist(K.REQUIREMENTS): [[incomplete, submission(K.REQUIREMENTS)]], } ) await make_engine(tmp_path, runtime).start("Ingestion") analyst = runtime.runs[1] assert "Non-functional requirements" in analyst.results[0][1] assert analyst.results[1] == ("submit_deliverable", "Submitted requirements.") # --- interactive approvals ---------------------------------------------------------------- async def test_approval_pauses_the_execution_until_the_user_decides(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [delegate(K.PRODUCT_SELECTION)], [ask_user("Pause here")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") assert paused.status == "awaiting_user" assert paused.prompt["approval"]["phase"] == "requirements_discovery" assert set(paused.prompt["approval"]["deliverables"]) == { "requirements", "technical_decomposition", } assert ( paused.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval" ) resumed = await engine.resume( paused.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved" assert ( resumed.state["ledger"]["deliverables"]["product_selection"]["status"] == "submitted" ) assert resumed.prompt == { "questions": [{"from": "orchestrator", "question": "Pause here"}] } async def test_free_text_reply_never_grants_an_approval(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [request_approval(P.REQUIREMENTS_DISCOVERY)], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") again = await engine.resume(paused.execution_id, {"reply": "approve, looks good"}) assert ( again.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval" ) assert again.prompt["approval"]["phase"] == "requirements_discovery" assert again.state["conversation"][-2:] == [ {"from": "user", "text": "approve, looks good"}, { "from": "orchestrator", "text": "Please approve requirements_discovery: requirements_discovery done", }, ] async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [ reopen(K.REQUIREMENTS, "Add the RTO target"), delegate(K.REQUIREMENTS), ], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], ], specialist(K.REQUIREMENTS): [ submits(K.REQUIREMENTS), submits(K.REQUIREMENTS), ], specialist(K.TECHNICAL_DECOMPOSITION): [ submits(K.TECHNICAL_DECOMPOSITION), submits(K.TECHNICAL_DECOMPOSITION), ], } ) engine = make_engine(tmp_path, runtime) paused = await engine.start("Ingestion") second = await engine.resume( paused.execution_id, { "approval": { "phase": "requirements_discovery", "decision": "changes", "feedback": "Add the RTO target", } }, ) assert second.prompt["approval"]["phase"] == "requirements_discovery" ledger = second.state["ledger"] assert ledger["deliverables"]["requirements"]["revisions"] == 1 second_analyst_run = [ r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS) ][1] assert second_analyst_run.task.context["feedback"] == ["Add the RTO target"] assert "changes_requested" in [e["type"] for e in ledger["events"]] # --- one voice --------------------------------------------------------------------------------- async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt( tmp_path, ): def diagrammer_turn(task): if task.context["answers"]: return submits(K.ARCHITECTURE_DIAGRAM) return [Call("needs_input", {"questions": ["Single region or multi-region?"]})] def advisor_turn(task): if task.context["answers"]: return submits(K.DESIGN_RECOMMENDATIONS) return [Call("needs_input", {"questions": ["What is the RTO?"]})] runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [delegate(K.PRODUCT_SELECTION)], [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], [ask_user()], [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], [ask_user("Continue?")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)], specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn], specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn], } ) engine = make_engine(tmp_path, runtime) approval = await engine.start("Ingestion") first = await engine.resume( approval.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert first.prompt == { "questions": [ { "from": "architecture_diagram", "question": "Single region or multi-region?", }, {"from": "design_recommendations", "question": "What is the RTO?"}, ] } second = await engine.resume( first.execution_id, {"reply": "Single region, RTO 1 hour"} ) deliverables = second.state["ledger"]["deliverables"] assert deliverables["architecture_diagram"]["status"] == "submitted" assert deliverables["design_recommendations"]["status"] == "submitted" advisor_retry = [ r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS) ][1] assert advisor_retry.task.context["answers"] == [ {"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"} ] # --- revision after validation ------------------------------------------------------------ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp_path): turns = happy_path_turns() turns["orchestrator"] = [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], [delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)], [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], [ reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"), delegate(K.DEPLOYMENT_GUIDANCE), ], [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], [request_approval(P.PACKAGING), FINISH], ] turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [ submits(K.DEPLOYMENT_GUIDANCE), submits(K.DEPLOYMENT_GUIDANCE), ] turns[specialist(K.VALIDATION_RESULTS)] = [ [submission(K.VALIDATION_RESULTS, "FAILED: missing IAM binding")], submits(K.VALIDATION_RESULTS), ] runtime = FakeAgentRuntime(turns) result = await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) assert result.status == "completed" events = result.state["ledger"]["events"] withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"] assert withdrawn == ["solution_architecture"] requests = [e["subject"] for e in events if e["type"] == "approval_requested"] assert requests.count("solution_architecture") == 2 iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)] assert iac_runs[1].task.context["feedback"] == [ "terraform plan: missing IAM binding" ] async def test_pre_approved_escalation_ends_with_warnings(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [reopen(K.REQUIREMENTS, "not good enough")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) result = await make_engine(tmp_path, runtime, max_revisions=0).start( "Ingestion", pre_approved=True ) assert result.status == "completed_with_warnings" assert "revision limit" in result.state["warnings"][0] assert not (tmp_path / "deliverables").exists() # --- budgets ------------------------------------------------------------------------------------- async def test_orchestrator_budget_exhaustion_hands_control_to_the_user(tmp_path): roles = {**ROLES, "orchestrator": RoleSpec("orchestrator", "", budget=2)} wandering = [Call("ready_work"), Call("ready_work"), Call("ready_work")] runtime = FakeAgentRuntime({"orchestrator": [wandering]}) result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") assert result.status == "awaiting_user" question = result.prompt["questions"][0] assert question["from"] == "orchestrator" assert "2 tool calls" in question["question"] async def test_specialist_budget_exhaustion_is_relayed_as_a_question(tmp_path): roles = { **ROLES, specialist(K.REQUIREMENTS): RoleSpec("requirements-analyst", "", budget=1), } runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]], specialist(K.REQUIREMENTS): [ [ Call("submit_deliverable", {"content": "## Current state\nx"}), submission(K.REQUIREMENTS), ] ], } ) result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") assert result.prompt["questions"][0]["from"] == "requirements" assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" # --- cloud read permission ------------------------------------------------------------------- def cloud_read_factory(calls: list[str]): async def list_projects() -> ToolResult: calls.append("list_projects") return ToolResult("projects: demo-project") tool = ToolSpec( "list_projects", "List projects.", {"type": "object", "properties": {}}, list_projects, tier="cloud_read", ) def factory(role: RoleSpec, deliverable: str | None): return [tool] if role.name == "requirements-analyst" else [] return factory async def test_cloud_read_asks_permission_once_then_runs_the_tool(tmp_path): calls: list[str] = [] runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.REQUIREMENTS)], [ask_user("Next?")], ], specialist(K.REQUIREMENTS): [ [Call("list_projects"), submission(K.REQUIREMENTS)], [Call("list_projects"), submission(K.REQUIREMENTS)], ], } ) engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) asked = await engine.start("Ingestion") assert asked.prompt == { "permission": {"tier": "cloud_read", "requested_by": ["requirements"]} } assert calls == [] granted = await engine.resume( asked.execution_id, {"permission": {"cloud_read": True}} ) assert calls == ["list_projects"] assert granted.state["allow_cloud_read"] is True assert ( granted.state["ledger"]["deliverables"]["requirements"]["status"] == "submitted" ) async def test_denied_cloud_read_is_refused_to_the_agent(tmp_path): calls: list[str] = [] runtime = FakeAgentRuntime( { "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], specialist(K.REQUIREMENTS): [ [Call("list_projects"), submission(K.REQUIREMENTS)] ], } ) engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) await engine.start("Ingestion", allow_cloud_read=False) analyst = runtime.runs[1] assert "not permitted" in analyst.results[0][1] assert calls == [] # --- isolation and durability ------------------------------------------------------------------- async def test_executions_in_one_session_share_no_state(tmp_path): runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [ask_user("?")], [ask_user("fresh?")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], } ) engine = make_engine(tmp_path, runtime) first = await engine.start("First", session_id="s-1") second = await engine.start("Second", session_id="s-1") assert first.execution_id != second.execution_id assert second.state["deliverables"] == {} assert second.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" assert second.state["workflow_request"] == "Second" assert second.state["session_id"] == "s-1" async def test_paused_execution_resumes_from_a_durable_checkpoint(tmp_path): turns = { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [request_approval(P.REQUIREMENTS_DISCOVERY)], [ask_user("Resumed")], ], specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], } runtime = FakeAgentRuntime(turns) db_path = str(tmp_path / "checkpoints.db") async with AsyncSqliteSaver.from_conn_string(db_path) as saver: paused = await make_engine(tmp_path, runtime, checkpointer=saver).start( "Ingestion" ) assert paused.prompt["approval"]["phase"] == "requirements_discovery" async with AsyncSqliteSaver.from_conn_string(db_path) as saver: resumed = await make_engine(tmp_path, runtime, checkpointer=saver).resume( paused.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert resumed.prompt == { "questions": [{"from": "orchestrator", "question": "Resumed"}] } assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved"