From d9214c671aeaa6b3c31918b47a443b5ee33bf3cd Mon Sep 17 00:00:00 2001 From: Jonathan Boniface Date: Wed, 16 Sep 2026 09:38:13 +0100 Subject: [PATCH] feat: add the hierarchical Execution engine on LangGraph Increment 4 of .scratch/solution-architect-agent/spec.md, test-first. - app/agents/runtime.py: framework-neutral AgentRuntime interface (RoleSpec, AgentTask, ToolSpec, outcomes); fake_runtime.py replays scripted tool calls through the real tools with step budgets. - app/execution/graph.py: Orchestrator turn fans out to parallel Specialist tasks (Send), pauses with interrupt() for questions, Approvals and cloud-read permission, and finalizes or ends with warnings. Only the Orchestrator turn writes the Ledger. - app/execution/tools.py: Orchestrator tools, Specialist submit and needs_input, and the cloud-read permission gate. - app/execution/engine.py: ExecutionEngine.start/resume returning an Interaction; the single interface for A2A, /generate and eval. - Recorders (in-memory, DatabaseManager), configurable checkpointer (memory, sqlite, postgres), engine settings, langgraph checkpoint dependencies. - 19 new tests; full suite 121 passed. Co-Authored-By: Claude Opus 5 --- .env.example | 8 + .scratch/solution-architect-agent/spec.md | 41 ++ app/agents/__init__.py | 1 + app/agents/fake_runtime.py | 89 +++ app/agents/runtime.py | 103 ++++ app/config.py | 8 + app/execution/checkpointing.py | 38 ++ app/execution/engine.py | 145 +++++ app/execution/graph.py | 478 ++++++++++++++++ app/execution/recorder.py | 156 ++++++ app/execution/state.py | 35 ++ app/execution/tools.py | 318 +++++++++++ requirements.txt | 5 +- tests/execution/test_engine.py | 628 ++++++++++++++++++++++ tests/execution/test_recording.py | 91 ++++ 15 files changed, 2143 insertions(+), 1 deletion(-) create mode 100644 app/agents/__init__.py create mode 100644 app/agents/fake_runtime.py create mode 100644 app/agents/runtime.py create mode 100644 app/execution/checkpointing.py create mode 100644 app/execution/engine.py create mode 100644 app/execution/graph.py create mode 100644 app/execution/recorder.py create mode 100644 app/execution/state.py create mode 100644 app/execution/tools.py create mode 100644 tests/execution/test_engine.py create mode 100644 tests/execution/test_recording.py diff --git a/.env.example b/.env.example index 2e3e063..d2820b8 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,14 @@ ADK_ENABLE_ARTIFACT_STORE=true CLOUD_PROVIDER=gcp MCP_ENABLE_PREVIEW=true +# Hierarchical Execution Engine +ORCHESTRATOR_BUDGET=40 +SPECIALIST_BUDGET=15 +MAX_REVISIONS=3 +MAX_ORCHESTRATOR_TURNS=100 +CHECKPOINTER=sqlite +CHECKPOINT_DB_PATH=checkpoints.db + # Google Developer Knowledge MCP Integration DEVELOPER_KNOWLEDGE_MCP_URL=https://developerknowledge.googleapis.com/mcp DEVELOPER_KNOWLEDGE_MCP_ENABLED=true diff --git a/.scratch/solution-architect-agent/spec.md b/.scratch/solution-architect-agent/spec.md index d5e67f7..ce1b703 100644 --- a/.scratch/solution-architect-agent/spec.md +++ b/.scratch/solution-architect-agent/spec.md @@ -569,3 +569,44 @@ Slowest tests: eval benchmark 7.3 s, then 3–4.5 s each for the orchestrator, e - `solution_guide`: the template's 8 top-level sections. - `architecture_diagram`: attachment `architecture.mmd`. - `technical_decomposition` and `architecture_description`: non-empty content only. + +### 2026-09-15 · Increment 4: Execution graph + +**Done (test-first)** +- `app/agents/runtime.py`: the `AgentRuntime` protocol, `RoleSpec`, `AgentTask`, `ToolSpec`/`ToolResult`, and the outcomes `Submitted | NeedsInput | BudgetExhausted | Stopped | TurnEnded`. +- `app/agents/fake_runtime.py`: `FakeAgentRuntime`, which replays scripted tool calls per role through the real `ToolSpec`s, enforces `RoleSpec.budget`, and records every run (task, tools offered, tool messages, outcome). +- `app/execution/state.py`: `ExecutionState` (JSON-compatible). `outcomes` uses an append reducer, so parallel Specialist tasks never write the Ledger. +- `app/execution/tools.py`: + - `OrchestratorToolbox`: `ready_work`, `delegate`, `ask_user`, `request_approval`, `reopen`, `read_deliverable`, `finish`. + - `specialist_tools`: `submit_deliverable` (checks the contract and refuses unsafe attachment paths) and `needs_input`, plus the role's extra tools. + - `gate_cloud_read`: allow / refuse / ask permission. +- `app/execution/graph.py`: `orchestrator_turn` → `Send` × N `specialist_task` → `orchestrator_turn`; `await_user` (`interrupt`); `finalize`; `end_with_warnings`. +- `app/execution/engine.py`: **`ExecutionEngine.start(request, session_id, pre_approved, allow_cloud_read)` / `resume(execution_id, reply)` / `interaction(execution_id)` → `Interaction(status, prompt, state)`**. This is the one interface for A2A, `/generate` and the eval harness. The tests use only it. +- `app/execution/recorder.py`: the `ExecutionRecorder` protocol with `InMemoryRecorder` and `DatabaseRecorder` (the existing tables via `asyncio.to_thread`). +- `app/execution/checkpointing.py`: `open_checkpointer(settings)` for `memory | sqlite | postgres`. +- Settings `ORCHESTRATOR_BUDGET`, `SPECIALIST_BUDGET`, `MAX_REVISIONS`, `MAX_ORCHESTRATOR_TURNS`, `CHECKPOINTER` (default `sqlite`) and `CHECKPOINT_DB_PATH`, plus `.env.example`. `requirements.txt`: `langgraph>=1.2`, `langgraph-checkpoint-sqlite`, `langgraph-checkpoint-postgres`, `psycopg[binary]`. +- Tests: + - `tests/execution/test_engine.py`: 17 tests covering the pre-approved happy path with written files, parallel start, dependency context, refusal messages, incomplete submissions, Approval pause/resume, free text never approving, changes → Revision → re-approval, one voice, Revision after a validation failure, pre-approved escalation, Orchestrator and Specialist budgets, cloud-read ask/grant/deny, clean Executions, and resuming from a durable SQLite checkpoint. + - `tests/execution/test_recording.py`: 2 tests covering `DatabaseRecorder` against a temp SQLite database, and the checkpointer factory. + - Full suite: **121 passed**. `ruff` and `mypy` are clean on `app/agents` and `app/execution`. +- **Checking the tests bite.** I broke each of these in the code on purpose, and each break failed at least one test: + - no cloud-read gate + - permission never asked + - Specialist questions dropped + - submission contract unchecked + +**Added to or changed from the spec text** +- **Engine and file names.** There's a new `app/execution/engine.py` facade, and tests live in `test_engine.py`/`test_recording.py`, not `test_graph.py`. The graph is an internal detail behind the engine. +- **`Stopped` outcome.** An agent that ends without a turn-ending tool is `Stopped`. Tools end a run by returning `ToolResult(outcome=...)`, so ending can depend on the result: pre-approved `request_approval` doesn't end the turn, and refusals never do. Engine-owned tools use `tier="execution"`. +- **Prompt and reply shapes.** + - `Interaction.prompt` is `{"questions": [{"from": , "question"}]}`, `{"approval": {"phase", "summary", "deliverables"}}` or `{"permission": {"tier", "requested_by"}}`. + - `resume` takes `{"reply": str}`, `{"approval": {"phase", "decision": approve|changes, "feedback"}}` or `{"permission": {"cloud_read": bool}}`. + - An approval reply that isn't structured goes into the conversation and never grants. +- **Single writer.** Specialist outcomes are applied by the next `orchestrator_turn` in graph order, using `outcomes_applied`. A superseded result (stale attempt) is discarded and noted to the Orchestrator. +- **Specialist problems as questions.** A Specialist that runs out of budget or stops is released and relayed as a question from its Deliverable, keeping one voice. A permission `NeedsInput` becomes a single permission prompt, not a question. +- **Pre-approved mode.** `allow_cloud_read=None` becomes `False`. An escalation, or an Orchestrator that can't continue, ends as `completed_with_warnings` with no files written. +- **Loop guard.** `MAX_ORCHESTRATOR_TURNS` (default 100) bounds loops, and the LangGraph `recursion_limit` is raised to 10 000. +- **Conversation.** It records the Orchestrator's rendered prompts as well as user replies. +- **Session name.** `DatabaseRecorder` session rows use `agent_name="solution-architect"`. +- **Deferred to increment 5:** the design advisor's pillar fan-out tool, which needs the `pillar-advisor` role file. +- **Not tested:** `AsyncPostgresSaver` against a real PostgreSQL (none runs locally). The import and wiring are in place. diff --git a/app/agents/__init__.py b/app/agents/__init__.py new file mode 100644 index 0000000..428175a --- /dev/null +++ b/app/agents/__init__.py @@ -0,0 +1 @@ +"""Agent runtimes: the framework-neutral interface agents run behind, and its implementations.""" diff --git a/app/agents/fake_runtime.py b/app/agents/fake_runtime.py new file mode 100644 index 0000000..7733627 --- /dev/null +++ b/app/agents/fake_runtime.py @@ -0,0 +1,89 @@ +"""A scripted AgentRuntime for deterministic tests and evaluation dry runs. + +Each role has a queue of turns; a turn is a list of tool calls, or a function of the +task that returns one. Calls go through the real ToolSpecs, so every rule the tools +enforce is exercised exactly as a model would meet it. +""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from app.agents.runtime import ( + AgentTask, + BudgetExhausted, + Outcome, + RoleSpec, + Stopped, + ToolSpec, +) + + +@dataclass(frozen=True) +class Call: + tool: str + args: Mapping[str, Any] = field(default_factory=dict) + + +Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]] + + +@dataclass(frozen=True) +class RunRecord: + role: str + task: AgentTask + tools: tuple[str, ...] + results: tuple[tuple[str, str], ...] # (tool name, message) + outcome: Outcome + + +class ScriptExhausted(AssertionError): + """A role was run more times than the test scripted.""" + + +class FakeAgentRuntime: + def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None: + self._turns = {role: deque(role_turns) for role, role_turns in turns.items()} + self.runs: list[RunRecord] = [] + + def remaining(self, role: str) -> int: + return len(self._turns.get(role, ())) + + async def run( + self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec] + ) -> Outcome: + queue = self._turns.get(role.name) + if not queue: + raise ScriptExhausted(f"No scripted turn left for role {role.name!r}") + turn = queue.popleft() + calls = turn(task) if callable(turn) else turn + by_name = {tool.name: tool for tool in tools} + + results: list[tuple[str, str]] = [] + outcome: Outcome = Stopped( + "The script ended without a tool that ends the turn." + ) + for index, call in enumerate(calls): + if index >= role.budget: + outcome = BudgetExhausted( + f"{role.name} used all {role.budget} tool calls before finishing." + ) + break + if call.tool not in by_name: + raise ValueError( + f"{role.name} called {call.tool!r}, which it was not offered: " + f"{sorted(by_name)}" + ) + result = await by_name[call.tool].handler(**call.args) + results.append((call.tool, result.message)) + if result.outcome is not None: + outcome = result.outcome + break + + self.runs.append( + RunRecord(role.name, task, tuple(by_name), tuple(results), outcome) + ) + return outcome diff --git a/app/agents/runtime.py b/app/agents/runtime.py new file mode 100644 index 0000000..8c810ca --- /dev/null +++ b/app/agents/runtime.py @@ -0,0 +1,103 @@ +"""The agent-runtime interface (ADR-0003). + +Everything an Orchestrator or Specialist needs crosses this seam as plain data: +a RoleSpec, an AgentTask and ToolSpecs in; an Outcome back. No framework types. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol + + +@dataclass(frozen=True) +class RoleSpec: + """One agent role, parsed from agents/.md.""" + + name: str + instructions: str + budget: int + skills: tuple[str, ...] = () + tool_tiers: tuple[str, ...] = () + mcp_servers: tuple[str, ...] = () + model: str | None = None + + +@dataclass(frozen=True) +class AgentTask: + """What one run of an agent is asked to do, with the context it may use.""" + + role: str + prompt: str + context: Mapping[str, Any] = field(default_factory=dict) + deliverable: str | None = None + + +# --- outcomes ---------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Submitted: + """A Specialist's contract-complete Deliverable.""" + + content: str + attachments: Mapping[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class NeedsInput: + """The agent can't continue without the user. `permission` names a tool tier to grant.""" + + questions: tuple[str, ...] + permission: str | None = None + + +@dataclass(frozen=True) +class BudgetExhausted: + summary: str + + +@dataclass(frozen=True) +class Stopped: + """The agent stopped without calling a tool that ends its turn.""" + + message: str + + +@dataclass(frozen=True) +class TurnEnded: + """The Orchestrator ended its turn by calling `tool`; the tool recorded its effect.""" + + tool: str + + +Outcome = Submitted | NeedsInput | BudgetExhausted | Stopped | TurnEnded + + +# --- tools ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolResult: + """What a tool returns to the agent. A non-None `outcome` ends the agent's run.""" + + message: str + outcome: Outcome | None = None + + +@dataclass(frozen=True) +class ToolSpec: + name: str + description: str + parameters: Mapping[str, Any] # JSON schema for the keyword arguments + handler: Callable[..., Awaitable[ToolResult]] + tier: str = "general" # general | cloud_read | execution + + +class AgentRuntime(Protocol): + async def run( + self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec] + ) -> Outcome: + """Run one ReAct loop until a tool ends the turn, the budget is spent, or it stops.""" + ... diff --git a/app/config.py b/app/config.py index 940fb39..263278f 100644 --- a/app/config.py +++ b/app/config.py @@ -53,6 +53,14 @@ class Settings(BaseSettings): CLOUD_PROVIDER: str = "gcp" MCP_ENABLE_PREVIEW: bool = True + # Hierarchical Execution Engine (LangGraph state, ReAct agents) + ORCHESTRATOR_BUDGET: int = 40 + SPECIALIST_BUDGET: int = 15 + MAX_REVISIONS: int = 3 + MAX_ORCHESTRATOR_TURNS: int = 100 + CHECKPOINTER: str = "sqlite" # memory | sqlite | postgres + CHECKPOINT_DB_PATH: str = "checkpoints.db" + # Google Developer Knowledge MCP Server Settings DEVELOPER_KNOWLEDGE_MCP_URL: str = "https://developerknowledge.googleapis.com/mcp" DEVELOPER_KNOWLEDGE_MCP_ENABLED: bool = True diff --git a/app/execution/checkpointing.py b/app/execution/checkpointing.py new file mode 100644 index 0000000..f44491c --- /dev/null +++ b/app/execution/checkpointing.py @@ -0,0 +1,38 @@ +"""The configured LangGraph checkpointer that makes paused Executions resumable.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from langgraph.checkpoint.base import BaseCheckpointSaver +from langgraph.checkpoint.memory import InMemorySaver + +from app.config import Settings + + +@asynccontextmanager +async def open_checkpointer(settings: Settings) -> AsyncIterator[BaseCheckpointSaver]: + """Yield a checkpointer per CHECKPOINTER: memory | sqlite | postgres.""" + kind = settings.CHECKPOINTER + if kind == "memory": + yield InMemorySaver() + elif kind == "sqlite": + from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + + async with AsyncSqliteSaver.from_conn_string( + settings.CHECKPOINT_DB_PATH + ) as saver: + yield saver + elif kind == "postgres": + from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver + + async with AsyncPostgresSaver.from_conn_string( + settings.get_database_url() + ) as saver: + await saver.setup() + yield saver + else: + raise ValueError( + f"CHECKPOINTER must be memory, sqlite or postgres, not {kind!r}" + ) diff --git a/app/execution/engine.py b/app/execution/engine.py new file mode 100644 index 0000000..0ebda5b --- /dev/null +++ b/app/execution/engine.py @@ -0,0 +1,145 @@ +"""ExecutionEngine: start and resume Executions. + +The one interface the A2A server, /generate and the evaluation harness use. Each +call runs the Execution until it needs the user or finishes, and returns an +Interaction describing which. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from langchain_core.runnables import RunnableConfig +from langgraph.checkpoint.base import BaseCheckpointSaver +from langgraph.types import Command + +from app.agents.runtime import AgentRuntime, RoleSpec +from app.execution.deliverables import DEPENDENCY_GRAPH +from app.execution.graph import ToolFactory, build_execution_graph +from app.execution.ledger import Ledger +from app.execution.recorder import ExecutionRecorder +from app.execution.tools import ORCHESTRATOR + +_RECURSION_LIMIT = 10_000 # loops are bounded by max_orchestrator_turns instead + + +@dataclass(frozen=True) +class Interaction: + execution_id: str + status: str # awaiting_user | completed | completed_with_warnings + prompt: Mapping[str, Any] | None # questions | approval | permission + state: Mapping[str, Any] + + +class ExecutionNotAwaitingUser(RuntimeError): + pass + + +class ExecutionEngine: + def __init__( + self, + *, + runtime: AgentRuntime, + roles: Mapping[str, RoleSpec], + recorder: ExecutionRecorder, + checkpointer: BaseCheckpointSaver, + output_root: Path, + max_revisions: int = 3, + max_orchestrator_turns: int = 100, + tool_factory: ToolFactory | None = None, + ) -> None: + required = { + ORCHESTRATOR, + *(spec.specialist for spec in DEPENDENCY_GRAPH.values()), + } + missing = sorted(required - set(roles)) + if missing: + raise ValueError(f"Missing roles: {', '.join(missing)}") + self._max_revisions = max_revisions + self._graph = build_execution_graph( + runtime=runtime, + roles=roles, + recorder=recorder, + output_root=output_root, + max_orchestrator_turns=max_orchestrator_turns, + tool_factory=tool_factory, + ).compile(checkpointer=checkpointer) + + async def start( + self, + workflow_request: str, + *, + session_id: str | None = None, + pre_approved: bool = False, + allow_cloud_read: bool | None = None, + ) -> Interaction: + execution_id = str(uuid.uuid4()) + if pre_approved and allow_cloud_read is None: + allow_cloud_read = False + initial: dict[str, Any] = { + "execution_id": execution_id, + "session_id": session_id or str(uuid.uuid4()), + "workflow_request": workflow_request, + "pre_approved": pre_approved, + "allow_cloud_read": allow_cloud_read, + "ledger": Ledger( + pre_approved=pre_approved, max_revisions=self._max_revisions + ).to_dict(), + "deliverables": {}, + "conversation": [{"from": "user", "text": workflow_request}], + "notes": [], + "pending_questions": [], + "awaiting_questions": [], + "answers": {}, + "pending_permission": None, + "prompt": None, + "outcomes": [], + "outcomes_applied": 0, + "events_recorded": 0, + "turns": 0, + "status": "running", + "warnings": [], + } + await self._graph.ainvoke(initial, self._config(execution_id)) + return await self.interaction(execution_id) + + async def resume(self, execution_id: str, reply: Mapping[str, Any]) -> Interaction: + """Answer the Execution's current prompt: {"reply"} | {"approval"} | {"permission"}.""" + current = await self.interaction(execution_id) + if current.status != "awaiting_user": + raise ExecutionNotAwaitingUser( + f"Execution {execution_id} is {current.status}, not awaiting the user" + ) + await self._graph.ainvoke( + Command(resume=dict(reply)), self._config(execution_id) + ) + return await self.interaction(execution_id) + + async def interaction(self, execution_id: str) -> Interaction: + snapshot = await self._graph.aget_state(self._config(execution_id)) + if not snapshot.values: + raise KeyError(f"Unknown Execution {execution_id}") + if snapshot.interrupts: + return Interaction( + execution_id, + "awaiting_user", + snapshot.interrupts[0].value, + snapshot.values, + ) + return Interaction( + execution_id, + snapshot.values.get("status", "running"), + None, + snapshot.values, + ) + + @staticmethod + def _config(execution_id: str) -> RunnableConfig: + return { + "configurable": {"thread_id": execution_id}, + "recursion_limit": _RECURSION_LIMIT, + } diff --git a/app/execution/graph.py b/app/execution/graph.py new file mode 100644 index 0000000..debc920 --- /dev/null +++ b/app/execution/graph.py @@ -0,0 +1,478 @@ +"""The LangGraph that holds an Execution (ADR-0003). + + START -> orchestrator_turn + orchestrator_turn -> specialist_task x N (Send, parallel) -> orchestrator_turn + orchestrator_turn -> await_user (interrupt) -> orchestrator_turn + orchestrator_turn -> finalize -> END + orchestrator_turn -> end_with_warnings -> END + +Only orchestrator_turn and await_user change the Ledger; parallel Specialist tasks +append outcomes that the next Orchestrator turn applies in order. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any, cast + +from langgraph.graph import END, START, StateGraph +from langgraph.types import Command, Send, interrupt + +from app.agents.runtime import ( + AgentRuntime, + AgentTask, + BudgetExhausted, + NeedsInput, + Outcome, + RoleSpec, + Stopped, + Submitted, + ToolSpec, + TurnEnded, +) +from app.execution.deliverables import DEPENDENCY_GRAPH, PHASE_ORDER, DeliverableKind +from app.execution.ledger import Ledger, LedgerRefusal +from app.execution.recorder import ExecutionRecorder +from app.execution.state import ExecutionState +from app.execution.tools import ORCHESTRATOR, OrchestratorToolbox, specialist_tools + +ToolFactory = Callable[[RoleSpec, str | None], Sequence[ToolSpec]] + +NO_USER_REPLY = ( + "No user is available. State your assumption, record it in the Deliverable under " + "'Assumptions', and proceed." +) +GUIDE_FILENAME = "solution-architecture-guide.md" + + +def _no_tools(role: RoleSpec, deliverable: str | None) -> Sequence[ToolSpec]: + return () + + +def build_execution_graph( + *, + runtime: AgentRuntime, + roles: Mapping[str, RoleSpec], + recorder: ExecutionRecorder, + output_root: Path, + max_orchestrator_turns: int = 100, + tool_factory: ToolFactory | None = None, +) -> StateGraph: + tool_factory = tool_factory or _no_tools + + async def record_progress( + state: ExecutionState, ledger: Ledger, status: str | None = None + ) -> int: + recorded = state.get("events_recorded", 0) + await recorder.record_events(state["execution_id"], ledger.events[recorded:]) + if status: + await recorder.record_status( + state["execution_id"], + session_id=state["session_id"], + workflow_request=state["workflow_request"], + status=status, + current_phase=_current_phase(ledger), + details={ + "revisions": sum( + ledger.revisions(kind) for kind in DEPENDENCY_GRAPH + ), + "warnings": state.get("warnings", []), + }, + ) + return len(ledger.events) + + async def orchestrator_turn(state: ExecutionState) -> Command: + ledger = Ledger.from_dict(state["ledger"]) + deliverables = dict(state.get("deliverables", {})) + pending_questions = list(state.get("pending_questions", [])) + notes = list(state.get("notes", [])) + pending_permission = state.get("pending_permission") + warnings = list(state.get("warnings", [])) + + outcomes = state.get("outcomes", []) + for record in outcomes[state.get("outcomes_applied", 0) :]: + kind = DeliverableKind(record["kind"]) + try: + if record["type"] == "submitted": + ledger.submit(kind, record["attempt"], actor=record["role"]) + deliverables[kind.value] = { + "content": record["content"], + "attachments": record["attachments"], + } + await recorder.record_submission( + state["execution_id"], + kind.value, + record["content"], + record["attachments"], + ) + continue + ledger.release(kind, record["attempt"], reason=record["type"]) + except LedgerRefusal as exc: + notes.append(f"Discarded a superseded result for {kind}: {exc}") + continue + if record["type"] == "needs_input" and record.get("permission"): + if state.get("allow_cloud_read") is None: + requested = (pending_permission or {}).get("requested_by", []) + pending_permission = { + "tier": record["permission"], + "requested_by": [*requested, kind.value], + } + continue + questions = record.get("questions") or [ + ( + f"The {record['role']} stopped: {record['summary']} " + "How should it proceed?" + ) + ] + pending_questions.append({"kind": kind.value, "questions": questions}) + + turns = state.get("turns", 0) + 1 + update: dict[str, Any] = { + "deliverables": deliverables, + "pending_questions": pending_questions, + "pending_permission": pending_permission, + "outcomes_applied": len(outcomes), + "turns": turns, + } + + def finish_with(goto: str, **extra: Any) -> Command: + return Command( + update={**update, "ledger": ledger.to_dict(), "notes": notes, **extra}, + goto=goto, + ) + + if pending_permission and not state.get("pre_approved"): + update["events_recorded"] = await record_progress( + state, ledger, "awaiting_user" + ) + return finish_with("await_user", prompt={"permission": pending_permission}) + + if turns > max_orchestrator_turns: + warnings.append( + f"Stopped after {max_orchestrator_turns} Orchestrator turns without " + "finishing." + ) + return finish_with("end_with_warnings", warnings=warnings) + + toolbox = OrchestratorToolbox(ledger, deliverables, pending_questions) + task = AgentTask( + role=ORCHESTRATOR, + prompt="Decide the next step for this Execution using your tools.", + context=_orchestrator_context(state, ledger, pending_questions, notes), + ) + outcome = await runtime.run(roles[ORCHESTRATOR], task, toolbox.tools()) + notes = [] + pre_approved = bool(state.get("pre_approved")) + conversation = list(state.get("conversation", [])) + + def ask(prompt: dict[str, Any], **extra: Any) -> Command: + conversation.append({"from": ORCHESTRATOR, "text": _render_prompt(prompt)}) + return finish_with( + "await_user", prompt=prompt, conversation=conversation, **extra + ) + + if isinstance(outcome, TurnEnded) and outcome.tool == "delegate": + sends = [ + Send( + "specialist_task", + _specialist_input(state, ledger, deliverables, kind, attempt), + ) + for kind, attempt in toolbox.started.items() + ] + update["events_recorded"] = await record_progress(state, ledger) + return Command( + update={**update, "ledger": ledger.to_dict(), "notes": notes}, + goto=sends, + ) + + if isinstance(outcome, TurnEnded) and outcome.tool == "finish": + update["events_recorded"] = await record_progress(state, ledger) + return finish_with("finalize") + + if isinstance(outcome, TurnEnded) and outcome.tool == "escalate": + warnings.append(toolbox.escalation or "Escalated without detail.") + update["events_recorded"] = await record_progress(state, ledger) + return finish_with("end_with_warnings", warnings=warnings) + + if isinstance(outcome, TurnEnded) and outcome.tool == "request_approval": + update["events_recorded"] = await record_progress( + state, ledger, None if pre_approved else "awaiting_user" + ) + return ask({"approval": toolbox.approval}) + + if isinstance(outcome, TurnEnded) and outcome.tool == "ask_user": + update["events_recorded"] = await record_progress( + state, ledger, None if pre_approved else "awaiting_user" + ) + return ask( + {"questions": toolbox.questions}, + pending_questions=[], + awaiting_questions=pending_questions, + ) + + message = _describe_unfinished(outcome) + if pre_approved: + warnings.append(f"The Orchestrator could not continue: {message}") + update["events_recorded"] = await record_progress(state, ledger) + return finish_with("end_with_warnings", warnings=warnings) + update["events_recorded"] = await record_progress( + state, ledger, "awaiting_user" + ) + return ask({"questions": [{"from": ORCHESTRATOR, "question": message}]}) + + async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]: + role = roles[task_input["role"]] + kind = DeliverableKind(task_input["kind"]) + tools = specialist_tools( + kind, + extra=tool_factory(role, kind.value), + allow_cloud_read=task_input["allow_cloud_read"], + ) + task = AgentTask( + role=role.name, + prompt=f"Produce the {kind} Deliverable.", + context=task_input["context"], + deliverable=kind.value, + ) + outcome = await runtime.run(role, task, tools) + return { + "outcomes": [ + _outcome_record(role.name, kind, task_input["attempt"], outcome) + ] + } + + async def await_user(state: ExecutionState) -> Command: + prompt = state["prompt"] or {} + if state.get("pre_approved"): + reply = _automatic_reply(prompt) + else: + reply = interrupt(prompt) + + ledger = Ledger.from_dict(state["ledger"]) + conversation = list(state.get("conversation", [])) + update: dict[str, Any] = {"prompt": None} + + if "approval" in prompt: + phase = prompt["approval"]["phase"] + decision = reply.get("approval") if isinstance(reply, Mapping) else None + if ( + isinstance(decision, Mapping) + and decision.get("phase") == phase + and decision.get("decision") in {"approve", "changes"} + ): + phase_id = next(p for p in PHASE_ORDER if p.value == phase) + if decision["decision"] == "approve": + ledger.approve(phase_id, actor="user") + text = f"Approved {phase}." + else: + feedback = str(decision.get("feedback", "")) + ledger.request_changes(phase_id, feedback, actor="user") + text = f"Requested changes to {phase}: {feedback}" + conversation.append({"from": "user", "text": text}) + else: + conversation.append({"from": "user", "text": _reply_text(reply)}) + elif "permission" in prompt: + tier = prompt["permission"]["tier"] + permission = reply.get("permission") if isinstance(reply, Mapping) else None + grant = permission.get(tier) if isinstance(permission, Mapping) else None + if isinstance(grant, bool): + update["allow_cloud_read"] = grant + update["pending_permission"] = None + verdict = "granted" if grant else "denied" + conversation.append( + { + "from": "user", + "text": f"Permission to use {tier} tools {verdict}.", + } + ) + else: + conversation.append({"from": "user", "text": _reply_text(reply)}) + else: + text = _reply_text(reply) + conversation.append({"from": "user", "text": text}) + answers = { + kind: list(items) for kind, items in state.get("answers", {}).items() + } + for asked in state.get("awaiting_questions", []): + answers.setdefault(asked["kind"], []).append( + {"questions": list(asked["questions"]), "reply": text} + ) + update.update(answers=answers, awaiting_questions=[]) + + update["events_recorded"] = await record_progress(state, ledger) + update.update(ledger=ledger.to_dict(), conversation=conversation) + return Command(update=update, goto="orchestrator_turn") + + async def finalize(state: ExecutionState) -> dict[str, Any]: + ledger = Ledger.from_dict(state["ledger"]) + _write_outputs( + output_root / "deliverables" / "executions" / state["execution_id"], + state.get("deliverables", {}), + ) + await record_progress(state, ledger, "completed") + return {"status": "completed"} + + async def end_with_warnings(state: ExecutionState) -> dict[str, Any]: + ledger = Ledger.from_dict(state["ledger"]) + await record_progress(state, ledger, "completed_with_warnings") + return {"status": "completed_with_warnings"} + + graph = StateGraph(ExecutionState) + graph.add_node("orchestrator_turn", orchestrator_turn) + # Specialist tasks receive their Send payload, not the Execution state. + graph.add_node("specialist_task", cast(Any, specialist_task)) + graph.add_node("await_user", await_user) + graph.add_node("finalize", finalize) + graph.add_node("end_with_warnings", end_with_warnings) + graph.add_edge(START, "orchestrator_turn") + graph.add_edge("specialist_task", "orchestrator_turn") + graph.add_edge("finalize", END) + graph.add_edge("end_with_warnings", END) + return graph + + +# --- helpers --------------------------------------------------------------------------- + + +def _current_phase(ledger: Ledger) -> str: + for phase in PHASE_ORDER: + if ledger.phase_status(phase).value != "approved": + return phase.value + return PHASE_ORDER[-1].value + + +def _orchestrator_context( + state: ExecutionState, + ledger: Ledger, + pending_questions: Sequence[Mapping[str, Any]], + notes: Sequence[str], +) -> dict[str, Any]: + return { + "workflow_request": state["workflow_request"], + "pre_approved": bool(state.get("pre_approved")), + "allow_cloud_read": state.get("allow_cloud_read"), + "deliverables": { + kind.value: { + "phase": spec.phase.value, + "status": ledger.status(kind).value, + "revisions": ledger.revisions(kind), + "feedback": list(ledger.feedback(kind)), + } + for kind, spec in DEPENDENCY_GRAPH.items() + }, + "phases": { + phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER + }, + "ready_work": [kind.value for kind in ledger.ready_work()], + "phases_ready_for_approval": [ + p.value for p in ledger.phases_ready_for_approval() + ], + "pending_questions": list(pending_questions), + "conversation": list(state.get("conversation", []))[-20:], + "notes": list(notes), + } + + +def _specialist_input( + state: ExecutionState, + ledger: Ledger, + deliverables: Mapping[str, Mapping[str, Any]], + kind: DeliverableKind, + attempt: int, +) -> dict[str, Any]: + spec = DEPENDENCY_GRAPH[kind] + return { + "role": spec.specialist, + "kind": kind.value, + "attempt": attempt, + "allow_cloud_read": state.get("allow_cloud_read"), + "context": { + "workflow_request": state["workflow_request"], + "pre_approved": bool(state.get("pre_approved")), + "dependencies": { + dependency.value: dict(deliverables[dependency.value]) + for dependency in spec.depends_on + }, + "feedback": list(ledger.feedback(kind)), + "answers": list(state.get("answers", {}).get(kind.value, [])), + "attempt": attempt, + }, + } + + +def _outcome_record( + role: str, kind: DeliverableKind, attempt: int, outcome: Outcome +) -> dict[str, Any]: + base = {"role": role, "kind": kind.value, "attempt": attempt} + if isinstance(outcome, Submitted): + return { + **base, + "type": "submitted", + "content": outcome.content, + "attachments": dict(outcome.attachments), + } + if isinstance(outcome, NeedsInput): + return { + **base, + "type": "needs_input", + "questions": list(outcome.questions), + "permission": outcome.permission, + } + if isinstance(outcome, BudgetExhausted): + return {**base, "type": "budget_exhausted", "summary": outcome.summary} + return {**base, "type": "stopped", "summary": _describe_unfinished(outcome)} + + +def _describe_unfinished(outcome: Outcome) -> str: + if isinstance(outcome, BudgetExhausted): + return f"I paused: {outcome.summary} How should I proceed?" + if isinstance(outcome, Stopped): + return outcome.message + if isinstance(outcome, NeedsInput): + return " ".join(outcome.questions) + return f"Ended with an unexpected outcome: {type(outcome).__name__}." + + +def _automatic_reply(prompt: Mapping[str, Any]) -> dict[str, Any]: + if "approval" in prompt: + return { + "approval": {"phase": prompt["approval"]["phase"], "decision": "approve"} + } + if "permission" in prompt: + return {"permission": {prompt["permission"]["tier"]: False}} + return {"reply": NO_USER_REPLY} + + +def _reply_text(reply: object) -> str: + if isinstance(reply, str): + return reply + if isinstance(reply, Mapping) and isinstance(reply.get("reply"), str): + return reply["reply"] + return str(reply) + + +def _render_prompt(prompt: Mapping[str, Any]) -> str: + if "approval" in prompt: + approval = prompt["approval"] + return f"Please approve {approval['phase']}: {approval['summary']}" + if "permission" in prompt: + return f"May I use {prompt['permission']['tier']} tools?" + return "\n".join(item["question"] for item in prompt.get("questions", [])) + + +def _write_outputs( + out_dir: Path, deliverables: Mapping[str, Mapping[str, Any]] +) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value) + if guide: + (out_dir / GUIDE_FILENAME).write_text(guide["content"], encoding="utf-8") + for kind, deliverable in deliverables.items(): + path = out_dir / "deliverables" / f"{kind}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(deliverable["content"], encoding="utf-8") + for name, content in deliverable.get("attachments", {}).items(): + attachment = out_dir / name + attachment.parent.mkdir(parents=True, exist_ok=True) + attachment.write_text(content, encoding="utf-8") diff --git a/app/execution/recorder.py b/app/execution/recorder.py new file mode 100644 index 0000000..bdcbf64 --- /dev/null +++ b/app/execution/recorder.py @@ -0,0 +1,156 @@ +"""Recording an Execution's progress for audit and reporting. + +Two adapters sit behind ExecutionRecorder: InMemoryRecorder for tests, and +DatabaseRecorder, which reuses the existing DatabaseManager tables until the +Execution store (review candidate 03) replaces them. +""" + +from __future__ import annotations + +import asyncio +import uuid +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any, Protocol + +from app.execution.ledger import LedgerEvent + +SESSION_AGENT_NAME = "solution-architect" + + +class ExecutionRecorder(Protocol): + async def record_events( + self, execution_id: str, events: Sequence[LedgerEvent] + ) -> None: ... + + async def record_submission( + self, + execution_id: str, + kind: str, + content: str, + attachments: Mapping[str, str], + ) -> None: ... + + async def record_status( + self, + execution_id: str, + *, + session_id: str, + workflow_request: str, + status: str, + current_phase: str, + details: Mapping[str, Any], + ) -> None: ... + + +class InMemoryRecorder: + def __init__(self) -> None: + self.events: dict[str, list[LedgerEvent]] = defaultdict(list) + self.submissions: dict[str, list[dict[str, Any]]] = defaultdict(list) + self.statuses: dict[str, list[dict[str, Any]]] = defaultdict(list) + + async def record_events( + self, execution_id: str, events: Sequence[LedgerEvent] + ) -> None: + self.events[execution_id].extend(events) + + async def record_submission( + self, + execution_id: str, + kind: str, + content: str, + attachments: Mapping[str, str], + ) -> None: + self.submissions[execution_id].append( + {"kind": kind, "content": content, "attachments": dict(attachments)} + ) + + async def record_status( + self, + execution_id: str, + *, + session_id: str, + workflow_request: str, + status: str, + current_phase: str, + details: Mapping[str, Any], + ) -> None: + self.statuses[execution_id].append( + { + "session_id": session_id, + "workflow_request": workflow_request, + "status": status, + "current_phase": current_phase, + "details": dict(details), + } + ) + + +class DatabaseRecorder: + """Writes to DatabaseManager (blocking) off the event loop.""" + + def __init__(self, db_manager: Any = None) -> None: + if db_manager is None: + from app.database import get_db_manager + + db_manager = get_db_manager() + self._db = db_manager + + async def record_events( + self, execution_id: str, events: Sequence[LedgerEvent] + ) -> None: + for event in events: + await asyncio.to_thread( + self._db.record_orchestrator_log, + log_id=str(uuid.uuid4()), + execution_id=execution_id, + iteration=event.sequence, + review_status=event.type, + reviewer_agent=event.actor, + feedback=f"{event.subject}: {event.detail}".rstrip(": "), + ) + + async def record_submission( + self, + execution_id: str, + kind: str, + content: str, + attachments: Mapping[str, str], + ) -> None: + files = {f"deliverables/{kind}.md": content, **attachments} + for file_path, file_content in files.items(): + await asyncio.to_thread( + self._db.save_artifact, + artifact_id=str(uuid.uuid4()), + execution_id=execution_id, + artifact_type=kind, + file_path=file_path, + content=file_content, + ) + + async def record_status( + self, + execution_id: str, + *, + session_id: str, + workflow_request: str, + status: str, + current_phase: str, + details: Mapping[str, Any], + ) -> None: + await asyncio.to_thread( + self._db.save_workflow_execution, + execution_id=execution_id, + session_id=session_id, + status=status, + current_phase=current_phase, + loop_count=int(details.get("revisions", 0)), + request_summary=workflow_request, + results=dict(details), + ) + await asyncio.to_thread( + self._db.save_session, + session_id=session_id, + agent_name=SESSION_AGENT_NAME, + state_data={"latest_execution_id": execution_id, "status": status}, + ) diff --git a/app/execution/state.py b/app/execution/state.py new file mode 100644 index 0000000..902aa93 --- /dev/null +++ b/app/execution/state.py @@ -0,0 +1,35 @@ +"""The checkpointed state of one Execution. Plain, JSON-compatible data only.""" + +from __future__ import annotations + +import operator +from typing import Annotated, Any, TypedDict + + +class ExecutionState(TypedDict, total=False): + execution_id: str + session_id: str + workflow_request: str + pre_approved: bool + allow_cloud_read: bool | None + + ledger: dict[str, Any] # Ledger.to_dict() + deliverables: dict[str, dict[str, Any]] # kind -> {content, attachments} + + conversation: list[dict[str, str]] # {"from": "user" | "orchestrator", "text"} + notes: list[str] # facts for the Orchestrator's next turn + pending_questions: list[dict[str, Any]] # {kind, questions} not yet asked + awaiting_questions: list[dict[str, Any]] # {kind, questions} asked, not answered + answers: dict[str, list[dict[str, Any]]] # kind -> [{questions, reply}] + pending_permission: dict[str, Any] | None # {tier, requested_by} + prompt: dict[str, Any] | None # what the user is being asked + + # Specialist outcomes, appended concurrently by parallel Specialist tasks and + # applied to the Ledger by the next Orchestrator turn. + outcomes: Annotated[list[dict[str, Any]], operator.add] + outcomes_applied: int + events_recorded: int + + turns: int + status: str # running | completed | completed_with_warnings + warnings: list[str] diff --git a/app/execution/tools.py b/app/execution/tools.py new file mode 100644 index 0000000..1488a06 --- /dev/null +++ b/app/execution/tools.py @@ -0,0 +1,318 @@ +"""Execution tools: how agents act on an Execution. + +Tools enforce the Ledger's rules at the moment an agent acts, and return refusals as +messages so a ReAct agent can recover. A tool ends the agent's turn by returning an +outcome. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import PurePosixPath +from typing import Any + +from app.agents.runtime import ( + NeedsInput, + Submitted, + ToolResult, + ToolSpec, + TurnEnded, +) +from app.execution.deliverables import ( + DEPENDENCY_GRAPH, + DeliverableKind, + PhaseId, + check_submission, + deliverables_in, +) +from app.execution.ledger import Ledger, LedgerRefusal, PhaseStatus + +ORCHESTRATOR = "orchestrator" +CLOUD_READ = "cloud_read" + +_NO_ARGS: Mapping[str, Any] = {"type": "object", "properties": {}} + + +def _schema(properties: Mapping[str, Any], required: Sequence[str]) -> dict[str, Any]: + return { + "type": "object", + "properties": dict(properties), + "required": list(required), + } + + +def _refused(reason: object) -> ToolResult: + return ToolResult(f"Refused: {reason}") + + +class OrchestratorToolbox: + """The Orchestrator's tools, bound to a working copy of the Ledger for one turn.""" + + def __init__( + self, + ledger: Ledger, + deliverables: Mapping[str, Mapping[str, Any]], + pending_questions: Sequence[Mapping[str, Any]], + ) -> None: + self.ledger = ledger + self._deliverables = deliverables + self._pending_questions = pending_questions + self.started: dict[DeliverableKind, int] = {} + self.questions: list[dict[str, str]] = [] + self.approval: dict[str, Any] | None = None + self.escalation: str | None = None + + def tools(self) -> list[ToolSpec]: + return [ + ToolSpec( + "ready_work", + "List Deliverables ready to delegate and Phases ready for Approval.", + _NO_ARGS, + self.ready_work, + tier="execution", + ), + ToolSpec( + "delegate", + "Start Specialists on ready Deliverables, in parallel. Ends your turn.", + _schema( + {"kinds": {"type": "array", "items": {"type": "string"}}}, + ["kinds"], + ), + self.delegate, + tier="execution", + ), + ToolSpec( + "ask_user", + "Ask the user your questions together with any waiting Specialist " + "questions. Ends your turn.", + _schema( + {"questions": {"type": "array", "items": {"type": "string"}}}, [] + ), + self.ask_user, + tier="execution", + ), + ToolSpec( + "request_approval", + "Ask the user to approve a Phase whose Deliverables are all " + "submitted. Ends your turn unless the Execution is pre-approved.", + _schema( + {"phase": {"type": "string"}, "summary": {"type": "string"}}, + ["phase", "summary"], + ), + self.request_approval, + tier="execution", + ), + ToolSpec( + "reopen", + "Revise a submitted Deliverable; everything depending on it returns " + "to draft.", + _schema( + {"kind": {"type": "string"}, "reason": {"type": "string"}}, + ["kind", "reason"], + ), + self.reopen, + tier="execution", + ), + ToolSpec( + "read_deliverable", + "Read a submitted Deliverable's content.", + _schema({"kind": {"type": "string"}}, ["kind"]), + self.read_deliverable, + tier="execution", + ), + ToolSpec( + "finish", + "Complete the Execution once packaging is approved. Ends your turn.", + _NO_ARGS, + self.finish, + tier="execution", + ), + ] + + async def ready_work(self) -> ToolResult: + ready = ", ".join(self.ledger.ready_work()) or "none" + phases = ", ".join(self.ledger.phases_ready_for_approval()) or "none" + return ToolResult( + f"Ready to delegate: {ready}. Phases ready for approval: {phases}." + ) + + async def delegate(self, kinds: Sequence[str]) -> ToolResult: + try: + parsed = [DeliverableKind(kind) for kind in kinds] + if not parsed: + raise LedgerRefusal("name at least one Deliverable") + started = self.ledger.start(parsed, actor=ORCHESTRATOR) + except (ValueError, LedgerRefusal) as exc: + return _refused(exc) + self.started.update(started) + return ToolResult(f"Started: {', '.join(started)}.", TurnEnded("delegate")) + + async def ask_user(self, questions: Sequence[str] = ()) -> ToolResult: + waiting = sorted( + self._pending_questions, + key=lambda item: list(DEPENDENCY_GRAPH).index( + DeliverableKind(item["kind"]) + ), + ) + combined = [ + {"from": item["kind"], "question": question} + for item in waiting + for question in item["questions"] + ] + combined += [{"from": ORCHESTRATOR, "question": q} for q in questions if q] + if not combined: + return _refused("there is nothing to ask the user") + self.questions = combined + return ToolResult("Asked the user.", TurnEnded("ask_user")) + + async def request_approval(self, phase: str, summary: str) -> ToolResult: + try: + phase_id = PhaseId(phase) + except ValueError as exc: + return _refused(exc) + if self.ledger.phase_status(phase_id) is not PhaseStatus.AWAITING_APPROVAL: + try: + self.ledger.request_approval(phase_id, actor=ORCHESTRATOR) + except LedgerRefusal as exc: + return _refused(exc) + if self.ledger.phase_status(phase_id) is PhaseStatus.APPROVED: + return ToolResult( + f"Phase {phase_id} approved (pre-approved Execution)." + ) + self.approval = { + "phase": phase_id.value, + "summary": summary, + "deliverables": [kind.value for kind in deliverables_in(phase_id)], + } + return ToolResult( + f"Asked the user to approve {phase_id}.", TurnEnded("request_approval") + ) + + async def reopen(self, kind: str, reason: str) -> ToolResult: + try: + result = self.ledger.reopen( + DeliverableKind(kind), reason, actor=ORCHESTRATOR + ) + except (ValueError, LedgerRefusal) as exc: + return _refused(exc) + if result.escalation: + if self.ledger.pre_approved: + self.escalation = result.escalation + return ToolResult(result.escalation, TurnEnded("escalate")) + return ToolResult( + f"{result.escalation} Use ask_user to put this to the user." + ) + withdrawn = ", ".join(result.withdrawn_approvals) or "none" + return ToolResult( + f"Reopened {kind}. Returned to draft: {', '.join(result.reset)}. " + f"Approvals withdrawn: {withdrawn}." + ) + + async def read_deliverable(self, kind: str) -> ToolResult: + deliverable = self._deliverables.get(kind) + if deliverable is None: + return ToolResult(f"{kind} has not been submitted.") + return ToolResult(deliverable["content"]) + + async def finish(self) -> ToolResult: + if not self.ledger.can_finish(): + return _refused("packaging has not been approved") + return ToolResult("Finishing the Execution.", TurnEnded("finish")) + + +def _safe_relative(name: str) -> bool: + path = PurePosixPath(name) + return bool(name) and not path.is_absolute() and ".." not in path.parts + + +def specialist_tools( + kind: DeliverableKind, + extra: Sequence[ToolSpec], + allow_cloud_read: bool | None, +) -> list[ToolSpec]: + """A Specialist's tools: submit, ask, and its role's tools with the cloud-read gate.""" + + async def submit_deliverable( + content: str, attachments: Mapping[str, str] | None = None + ) -> ToolResult: + attachments = dict(attachments or {}) + unsafe = [name for name in attachments if not _safe_relative(name)] + if unsafe: + return _refused( + "attachment names must be relative paths without '..': " + + ", ".join(unsafe) + ) + missing = check_submission(kind, content, attachments) + if missing: + return ToolResult( + "Not submitted. The Deliverable is missing: " + "; ".join(missing) + ) + return ToolResult(f"Submitted {kind}.", Submitted(content, attachments)) + + async def needs_input(questions: Sequence[str]) -> ToolResult: + questions = tuple(q for q in questions if q) + if not questions: + return _refused("ask at least one question") + return ToolResult("Questions sent to the user.", NeedsInput(questions)) + + return [ + ToolSpec( + "submit_deliverable", + f"Submit the {kind} Deliverable once it meets its contract. Ends your turn.", + _schema( + { + "content": {"type": "string"}, + "attachments": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + }, + ["content"], + ), + submit_deliverable, + tier="execution", + ), + ToolSpec( + "needs_input", + "Ask the user questions you can't proceed without. Ends your turn.", + _schema( + {"questions": {"type": "array", "items": {"type": "string"}}}, + ["questions"], + ), + needs_input, + tier="execution", + ), + *(gate_cloud_read(tool, allow_cloud_read) for tool in extra), + ] + + +def gate_cloud_read(tool: ToolSpec, allow_cloud_read: bool | None) -> ToolSpec: + """Apply the Execution's cloud-read permission to a tool in that tier.""" + if tool.tier != CLOUD_READ or allow_cloud_read is True: + return tool + + if allow_cloud_read is False: + + async def refuse(**_: Any) -> ToolResult: + return _refused( + "reading live cloud resources is not permitted for this Execution. " + "Work from what the user has told you and record the gap." + ) + + handler = refuse + else: + + async def ask_permission(**_: Any) -> ToolResult: + return ToolResult( + "Permission to read live cloud resources is needed first.", + NeedsInput( + ( + f"May I read live resources in your cloud environment? ({tool.name})", + ), + permission=CLOUD_READ, + ), + ) + + handler = ask_permission + + return ToolSpec(tool.name, tool.description, tool.parameters, handler, tool.tier) diff --git a/requirements.txt b/requirements.txt index 0e8f557..faeedfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ langchain>=0.2.0 langchain-core>=0.2.0 langchain-openai>=0.1.0 -langgraph>=0.1.0 +langgraph>=1.2 +langgraph-checkpoint-sqlite>=3.1 +langgraph-checkpoint-postgres>=3.1 +psycopg[binary]>=3.2 starlette>=0.37.0 uvicorn>=0.30.0 pydantic>=2.7.0 diff --git a/tests/execution/test_engine.py b/tests/execution/test_engine.py new file mode 100644 index 0000000..79b163c --- /dev/null +++ b/tests/execution/test_engine.py @@ -0,0 +1,628 @@ +"""The Execution engine: LangGraph state, Specialist fan-out, interrupts and recording. + +Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime. +""" + +from pathlib import Path + +import pytest +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + +from app.agents.fake_runtime import Call, FakeAgentRuntime +from app.agents.runtime import RoleSpec, ToolResult, ToolSpec +from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId +from app.execution.engine import ExecutionEngine +from app.execution.recorder import InMemoryRecorder + +K = DeliverableKind +P = PhaseId + +pytestmark = pytest.mark.asyncio + +ROLES = { + "orchestrator": RoleSpec("orchestrator", "Coordinate the Execution.", budget=40), + **{ + spec.specialist: RoleSpec(spec.specialist, f"Produce {spec.kind}.", budget=15) + for spec in DEPENDENCY_GRAPH.values() + }, +} + + +# --- scripting helpers --------------------------------------------------------------- + + +def delegate(*kinds: DeliverableKind) -> Call: + return Call("delegate", {"kinds": [kind.value for kind in kinds]}) + + +def request_approval(phase: PhaseId) -> Call: + return Call("request_approval", {"phase": phase.value, "summary": f"{phase} done"}) + + +def ask_user(*questions: str) -> Call: + return Call("ask_user", {"questions": list(questions)}) + + +def reopen(kind: DeliverableKind, reason: str) -> Call: + return Call("reopen", {"kind": kind.value, "reason": reason}) + + +FINISH = Call("finish") + + +def submission(kind: DeliverableKind, note: str = "") -> Call: + spec = DEPENDENCY_GRAPH[kind] + body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections) + content = f"# {kind}\n{note or 'Summary.'}\n{body}\n" + attachments = {name: f"# {name}\n" for name in spec.attachments} + return Call("submit_deliverable", {"content": content, "attachments": attachments}) + + +def submits(kind: DeliverableKind) -> list[Call]: + return [submission(kind)] + + +def specialist(kind: DeliverableKind) -> str: + return DEPENDENCY_GRAPH[kind].specialist + + +def make_engine(tmp_path: Path, runtime, **overrides) -> ExecutionEngine: + options = { + "runtime": runtime, + "roles": ROLES, + "recorder": InMemoryRecorder(), + "checkpointer": InMemorySaver(), + "output_root": tmp_path, + } + options.update(overrides) + return ExecutionEngine(**options) + + +def happy_path_turns(**extra_specialist_turns) -> dict: + turns = { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], + [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], + [delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)], + [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], + [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], + [request_approval(P.PACKAGING), FINISH], + ], + **{spec.specialist: [submits(kind)] for kind, spec in DEPENDENCY_GRAPH.items()}, + } + turns.update(extra_specialist_turns) + return turns + + +# --- happy path and parallel start ----------------------------------------------------- + + +async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path): + runtime = FakeAgentRuntime(happy_path_turns()) + recorder = InMemoryRecorder() + engine = make_engine(tmp_path, runtime, recorder=recorder) + + result = await engine.start("Event-driven ingestion", pre_approved=True) + + assert result.status == "completed" + assert result.prompt is None + ledger = result.state["ledger"] + assert set(ledger["phases"].values()) == {"approved"} + assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"} + + out = tmp_path / "deliverables" / "executions" / result.execution_id + assert (out / "solution-architecture-guide.md").is_file() + for kind in K: + assert (out / "deliverables" / f"{kind}.md").is_file(), kind + assert (out / "terraform" / "main.tf").is_file() + assert (out / "architecture.mmd").is_file() + + assert recorder.statuses[result.execution_id][-1]["status"] == "completed" + assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K) + recorded_types = [e.type for e in recorder.events[result.execution_id]] + assert recorded_types.count("approved") == 4 + + +async def test_independent_specialists_run_together_between_orchestrator_turns( + tmp_path, +): + runtime = FakeAgentRuntime(happy_path_turns()) + + await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) + + roles = [run.role for run in runtime.runs] + fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3] + assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == { + specialist(K.ARCHITECTURE_DIAGRAM), + specialist(K.DESIGN_RECOMMENDATIONS), + } + assert roles[fourth_turn + 3] == "orchestrator" + + +async def test_specialist_receives_its_dependencies_content(tmp_path): + runtime = FakeAgentRuntime(happy_path_turns()) + + await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) + + describer_run = next( + r for r in runtime.runs if r.role == specialist(K.ARCHITECTURE_DESCRIPTION) + ) + dependencies = describer_run.task.context["dependencies"] + assert set(dependencies) == {"architecture_diagram", "product_selection"} + assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"] + assert describer_run.task.deliverable == "architecture_description" + + +# --- refusals reach the agent ------------------------------------------------------------ + + +async def test_orchestrator_cannot_delegate_work_whose_dependencies_are_unmet(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.PRODUCT_SELECTION), delegate(K.REQUIREMENTS)], + [ask_user("Anything else?")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + } + ) + + result = await make_engine(tmp_path, runtime).start("Ingestion") + + first_turn = runtime.runs[0] + assert first_turn.results[0][0] == "delegate" + assert "Refused" in first_turn.results[0][1] + assert "product_selection" in first_turn.results[0][1] + assert result.status == "awaiting_user" + + +async def test_incomplete_submission_is_returned_to_the_specialist(tmp_path): + incomplete = Call( + "submit_deliverable", {"content": "## Functional requirements\nx"} + ) + runtime = FakeAgentRuntime( + { + "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], + specialist(K.REQUIREMENTS): [[incomplete, submission(K.REQUIREMENTS)]], + } + ) + + await make_engine(tmp_path, runtime).start("Ingestion") + + analyst = runtime.runs[1] + assert "Non-functional requirements" in analyst.results[0][1] + assert analyst.results[1] == ("submit_deliverable", "Submitted requirements.") + + +# --- interactive approvals ---------------------------------------------------------------- + + +async def test_approval_pauses_the_execution_until_the_user_decides(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + [delegate(K.PRODUCT_SELECTION)], + [ask_user("Pause here")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], + specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)], + } + ) + engine = make_engine(tmp_path, runtime) + + paused = await engine.start("Ingestion") + + assert paused.status == "awaiting_user" + assert paused.prompt["approval"]["phase"] == "requirements_discovery" + assert set(paused.prompt["approval"]["deliverables"]) == { + "requirements", + "technical_decomposition", + } + assert ( + paused.state["ledger"]["phases"]["requirements_discovery"] + == "awaiting_approval" + ) + + resumed = await engine.resume( + paused.execution_id, + {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, + ) + + assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved" + assert ( + resumed.state["ledger"]["deliverables"]["product_selection"]["status"] + == "submitted" + ) + assert resumed.prompt == { + "questions": [{"from": "orchestrator", "question": "Pause here"}] + } + + +async def test_free_text_reply_never_grants_an_approval(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], + } + ) + engine = make_engine(tmp_path, runtime) + paused = await engine.start("Ingestion") + + again = await engine.resume(paused.execution_id, {"reply": "approve, looks good"}) + + assert ( + again.state["ledger"]["phases"]["requirements_discovery"] == "awaiting_approval" + ) + assert again.prompt["approval"]["phase"] == "requirements_discovery" + assert again.state["conversation"][-2:] == [ + {"from": "user", "text": "approve, looks good"}, + { + "from": "orchestrator", + "text": "Please approve requirements_discovery: requirements_discovery done", + }, + ] + + +async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + [ + reopen(K.REQUIREMENTS, "Add the RTO target"), + delegate(K.REQUIREMENTS), + ], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + ], + specialist(K.REQUIREMENTS): [ + submits(K.REQUIREMENTS), + submits(K.REQUIREMENTS), + ], + specialist(K.TECHNICAL_DECOMPOSITION): [ + submits(K.TECHNICAL_DECOMPOSITION), + submits(K.TECHNICAL_DECOMPOSITION), + ], + } + ) + engine = make_engine(tmp_path, runtime) + paused = await engine.start("Ingestion") + + second = await engine.resume( + paused.execution_id, + { + "approval": { + "phase": "requirements_discovery", + "decision": "changes", + "feedback": "Add the RTO target", + } + }, + ) + + assert second.prompt["approval"]["phase"] == "requirements_discovery" + ledger = second.state["ledger"] + assert ledger["deliverables"]["requirements"]["revisions"] == 1 + second_analyst_run = [ + r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS) + ][1] + assert second_analyst_run.task.context["feedback"] == ["Add the RTO target"] + assert "changes_requested" in [e["type"] for e in ledger["events"]] + + +# --- one voice --------------------------------------------------------------------------------- + + +async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt( + tmp_path, +): + def diagrammer_turn(task): + if task.context["answers"]: + return submits(K.ARCHITECTURE_DIAGRAM) + return [Call("needs_input", {"questions": ["Single region or multi-region?"]})] + + def advisor_turn(task): + if task.context["answers"]: + return submits(K.DESIGN_RECOMMENDATIONS) + return [Call("needs_input", {"questions": ["What is the RTO?"]})] + + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + [delegate(K.PRODUCT_SELECTION)], + [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], + [ask_user()], + [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], + [ask_user("Continue?")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], + specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)], + specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn], + specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn], + } + ) + engine = make_engine(tmp_path, runtime) + approval = await engine.start("Ingestion") + first = await engine.resume( + approval.execution_id, + {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, + ) + + assert first.prompt == { + "questions": [ + { + "from": "architecture_diagram", + "question": "Single region or multi-region?", + }, + {"from": "design_recommendations", "question": "What is the RTO?"}, + ] + } + + second = await engine.resume( + first.execution_id, {"reply": "Single region, RTO 1 hour"} + ) + + deliverables = second.state["ledger"]["deliverables"] + assert deliverables["architecture_diagram"]["status"] == "submitted" + assert deliverables["design_recommendations"]["status"] == "submitted" + advisor_retry = [ + r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS) + ][1] + assert advisor_retry.task.context["answers"] == [ + {"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"} + ] + + +# --- revision after validation ------------------------------------------------------------ + + +async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp_path): + turns = happy_path_turns() + turns["orchestrator"] = [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], + [delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)], + [delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)], + [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], + [ + reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"), + delegate(K.DEPLOYMENT_GUIDANCE), + ], + [request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)], + [request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], + [request_approval(P.PACKAGING), FINISH], + ] + turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [ + submits(K.DEPLOYMENT_GUIDANCE), + submits(K.DEPLOYMENT_GUIDANCE), + ] + turns[specialist(K.VALIDATION_RESULTS)] = [ + [submission(K.VALIDATION_RESULTS, "FAILED: missing IAM binding")], + submits(K.VALIDATION_RESULTS), + ] + runtime = FakeAgentRuntime(turns) + + result = await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True) + + assert result.status == "completed" + events = result.state["ledger"]["events"] + withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"] + assert withdrawn == ["solution_architecture"] + requests = [e["subject"] for e in events if e["type"] == "approval_requested"] + assert requests.count("solution_architecture") == 2 + iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)] + assert iac_runs[1].task.context["feedback"] == [ + "terraform plan: missing IAM binding" + ] + + +async def test_pre_approved_escalation_ends_with_warnings(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [reopen(K.REQUIREMENTS, "not good enough")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + } + ) + + result = await make_engine(tmp_path, runtime, max_revisions=0).start( + "Ingestion", pre_approved=True + ) + + assert result.status == "completed_with_warnings" + assert "revision limit" in result.state["warnings"][0] + assert not (tmp_path / "deliverables").exists() + + +# --- budgets ------------------------------------------------------------------------------------- + + +async def test_orchestrator_budget_exhaustion_hands_control_to_the_user(tmp_path): + roles = {**ROLES, "orchestrator": RoleSpec("orchestrator", "", budget=2)} + wandering = [Call("ready_work"), Call("ready_work"), Call("ready_work")] + runtime = FakeAgentRuntime({"orchestrator": [wandering]}) + + result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") + + assert result.status == "awaiting_user" + question = result.prompt["questions"][0] + assert question["from"] == "orchestrator" + assert "2 tool calls" in question["question"] + + +async def test_specialist_budget_exhaustion_is_relayed_as_a_question(tmp_path): + roles = { + **ROLES, + specialist(K.REQUIREMENTS): RoleSpec("requirements-analyst", "", budget=1), + } + runtime = FakeAgentRuntime( + { + "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user()]], + specialist(K.REQUIREMENTS): [ + [ + Call("submit_deliverable", {"content": "## Current state\nx"}), + submission(K.REQUIREMENTS), + ] + ], + } + ) + + result = await make_engine(tmp_path, runtime, roles=roles).start("Ingestion") + + assert result.prompt["questions"][0]["from"] == "requirements" + assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" + + +# --- cloud read permission ------------------------------------------------------------------- + + +def cloud_read_factory(calls: list[str]): + async def list_projects() -> ToolResult: + calls.append("list_projects") + return ToolResult("projects: demo-project") + + tool = ToolSpec( + "list_projects", + "List projects.", + {"type": "object", "properties": {}}, + list_projects, + tier="cloud_read", + ) + + def factory(role: RoleSpec, deliverable: str | None): + return [tool] if role.name == "requirements-analyst" else [] + + return factory + + +async def test_cloud_read_asks_permission_once_then_runs_the_tool(tmp_path): + calls: list[str] = [] + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.REQUIREMENTS)], + [ask_user("Next?")], + ], + specialist(K.REQUIREMENTS): [ + [Call("list_projects"), submission(K.REQUIREMENTS)], + [Call("list_projects"), submission(K.REQUIREMENTS)], + ], + } + ) + engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) + + asked = await engine.start("Ingestion") + + assert asked.prompt == { + "permission": {"tier": "cloud_read", "requested_by": ["requirements"]} + } + assert calls == [] + + granted = await engine.resume( + asked.execution_id, {"permission": {"cloud_read": True}} + ) + + assert calls == ["list_projects"] + assert granted.state["allow_cloud_read"] is True + assert ( + granted.state["ledger"]["deliverables"]["requirements"]["status"] == "submitted" + ) + + +async def test_denied_cloud_read_is_refused_to_the_agent(tmp_path): + calls: list[str] = [] + runtime = FakeAgentRuntime( + { + "orchestrator": [[delegate(K.REQUIREMENTS)], [ask_user("Next?")]], + specialist(K.REQUIREMENTS): [ + [Call("list_projects"), submission(K.REQUIREMENTS)] + ], + } + ) + engine = make_engine(tmp_path, runtime, tool_factory=cloud_read_factory(calls)) + + await engine.start("Ingestion", allow_cloud_read=False) + + analyst = runtime.runs[1] + assert "not permitted" in analyst.results[0][1] + assert calls == [] + + +# --- isolation and durability ------------------------------------------------------------------- + + +async def test_executions_in_one_session_share_no_state(tmp_path): + runtime = FakeAgentRuntime( + { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [ask_user("?")], + [ask_user("fresh?")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + } + ) + engine = make_engine(tmp_path, runtime) + + first = await engine.start("First", session_id="s-1") + second = await engine.start("Second", session_id="s-1") + + assert first.execution_id != second.execution_id + assert second.state["deliverables"] == {} + assert second.state["ledger"]["deliverables"]["requirements"]["status"] == "pending" + assert second.state["workflow_request"] == "Second" + assert second.state["session_id"] == "s-1" + + +async def test_paused_execution_resumes_from_a_durable_checkpoint(tmp_path): + turns = { + "orchestrator": [ + [delegate(K.REQUIREMENTS)], + [delegate(K.TECHNICAL_DECOMPOSITION)], + [request_approval(P.REQUIREMENTS_DISCOVERY)], + [ask_user("Resumed")], + ], + specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)], + specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)], + } + runtime = FakeAgentRuntime(turns) + db_path = str(tmp_path / "checkpoints.db") + + async with AsyncSqliteSaver.from_conn_string(db_path) as saver: + paused = await make_engine(tmp_path, runtime, checkpointer=saver).start( + "Ingestion" + ) + assert paused.prompt["approval"]["phase"] == "requirements_discovery" + + async with AsyncSqliteSaver.from_conn_string(db_path) as saver: + resumed = await make_engine(tmp_path, runtime, checkpointer=saver).resume( + paused.execution_id, + {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, + ) + + assert resumed.prompt == { + "questions": [{"from": "orchestrator", "question": "Resumed"}] + } + assert resumed.state["ledger"]["phases"]["requirements_discovery"] == "approved" diff --git a/tests/execution/test_recording.py b/tests/execution/test_recording.py new file mode 100644 index 0000000..23ad184 --- /dev/null +++ b/tests/execution/test_recording.py @@ -0,0 +1,91 @@ +"""The database recorder and the configured checkpointer.""" + +import sqlite3 + +import pytest +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + +from app.config import get_settings +from app.database import DatabaseManager +from app.execution.checkpointing import open_checkpointer +from app.execution.ledger import LedgerEvent +from app.execution.recorder import DatabaseRecorder + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def sqlite_db(tmp_path, monkeypatch): + path = tmp_path / "records.db" + monkeypatch.setattr(get_settings(), "SQLITE_FALLBACK_DB", str(path)) + return DatabaseManager(db_url="sqlite:///unused"), path + + +async def test_database_recorder_writes_events_submissions_and_status(sqlite_db): + manager, path = sqlite_db + recorder = DatabaseRecorder(manager) + + await recorder.record_events( + "exec-1", + [ + LedgerEvent( + 1, "submitted", "requirements", "requirements-analyst", "attempt 1" + ) + ], + ) + await recorder.record_submission( + "exec-1", "deployment_guidance", "# Guide", {"terraform/main.tf": "resource {}"} + ) + await recorder.record_status( + "exec-1", + session_id="session-1", + workflow_request="Ingestion", + status="awaiting_user", + current_phase="solution_architecture", + details={"revisions": 2}, + ) + + with sqlite3.connect(path) as conn: + logs = conn.execute( + "SELECT iteration, review_status, reviewer_agent, feedback " + "FROM orchestrator_review_logs WHERE execution_id = 'exec-1'" + ).fetchall() + artifacts = conn.execute( + "SELECT artifact_type, file_path FROM adk_artifacts WHERE execution_id = 'exec-1' " + "ORDER BY file_path" + ).fetchall() + execution = conn.execute( + "SELECT status, current_phase, loop_count FROM workflow_executions " + "WHERE execution_id = 'exec-1'" + ).fetchone() + session = conn.execute( + "SELECT agent_name, state_data FROM adk_sessions WHERE session_id = 'session-1'" + ).fetchone() + + assert logs == [(1, "submitted", "requirements-analyst", "requirements: attempt 1")] + assert artifacts == [ + ("deployment_guidance", "deliverables/deployment_guidance.md"), + ("deployment_guidance", "terraform/main.tf"), + ] + assert execution == ("awaiting_user", "solution_architecture", 2) + assert session[0] == "solution-architect" + assert "exec-1" in session[1] + + +async def test_checkpointer_follows_settings(tmp_path, monkeypatch): + settings = get_settings() + + monkeypatch.setattr(settings, "CHECKPOINTER", "memory") + async with open_checkpointer(settings) as saver: + assert isinstance(saver, InMemorySaver) + + monkeypatch.setattr(settings, "CHECKPOINTER", "sqlite") + monkeypatch.setattr(settings, "CHECKPOINT_DB_PATH", str(tmp_path / "cp.db")) + async with open_checkpointer(settings) as saver: + assert isinstance(saver, AsyncSqliteSaver) + + monkeypatch.setattr(settings, "CHECKPOINTER", "redis") + with pytest.raises(ValueError, match="redis"): + async with open_checkpointer(settings): + pass