Files
orchestration_engine/execution_engine/adk/fake_runtime.py

131 lines
4.3 KiB
Python

"""A scripted AgentRuntime for deterministic tests and evaluation dry runs.
Each role has a queue of turns; a turn is a list of tool calls, or a function of the
task that returns one. Calls go through the real ToolSpecs, so every rule the tools
enforce is exercised exactly as a model would meet it.
"""
from __future__ import annotations
from collections import deque
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any
from execution_engine.adk.runtime import (
AgentTask,
Answer,
BudgetExhausted,
Outcome,
Question,
RoleSpec,
Stopped,
ToolSpec,
)
from execution_engine.workflows.workflow import Workflow
@dataclass(frozen=True)
class Call:
tool: str
args: Mapping[str, Any] = field(default_factory=dict)
Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]]
def sample_submission(workflow: Workflow, kind: str, note: str = "") -> Call:
"""A `submit_deliverable` call that meets the Deliverable's contract."""
spec = workflow.deliverables[kind]
parts = [f"# {kind}", note or "Summary."]
parts += [f"## {section}\n{note or 'Details.'}" for section in spec.sections]
attachments = {
name: workflow.samples.get(name, f"# {name}\n") for name in spec.attachments
}
return Call(
"submit_deliverable",
{"content": "\n".join(parts) + "\n", "attachments": attachments},
)
@dataclass(frozen=True)
class RunRecord:
role: str
task: AgentTask
tools: tuple[str, ...]
results: tuple[tuple[str, str], ...] # (tool name, message)
outcome: Outcome
class ScriptExhausted(AssertionError):
"""A role was run more times than the test scripted."""
class FakeAgentRuntime:
def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None:
self._turns = {role: deque(role_turns) for role, role_turns in turns.items()}
self.runs: list[RunRecord] = []
def remaining(self, role: str) -> int:
return len(self._turns.get(role, ()))
async def run(
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
) -> Outcome:
queue = self._turns.get(role.name)
if not queue:
raise ScriptExhausted(f"No scripted turn left for role {role.name!r}")
turn = queue.popleft()
calls = turn(task) if callable(turn) else turn
by_name = {tool.name: tool for tool in tools}
results: list[tuple[str, str]] = []
outcome: Outcome = Stopped(
"The script ended without a tool that ends the turn."
)
for index, call in enumerate(calls):
if index >= role.budget:
outcome = BudgetExhausted(
f"{role.name} used all {role.budget} tool calls before finishing."
)
break
if call.tool not in by_name:
raise ValueError(
f"{role.name} called {call.tool!r}, which it was not offered: "
f"{sorted(by_name)}"
)
result = await by_name[call.tool].handler(**call.args)
results.append((call.tool, result.message))
if result.outcome is not None:
outcome = result.outcome
break
self.runs.append(
RunRecord(role.name, task, tuple(by_name), tuple(results), outcome)
)
return outcome
class FakeDecider:
"""A scripted Decider: answers by question id, or declines so callers fall back.
`answers` maps a question id to an Answer, or to a function of (state, question)
returning one. An id with no entry makes the whole decision unavailable.
"""
def __init__(self, answers: Mapping[str, Any] | None = None) -> None:
self.answers = dict(answers or {})
self.asked: list[tuple[str, dict[str, Question], str]] = []
async def decide(
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
) -> Mapping[str, Answer] | None:
self.asked.append((state, dict(questions), scope))
decided = {}
for name, question in questions.items():
if name not in self.answers:
return None
answer = self.answers[name]
decided[name] = answer(state, question) if callable(answer) else answer
return decided