131 lines
4.3 KiB
Python
131 lines
4.3 KiB
Python
"""A scripted AgentRuntime for deterministic tests and evaluation dry runs.
|
|
|
|
Each role has a queue of turns; a turn is a list of tool calls, or a function of the
|
|
task that returns one. Calls go through the real ToolSpecs, so every rule the tools
|
|
enforce is exercised exactly as a model would meet it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from execution_engine.adk.runtime import (
|
|
AgentTask,
|
|
Answer,
|
|
BudgetExhausted,
|
|
Outcome,
|
|
Question,
|
|
RoleSpec,
|
|
Stopped,
|
|
ToolSpec,
|
|
)
|
|
from execution_engine.workflows.workflow import Workflow
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Call:
|
|
tool: str
|
|
args: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]]
|
|
|
|
|
|
def sample_submission(workflow: Workflow, kind: str, note: str = "") -> Call:
|
|
"""A `submit_deliverable` call that meets the Deliverable's contract."""
|
|
spec = workflow.deliverables[kind]
|
|
parts = [f"# {kind}", note or "Summary."]
|
|
parts += [f"## {section}\n{note or 'Details.'}" for section in spec.sections]
|
|
attachments = {
|
|
name: workflow.samples.get(name, f"# {name}\n") for name in spec.attachments
|
|
}
|
|
return Call(
|
|
"submit_deliverable",
|
|
{"content": "\n".join(parts) + "\n", "attachments": attachments},
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunRecord:
|
|
role: str
|
|
task: AgentTask
|
|
tools: tuple[str, ...]
|
|
results: tuple[tuple[str, str], ...] # (tool name, message)
|
|
outcome: Outcome
|
|
|
|
|
|
class ScriptExhausted(AssertionError):
|
|
"""A role was run more times than the test scripted."""
|
|
|
|
|
|
class FakeAgentRuntime:
|
|
def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None:
|
|
self._turns = {role: deque(role_turns) for role, role_turns in turns.items()}
|
|
self.runs: list[RunRecord] = []
|
|
|
|
def remaining(self, role: str) -> int:
|
|
return len(self._turns.get(role, ()))
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
queue = self._turns.get(role.name)
|
|
if not queue:
|
|
raise ScriptExhausted(f"No scripted turn left for role {role.name!r}")
|
|
turn = queue.popleft()
|
|
calls = turn(task) if callable(turn) else turn
|
|
by_name = {tool.name: tool for tool in tools}
|
|
|
|
results: list[tuple[str, str]] = []
|
|
outcome: Outcome = Stopped(
|
|
"The script ended without a tool that ends the turn."
|
|
)
|
|
for index, call in enumerate(calls):
|
|
if index >= role.budget:
|
|
outcome = BudgetExhausted(
|
|
f"{role.name} used all {role.budget} tool calls before finishing."
|
|
)
|
|
break
|
|
if call.tool not in by_name:
|
|
raise ValueError(
|
|
f"{role.name} called {call.tool!r}, which it was not offered: "
|
|
f"{sorted(by_name)}"
|
|
)
|
|
result = await by_name[call.tool].handler(**call.args)
|
|
results.append((call.tool, result.message))
|
|
if result.outcome is not None:
|
|
outcome = result.outcome
|
|
break
|
|
|
|
self.runs.append(
|
|
RunRecord(role.name, task, tuple(by_name), tuple(results), outcome)
|
|
)
|
|
return outcome
|
|
|
|
|
|
class FakeDecider:
|
|
"""A scripted Decider: answers by question id, or declines so callers fall back.
|
|
|
|
`answers` maps a question id to an Answer, or to a function of (state, question)
|
|
returning one. An id with no entry makes the whole decision unavailable.
|
|
"""
|
|
|
|
def __init__(self, answers: Mapping[str, Any] | None = None) -> None:
|
|
self.answers = dict(answers or {})
|
|
self.asked: list[tuple[str, dict[str, Question], str]] = []
|
|
|
|
async def decide(
|
|
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
|
|
) -> Mapping[str, Answer] | None:
|
|
self.asked.append((state, dict(questions), scope))
|
|
decided = {}
|
|
for name, question in questions.items():
|
|
if name not in self.answers:
|
|
return None
|
|
answer = self.answers[name]
|
|
decided[name] = answer(state, question) if callable(answer) else answer
|
|
return decided
|