388 lines
12 KiB
Python
388 lines
12 KiB
Python
"""The ADK implementation of AgentRuntime, driven by a scripted model (no network)."""
|
|
|
|
from collections.abc import AsyncGenerator
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
from google.adk.models.base_llm import BaseLlm
|
|
from google.adk.models.google_llm import Gemini
|
|
from google.adk.models.lite_llm import LiteLlm
|
|
from google.adk.models.llm_request import LlmRequest
|
|
from google.adk.models.llm_response import LlmResponse
|
|
from google.genai import types
|
|
from pydantic import Field
|
|
|
|
from app.adk.adk_runtime import AdkAgentRuntime, default_model_factory
|
|
from app.adk.runtime import (
|
|
AgentTask,
|
|
BudgetExhausted,
|
|
NeedsInput,
|
|
RoleSpec,
|
|
Stopped,
|
|
Submitted,
|
|
ToolResult,
|
|
ToolSpec,
|
|
TurnEnded,
|
|
)
|
|
from app.providers import load_provider
|
|
|
|
SKILLS = {name: meta.path for name, meta in load_provider("gcp").skills.items()}
|
|
|
|
|
|
class ScriptedLlm(BaseLlm):
|
|
"""Returns one scripted response per model call and records every request."""
|
|
|
|
model: str = "scripted"
|
|
script: list[list[types.Part]] = Field(default_factory=list)
|
|
requests: list[LlmRequest] = Field(default_factory=list)
|
|
|
|
async def generate_content_async(
|
|
self, llm_request: LlmRequest, stream: bool = False
|
|
) -> AsyncGenerator[LlmResponse, None]:
|
|
self.requests.append(llm_request)
|
|
parts = self.script.pop(0) if self.script else [types.Part(text="(no script)")]
|
|
yield LlmResponse(content=types.Content(role="model", parts=parts))
|
|
|
|
|
|
def call(tool: str, **args) -> types.Part:
|
|
return types.Part(function_call=types.FunctionCall(name=tool, args=args))
|
|
|
|
|
|
def text(value: str) -> types.Part:
|
|
return types.Part(text=value)
|
|
|
|
|
|
def runtime_with(model: ScriptedLlm) -> AdkAgentRuntime:
|
|
return AdkAgentRuntime(skill_dirs=SKILLS, model_factory=lambda role: model)
|
|
|
|
|
|
class Recorder:
|
|
"""ToolSpecs whose handlers record their calls."""
|
|
|
|
def __init__(self) -> None:
|
|
self.calls: list[tuple[str, dict]] = []
|
|
|
|
def tool(self, name: str, outcome=None, message: str = "ok") -> ToolSpec:
|
|
async def handler(**kwargs) -> ToolResult:
|
|
self.calls.append((name, kwargs))
|
|
return ToolResult(f"{message}: {kwargs}", outcome)
|
|
|
|
return ToolSpec(
|
|
name,
|
|
f"The {name} tool.",
|
|
{"type": "object", "properties": {"note": {"type": "string"}}},
|
|
handler,
|
|
)
|
|
|
|
|
|
ROLE = RoleSpec(
|
|
"iac-author",
|
|
"Write Terraform. Keep {braces} and {{doubles}} verbatim.",
|
|
budget=5,
|
|
skills=("google-cloud-solution-architecture",),
|
|
)
|
|
TASK = AgentTask("iac-author", "Produce deployment_guidance.", {"feedback": ["x"]})
|
|
|
|
|
|
async def test_specialist_loads_its_skill_and_submits():
|
|
recorder = Recorder()
|
|
submitted = Submitted("# Guidance", {"terraform/main.tf": "resource {}"})
|
|
model = ScriptedLlm(
|
|
script=[
|
|
[call("load_skill", skill_name="google-cloud-solution-architecture")],
|
|
[call("submit_deliverable", note="done")],
|
|
]
|
|
)
|
|
|
|
outcome = await runtime_with(model).run(
|
|
ROLE, TASK, [recorder.tool("submit_deliverable", submitted)]
|
|
)
|
|
|
|
assert outcome == submitted
|
|
assert recorder.calls == [("submit_deliverable", {"note": "done"})]
|
|
skill_response = _function_responses(model.requests[1])["load_skill"]
|
|
assert "Phase 1: Requirements discovery" in str(skill_response)
|
|
|
|
|
|
async def test_agent_is_offered_its_tools_and_skill_tools_without_scripts():
|
|
model = ScriptedLlm(script=[[text("nothing to do")]])
|
|
|
|
await runtime_with(model).run(ROLE, TASK, [Recorder().tool("submit_deliverable")])
|
|
|
|
request = model.requests[0]
|
|
assert set(request.tools_dict) == {
|
|
"submit_deliverable",
|
|
"list_skills",
|
|
"load_skill",
|
|
"load_skill_resource",
|
|
}
|
|
declaration = request.tools_dict["submit_deliverable"]._get_declaration()
|
|
assert declaration.parameters_json_schema["properties"] == {
|
|
"note": {"type": "string"}
|
|
}
|
|
|
|
|
|
async def test_role_without_skills_gets_no_skill_tools():
|
|
model = ScriptedLlm(script=[[text("done")]])
|
|
role = RoleSpec("decomposer", "Decompose.", budget=3)
|
|
|
|
await runtime_with(model).run(role, TASK, [Recorder().tool("needs_input")])
|
|
|
|
assert set(model.requests[0].tools_dict) == {"needs_input"}
|
|
|
|
|
|
async def test_only_the_roles_skills_can_be_loaded():
|
|
model = ScriptedLlm(
|
|
script=[
|
|
[call("load_skill", skill_name="google-cloud-waf-security")],
|
|
[text("ok")],
|
|
]
|
|
)
|
|
|
|
await runtime_with(model).run(ROLE, TASK, [])
|
|
|
|
response = _function_responses(model.requests[1])["load_skill"]
|
|
assert "error" in response
|
|
assert "Well-Architected" not in str(response)
|
|
|
|
|
|
async def test_instructions_and_task_context_reach_the_model_verbatim():
|
|
model = ScriptedLlm(script=[[text("done")]])
|
|
|
|
await runtime_with(model).run(ROLE, TASK, [])
|
|
|
|
request = model.requests[0]
|
|
assert (
|
|
"Keep {braces} and {{doubles}} verbatim." in request.config.system_instruction
|
|
)
|
|
user_text = request.contents[0].parts[0].text
|
|
assert "Produce deployment_guidance." in user_text
|
|
assert '"feedback": [\n "x"\n ]' in user_text
|
|
|
|
|
|
async def test_turn_ends_as_soon_as_a_tool_returns_an_outcome():
|
|
recorder = Recorder()
|
|
model = ScriptedLlm(
|
|
script=[
|
|
[call("delegate"), call("ask_user")],
|
|
[call("ask_user")],
|
|
]
|
|
)
|
|
|
|
outcome = await runtime_with(model).run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=10),
|
|
TASK,
|
|
[
|
|
recorder.tool("delegate", TurnEnded("delegate")),
|
|
recorder.tool("ask_user", TurnEnded("ask_user")),
|
|
],
|
|
)
|
|
|
|
assert outcome == TurnEnded("delegate")
|
|
assert [name for name, _ in recorder.calls] == ["delegate"]
|
|
assert len(model.requests) == 1
|
|
|
|
|
|
async def test_budget_counts_every_tool_call_and_stops_the_run():
|
|
recorder = Recorder()
|
|
model = ScriptedLlm(script=[[call("ready_work")] for _ in range(5)])
|
|
|
|
outcome = await runtime_with(model).run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=2),
|
|
TASK,
|
|
[recorder.tool("ready_work")],
|
|
)
|
|
|
|
assert isinstance(outcome, BudgetExhausted)
|
|
assert "2 tool calls" in outcome.summary
|
|
assert len(recorder.calls) == 2
|
|
|
|
|
|
async def test_model_reply_without_a_turn_ending_tool_is_stopped_after_nudges():
|
|
model = ScriptedLlm(
|
|
script=[[text("I need the RTO before I can continue.")], [text("")], [text("")]]
|
|
)
|
|
|
|
outcome = await runtime_with(model).run(ROLE, TASK, [])
|
|
|
|
assert outcome == Stopped("I need the RTO before I can continue.")
|
|
assert len(model.requests) == 3
|
|
assert "without calling a tool that ends your turn" in (
|
|
model.requests[1].contents[-1].parts[0].text
|
|
)
|
|
|
|
|
|
async def test_a_nudged_agent_that_then_ends_its_turn_succeeds_in_one_session():
|
|
recorder = Recorder()
|
|
model = ScriptedLlm(script=[[text("")], [call("delegate", note="go")]])
|
|
|
|
outcome = await runtime_with(model).run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
|
TASK,
|
|
[recorder.tool("delegate", TurnEnded("delegate"))],
|
|
)
|
|
|
|
assert outcome == TurnEnded("delegate")
|
|
assert "without calling a tool" in model.requests[1].contents[-1].parts[0].text
|
|
|
|
|
|
class TimingOutLlm(ScriptedLlm):
|
|
"""Times out on the first request, then follows its script."""
|
|
|
|
timed_out: bool = False
|
|
|
|
async def generate_content_async(
|
|
self, llm_request: LlmRequest, stream: bool = False
|
|
) -> AsyncGenerator[LlmResponse, None]:
|
|
if not self.timed_out:
|
|
self.timed_out = True
|
|
raise httpx.ReadTimeout("no response")
|
|
async for response in super().generate_content_async(llm_request, stream):
|
|
yield response
|
|
|
|
|
|
async def test_a_timed_out_model_request_is_retried_in_the_same_session():
|
|
recorder = Recorder()
|
|
model = TimingOutLlm(script=[[call("delegate", note="go")]])
|
|
|
|
outcome = await runtime_with(model).run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
|
TASK,
|
|
[recorder.tool("delegate", TurnEnded("delegate"))],
|
|
)
|
|
|
|
assert outcome == TurnEnded("delegate")
|
|
assert "timed out" in model.requests[0].contents[-1].parts[0].text
|
|
|
|
|
|
async def test_model_timeout_is_set_on_every_request():
|
|
model = ScriptedLlm(script=[[call("delegate")]])
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
|
)
|
|
|
|
await runtime.run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
|
TASK,
|
|
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
|
)
|
|
|
|
assert model.requests[0].config.http_options.timeout == 90_000
|
|
|
|
|
|
async def test_a_role_with_its_own_timeout_is_given_it_over_the_default():
|
|
model = ScriptedLlm(script=[[call("delegate")]])
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
|
)
|
|
|
|
await runtime.run(
|
|
RoleSpec("orchestrator", "Coordinate.", budget=5, model_timeout_seconds=420),
|
|
TASK,
|
|
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
|
)
|
|
|
|
assert model.requests[0].config.http_options.timeout == 420_000
|
|
|
|
|
|
class MeteredLlm(ScriptedLlm):
|
|
async def generate_content_async(
|
|
self, llm_request: LlmRequest, stream: bool = False
|
|
) -> AsyncGenerator[LlmResponse, None]:
|
|
async for response in super().generate_content_async(llm_request, stream):
|
|
response.usage_metadata = types.GenerateContentResponseUsageMetadata(
|
|
prompt_token_count=100,
|
|
cached_content_token_count=40,
|
|
candidates_token_count=10,
|
|
thoughts_token_count=5,
|
|
total_token_count=115,
|
|
)
|
|
yield response
|
|
|
|
|
|
async def test_token_usage_is_summed_per_role_and_model():
|
|
model = MeteredLlm(script=[[text("")], [call("delegate")]])
|
|
role = RoleSpec("orchestrator", "Coordinate.", budget=5, model="gemini/x")
|
|
runtime = runtime_with(model)
|
|
|
|
await runtime.run(role, TASK, [Recorder().tool("delegate", TurnEnded("delegate"))])
|
|
|
|
usage = runtime.usage[("orchestrator", "gemini/x")]
|
|
assert (usage.requests, usage.prompt, usage.cached, usage.output) == (
|
|
2,
|
|
200,
|
|
80,
|
|
20,
|
|
)
|
|
assert (usage.thinking, usage.total) == (10, 230)
|
|
|
|
|
|
async def test_task_message_names_the_roles_skills():
|
|
model = ScriptedLlm(script=[[text("done")]] * 3)
|
|
|
|
await runtime_with(model).run(ROLE, TASK, [])
|
|
|
|
first = model.requests[0].contents[0].parts[0].text
|
|
assert (
|
|
"Skills available to you (load with load_skill): google-cloud-solution-architecture."
|
|
in first
|
|
)
|
|
|
|
|
|
async def test_needs_input_outcome_passes_through():
|
|
needs = NeedsInput(("What region?",))
|
|
model = ScriptedLlm(script=[[call("needs_input")]])
|
|
|
|
outcome = await runtime_with(model).run(
|
|
ROLE, TASK, [Recorder().tool("needs_input", needs)]
|
|
)
|
|
|
|
assert outcome == needs
|
|
|
|
|
|
async def test_each_run_starts_with_a_fresh_session():
|
|
model = ScriptedLlm(script=[[text("first")], [text("second")]])
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs=SKILLS, model_factory=lambda role: model, max_nudges=0
|
|
)
|
|
|
|
await runtime.run(ROLE, TASK, [])
|
|
await runtime.run(ROLE, TASK, [])
|
|
|
|
assert len(model.requests[1].contents) == 1
|
|
|
|
|
|
def test_default_model_factory_uses_native_gemini_and_litellm_for_others():
|
|
gemini = default_model_factory(
|
|
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-3.8-flash")
|
|
)
|
|
other = default_model_factory(
|
|
RoleSpec("diagrammer", "", budget=1, model="anthropic/claude-sonnet-5")
|
|
)
|
|
|
|
assert isinstance(gemini, Gemini)
|
|
assert gemini.model == "gemini-3.8-flash"
|
|
assert gemini.retry_options.attempts == 5
|
|
assert isinstance(other, LiteLlm)
|
|
assert other.model == "anthropic/claude-sonnet-5"
|
|
with pytest.raises(ValueError, match="model"):
|
|
default_model_factory(RoleSpec("diagrammer", "", budget=1))
|
|
|
|
|
|
def test_unknown_skill_is_rejected_when_the_runtime_is_built():
|
|
with pytest.raises(ValueError, match="no-such-skill"):
|
|
AdkAgentRuntime(
|
|
skill_dirs={"no-such-skill": Path("/nonexistent")},
|
|
model_factory=lambda role: ScriptedLlm(),
|
|
)
|
|
|
|
|
|
def _function_responses(request: LlmRequest) -> dict:
|
|
responses = {}
|
|
for content in request.contents:
|
|
for part in content.parts or []:
|
|
if part.function_response:
|
|
responses[part.function_response.name] = part.function_response.response
|
|
return responses
|