feat: working orhcestration engine
This commit is contained in:
0
tests/agents/__init__.py
Normal file
0
tests/agents/__init__.py
Normal file
421
tests/agents/test_adk_runtime.py
Normal file
421
tests/agents/test_adk_runtime.py
Normal file
@@ -0,0 +1,421 @@
|
||||
"""The ADK implementation of AgentRuntime, driven by a scripted model (no network)."""
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from google.adk.models.base_llm import BaseLlm
|
||||
from google.adk.models.google_llm import Gemini
|
||||
from google.adk.models.lite_llm import LiteLlm
|
||||
from google.adk.models.llm_request import LlmRequest
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types
|
||||
from pydantic import Field
|
||||
|
||||
from execution_engine.adk.adk_runtime import AdkAgentRuntime, default_model_factory
|
||||
from execution_engine.adk.runtime import (
|
||||
AgentTask,
|
||||
BudgetExhausted,
|
||||
NeedsInput,
|
||||
RoleSpec,
|
||||
Stopped,
|
||||
Submitted,
|
||||
ToolResult,
|
||||
ToolSpec,
|
||||
TurnEnded,
|
||||
)
|
||||
from execution_engine.skills.agent_skills import discover_skills
|
||||
|
||||
# Stand-in Skills: the engine has none of its own.
|
||||
SKILLS = {
|
||||
name: meta.path
|
||||
for name, meta in discover_skills(
|
||||
[Path(__file__).parents[1] / "fixtures" / "skills"]
|
||||
).items()
|
||||
}
|
||||
|
||||
|
||||
class ScriptedLlm(BaseLlm):
|
||||
"""Returns one scripted response per model call and records every request."""
|
||||
|
||||
model: str = "scripted"
|
||||
script: list[list[types.Part]] = Field(default_factory=list)
|
||||
requests: list[LlmRequest] = Field(default_factory=list)
|
||||
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
self.requests.append(llm_request)
|
||||
parts = self.script.pop(0) if self.script else [types.Part(text="(no script)")]
|
||||
yield LlmResponse(content=types.Content(role="model", parts=parts))
|
||||
|
||||
|
||||
def call(tool: str, **args) -> types.Part:
|
||||
return types.Part(function_call=types.FunctionCall(name=tool, args=args))
|
||||
|
||||
|
||||
def text(value: str) -> types.Part:
|
||||
return types.Part(text=value)
|
||||
|
||||
|
||||
def runtime_with(model: ScriptedLlm) -> AdkAgentRuntime:
|
||||
return AdkAgentRuntime(skill_dirs=SKILLS, model_factory=lambda role: model)
|
||||
|
||||
|
||||
class Recorder:
|
||||
"""ToolSpecs whose handlers record their calls."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
def tool(self, name: str, outcome=None, message: str = "ok") -> ToolSpec:
|
||||
async def handler(**kwargs) -> ToolResult:
|
||||
self.calls.append((name, kwargs))
|
||||
return ToolResult(f"{message}: {kwargs}", outcome)
|
||||
|
||||
return ToolSpec(
|
||||
name,
|
||||
f"The {name} tool.",
|
||||
{"type": "object", "properties": {"note": {"type": "string"}}},
|
||||
handler,
|
||||
)
|
||||
|
||||
|
||||
ROLE = RoleSpec(
|
||||
"iac-author",
|
||||
"Write Terraform. Keep {braces} and {{doubles}} verbatim.",
|
||||
budget=5,
|
||||
skills=("toy-design",),
|
||||
)
|
||||
TASK = AgentTask("iac-author", "Produce deployment_guidance.", {"feedback": ["x"]})
|
||||
|
||||
|
||||
async def test_specialist_loads_its_skill_and_submits():
|
||||
recorder = Recorder()
|
||||
submitted = Submitted("# Guidance", {"terraform/main.tf": "resource {}"})
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("load_skill", skill_name="toy-design")],
|
||||
[call("submit_deliverable", note="done")],
|
||||
]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
ROLE, TASK, [recorder.tool("submit_deliverable", submitted)]
|
||||
)
|
||||
|
||||
assert outcome == submitted
|
||||
assert recorder.calls == [("submit_deliverable", {"note": "done"})]
|
||||
skill_response = _function_responses(model.requests[1])["load_skill"]
|
||||
assert "Nothing here is guidance" in str(skill_response)
|
||||
|
||||
|
||||
async def test_agent_is_offered_its_tools_and_skill_tools_without_scripts():
|
||||
model = ScriptedLlm(script=[[text("nothing to do")]])
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [Recorder().tool("submit_deliverable")])
|
||||
|
||||
request = model.requests[0]
|
||||
assert set(request.tools_dict) == {
|
||||
"submit_deliverable",
|
||||
"list_skills",
|
||||
"load_skill",
|
||||
"load_skill_resource",
|
||||
}
|
||||
declaration = request.tools_dict["submit_deliverable"]._get_declaration()
|
||||
assert declaration.parameters_json_schema["properties"] == {
|
||||
"note": {"type": "string"}
|
||||
}
|
||||
|
||||
|
||||
async def test_role_without_skills_gets_no_skill_tools():
|
||||
model = ScriptedLlm(script=[[text("done")]])
|
||||
role = RoleSpec("decomposer", "Decompose.", budget=3)
|
||||
|
||||
await runtime_with(model).run(role, TASK, [Recorder().tool("needs_input")])
|
||||
|
||||
assert set(model.requests[0].tools_dict) == {"needs_input"}
|
||||
|
||||
|
||||
async def test_only_the_roles_skills_can_be_loaded():
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("load_skill", skill_name="toy-security")],
|
||||
[text("ok")],
|
||||
]
|
||||
)
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
response = _function_responses(model.requests[1])["load_skill"]
|
||||
assert "error" in response
|
||||
assert "toy-security-body" not in str(response)
|
||||
|
||||
|
||||
async def test_instructions_and_task_context_reach_the_model_verbatim():
|
||||
model = ScriptedLlm(script=[[text("done")]])
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
request = model.requests[0]
|
||||
assert (
|
||||
"Keep {braces} and {{doubles}} verbatim." in request.config.system_instruction
|
||||
)
|
||||
user_text = request.contents[0].parts[0].text
|
||||
assert "Produce deployment_guidance." in user_text
|
||||
# Written compactly: the context is sent again with every request of the run.
|
||||
assert '"feedback":["x"]' in user_text
|
||||
|
||||
|
||||
async def test_turn_ends_as_soon_as_a_tool_returns_an_outcome():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(
|
||||
script=[
|
||||
[call("delegate"), call("ask_user")],
|
||||
[call("ask_user")],
|
||||
]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=10),
|
||||
TASK,
|
||||
[
|
||||
recorder.tool("delegate", TurnEnded("delegate")),
|
||||
recorder.tool("ask_user", TurnEnded("ask_user")),
|
||||
],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert [name for name, _ in recorder.calls] == ["delegate"]
|
||||
assert len(model.requests) == 1
|
||||
|
||||
|
||||
async def test_budget_counts_every_tool_call_and_stops_the_run():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(script=[[call("ready_work")] for _ in range(5)])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=2),
|
||||
TASK,
|
||||
[recorder.tool("ready_work")],
|
||||
)
|
||||
|
||||
assert isinstance(outcome, BudgetExhausted)
|
||||
assert "2 tool calls" in outcome.summary
|
||||
assert len(recorder.calls) == 2
|
||||
|
||||
|
||||
async def test_model_reply_without_a_turn_ending_tool_is_stopped_after_nudges():
|
||||
model = ScriptedLlm(
|
||||
script=[[text("I need the RTO before I can continue.")], [text("")], [text("")]]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
assert outcome == Stopped("I need the RTO before I can continue.")
|
||||
assert len(model.requests) == 3
|
||||
assert "without calling a tool that ends your turn" in (
|
||||
model.requests[1].contents[-1].parts[0].text
|
||||
)
|
||||
|
||||
|
||||
async def test_a_nudged_agent_that_then_ends_its_turn_succeeds_in_one_session():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(script=[[text("")], [call("delegate", note="go")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[recorder.tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert "without calling a tool" in model.requests[1].contents[-1].parts[0].text
|
||||
|
||||
|
||||
class TimingOutLlm(ScriptedLlm):
|
||||
"""Times out on the first request, then follows its script."""
|
||||
|
||||
timed_out: bool = False
|
||||
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
if not self.timed_out:
|
||||
self.timed_out = True
|
||||
raise httpx.ReadTimeout("no response")
|
||||
async for response in super().generate_content_async(llm_request, stream):
|
||||
yield response
|
||||
|
||||
|
||||
async def test_a_timed_out_model_request_is_retried_in_the_same_session():
|
||||
recorder = Recorder()
|
||||
model = TimingOutLlm(script=[[call("delegate", note="go")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[recorder.tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert "timed out" in model.requests[0].contents[-1].parts[0].text
|
||||
|
||||
|
||||
async def test_model_timeout_is_set_on_every_request():
|
||||
model = ScriptedLlm(script=[[call("delegate")]])
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
||||
)
|
||||
|
||||
await runtime.run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert model.requests[0].config.http_options.timeout == 90_000
|
||||
|
||||
|
||||
async def test_a_role_with_its_own_timeout_is_given_it_over_the_default():
|
||||
model = ScriptedLlm(script=[[call("delegate")]])
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
||||
)
|
||||
|
||||
await runtime.run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5, model_timeout_seconds=420),
|
||||
TASK,
|
||||
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert model.requests[0].config.http_options.timeout == 420_000
|
||||
|
||||
|
||||
class MeteredLlm(ScriptedLlm):
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
async for response in super().generate_content_async(llm_request, stream):
|
||||
response.usage_metadata = types.GenerateContentResponseUsageMetadata(
|
||||
prompt_token_count=100,
|
||||
cached_content_token_count=40,
|
||||
candidates_token_count=10,
|
||||
thoughts_token_count=5,
|
||||
total_token_count=115,
|
||||
)
|
||||
yield response
|
||||
|
||||
|
||||
async def test_token_usage_is_summed_per_role_and_model():
|
||||
model = MeteredLlm(script=[[text("")], [call("delegate")]])
|
||||
role = RoleSpec("orchestrator", "Coordinate.", budget=5, model="gemini/x")
|
||||
runtime = runtime_with(model)
|
||||
|
||||
await runtime.run(role, TASK, [Recorder().tool("delegate", TurnEnded("delegate"))])
|
||||
|
||||
usage = runtime.usage[("orchestrator", "gemini/x")]
|
||||
assert (usage.requests, usage.prompt, usage.cached, usage.output) == (
|
||||
2,
|
||||
200,
|
||||
80,
|
||||
20,
|
||||
)
|
||||
assert (usage.thinking, usage.total) == (10, 230)
|
||||
|
||||
|
||||
async def test_task_message_names_the_roles_skills():
|
||||
model = ScriptedLlm(script=[[text("done")]] * 3)
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
first = model.requests[0].contents[0].parts[0].text
|
||||
assert "Skills available to you (load with load_skill): toy-design." in first
|
||||
|
||||
|
||||
async def test_needs_input_outcome_passes_through():
|
||||
needs = NeedsInput(("What region?",))
|
||||
model = ScriptedLlm(script=[[call("needs_input")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
ROLE, TASK, [Recorder().tool("needs_input", needs)]
|
||||
)
|
||||
|
||||
assert outcome == needs
|
||||
|
||||
|
||||
async def test_each_run_starts_with_a_fresh_session():
|
||||
model = ScriptedLlm(script=[[text("first")], [text("second")]])
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, max_nudges=0
|
||||
)
|
||||
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
|
||||
assert len(model.requests[1].contents) == 1
|
||||
|
||||
|
||||
def test_default_model_factory_uses_native_gemini_and_litellm_for_others():
|
||||
gemini = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-3.8-flash")
|
||||
)
|
||||
other = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="anthropic/claude-sonnet-5")
|
||||
)
|
||||
|
||||
assert isinstance(gemini, Gemini)
|
||||
assert gemini.model == "gemini-3.8-flash"
|
||||
assert gemini.retry_options.attempts == 5
|
||||
assert isinstance(other, LiteLlm)
|
||||
assert other.model == "anthropic/claude-sonnet-5"
|
||||
with pytest.raises(ValueError, match="model"):
|
||||
default_model_factory(RoleSpec("diagrammer", "", budget=1))
|
||||
|
||||
|
||||
def test_unknown_skill_is_rejected_when_the_runtime_is_built():
|
||||
with pytest.raises(ValueError, match="no-such-skill"):
|
||||
AdkAgentRuntime(
|
||||
skill_dirs={"no-such-skill": Path("/nonexistent")},
|
||||
model_factory=lambda role: ScriptedLlm(),
|
||||
)
|
||||
|
||||
|
||||
def _function_responses(request: LlmRequest) -> dict:
|
||||
responses = {}
|
||||
for content in request.contents:
|
||||
for part in content.parts or []:
|
||||
if part.function_response:
|
||||
responses[part.function_response.name] = part.function_response.response
|
||||
return responses
|
||||
|
||||
|
||||
def test_a_data_file_is_written_for_the_model_as_data_not_as_escaped_text():
|
||||
from execution_engine.adk.adk_runtime import _render_task
|
||||
from execution_engine.adk.runtime import AgentTask
|
||||
|
||||
task = AgentTask(
|
||||
"iac-author",
|
||||
"Produce.",
|
||||
{
|
||||
"dependencies": {
|
||||
"target_architecture": {
|
||||
"content": "prose",
|
||||
"attachments": {
|
||||
"target-architecture.json": '{"elements": [{"id": "E-1"}]}',
|
||||
"architecture.mmd": 'graph TD\n A["x"]',
|
||||
"broken.json": "{not json",
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
text = _render_task(task)
|
||||
|
||||
assert '"target-architecture.json":{"elements":[{"id":"E-1"}]}' in text
|
||||
assert '\\"elements\\"' not in text
|
||||
# Anything that is not JSON, or does not parse, is passed as it is.
|
||||
assert '"architecture.mmd":"graph TD\\n A[\\"x\\"]"' in text
|
||||
assert '"broken.json":"{not json"' in text
|
||||
Reference in New Issue
Block a user