feat: working orhcestration engine

This commit is contained in:
2026-10-21 19:50:24 +01:00
commit ce5e2f20cd
161 changed files with 31794 additions and 0 deletions

0
tests/agents/__init__.py Normal file
View File

View File

@@ -0,0 +1,421 @@
"""The ADK implementation of AgentRuntime, driven by a scripted model (no network)."""
from collections.abc import AsyncGenerator
from pathlib import Path
import httpx
import pytest
from google.adk.models.base_llm import BaseLlm
from google.adk.models.google_llm import Gemini
from google.adk.models.lite_llm import LiteLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.genai import types
from pydantic import Field
from execution_engine.adk.adk_runtime import AdkAgentRuntime, default_model_factory
from execution_engine.adk.runtime import (
AgentTask,
BudgetExhausted,
NeedsInput,
RoleSpec,
Stopped,
Submitted,
ToolResult,
ToolSpec,
TurnEnded,
)
from execution_engine.skills.agent_skills import discover_skills
# Stand-in Skills: the engine has none of its own.
SKILLS = {
name: meta.path
for name, meta in discover_skills(
[Path(__file__).parents[1] / "fixtures" / "skills"]
).items()
}
class ScriptedLlm(BaseLlm):
"""Returns one scripted response per model call and records every request."""
model: str = "scripted"
script: list[list[types.Part]] = Field(default_factory=list)
requests: list[LlmRequest] = Field(default_factory=list)
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
self.requests.append(llm_request)
parts = self.script.pop(0) if self.script else [types.Part(text="(no script)")]
yield LlmResponse(content=types.Content(role="model", parts=parts))
def call(tool: str, **args) -> types.Part:
return types.Part(function_call=types.FunctionCall(name=tool, args=args))
def text(value: str) -> types.Part:
return types.Part(text=value)
def runtime_with(model: ScriptedLlm) -> AdkAgentRuntime:
return AdkAgentRuntime(skill_dirs=SKILLS, model_factory=lambda role: model)
class Recorder:
"""ToolSpecs whose handlers record their calls."""
def __init__(self) -> None:
self.calls: list[tuple[str, dict]] = []
def tool(self, name: str, outcome=None, message: str = "ok") -> ToolSpec:
async def handler(**kwargs) -> ToolResult:
self.calls.append((name, kwargs))
return ToolResult(f"{message}: {kwargs}", outcome)
return ToolSpec(
name,
f"The {name} tool.",
{"type": "object", "properties": {"note": {"type": "string"}}},
handler,
)
ROLE = RoleSpec(
"iac-author",
"Write Terraform. Keep {braces} and {{doubles}} verbatim.",
budget=5,
skills=("toy-design",),
)
TASK = AgentTask("iac-author", "Produce deployment_guidance.", {"feedback": ["x"]})
async def test_specialist_loads_its_skill_and_submits():
recorder = Recorder()
submitted = Submitted("# Guidance", {"terraform/main.tf": "resource {}"})
model = ScriptedLlm(
script=[
[call("load_skill", skill_name="toy-design")],
[call("submit_deliverable", note="done")],
]
)
outcome = await runtime_with(model).run(
ROLE, TASK, [recorder.tool("submit_deliverable", submitted)]
)
assert outcome == submitted
assert recorder.calls == [("submit_deliverable", {"note": "done"})]
skill_response = _function_responses(model.requests[1])["load_skill"]
assert "Nothing here is guidance" in str(skill_response)
async def test_agent_is_offered_its_tools_and_skill_tools_without_scripts():
model = ScriptedLlm(script=[[text("nothing to do")]])
await runtime_with(model).run(ROLE, TASK, [Recorder().tool("submit_deliverable")])
request = model.requests[0]
assert set(request.tools_dict) == {
"submit_deliverable",
"list_skills",
"load_skill",
"load_skill_resource",
}
declaration = request.tools_dict["submit_deliverable"]._get_declaration()
assert declaration.parameters_json_schema["properties"] == {
"note": {"type": "string"}
}
async def test_role_without_skills_gets_no_skill_tools():
model = ScriptedLlm(script=[[text("done")]])
role = RoleSpec("decomposer", "Decompose.", budget=3)
await runtime_with(model).run(role, TASK, [Recorder().tool("needs_input")])
assert set(model.requests[0].tools_dict) == {"needs_input"}
async def test_only_the_roles_skills_can_be_loaded():
model = ScriptedLlm(
script=[
[call("load_skill", skill_name="toy-security")],
[text("ok")],
]
)
await runtime_with(model).run(ROLE, TASK, [])
response = _function_responses(model.requests[1])["load_skill"]
assert "error" in response
assert "toy-security-body" not in str(response)
async def test_instructions_and_task_context_reach_the_model_verbatim():
model = ScriptedLlm(script=[[text("done")]])
await runtime_with(model).run(ROLE, TASK, [])
request = model.requests[0]
assert (
"Keep {braces} and {{doubles}} verbatim." in request.config.system_instruction
)
user_text = request.contents[0].parts[0].text
assert "Produce deployment_guidance." in user_text
# Written compactly: the context is sent again with every request of the run.
assert '"feedback":["x"]' in user_text
async def test_turn_ends_as_soon_as_a_tool_returns_an_outcome():
recorder = Recorder()
model = ScriptedLlm(
script=[
[call("delegate"), call("ask_user")],
[call("ask_user")],
]
)
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=10),
TASK,
[
recorder.tool("delegate", TurnEnded("delegate")),
recorder.tool("ask_user", TurnEnded("ask_user")),
],
)
assert outcome == TurnEnded("delegate")
assert [name for name, _ in recorder.calls] == ["delegate"]
assert len(model.requests) == 1
async def test_budget_counts_every_tool_call_and_stops_the_run():
recorder = Recorder()
model = ScriptedLlm(script=[[call("ready_work")] for _ in range(5)])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=2),
TASK,
[recorder.tool("ready_work")],
)
assert isinstance(outcome, BudgetExhausted)
assert "2 tool calls" in outcome.summary
assert len(recorder.calls) == 2
async def test_model_reply_without_a_turn_ending_tool_is_stopped_after_nudges():
model = ScriptedLlm(
script=[[text("I need the RTO before I can continue.")], [text("")], [text("")]]
)
outcome = await runtime_with(model).run(ROLE, TASK, [])
assert outcome == Stopped("I need the RTO before I can continue.")
assert len(model.requests) == 3
assert "without calling a tool that ends your turn" in (
model.requests[1].contents[-1].parts[0].text
)
async def test_a_nudged_agent_that_then_ends_its_turn_succeeds_in_one_session():
recorder = Recorder()
model = ScriptedLlm(script=[[text("")], [call("delegate", note="go")]])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[recorder.tool("delegate", TurnEnded("delegate"))],
)
assert outcome == TurnEnded("delegate")
assert "without calling a tool" in model.requests[1].contents[-1].parts[0].text
class TimingOutLlm(ScriptedLlm):
"""Times out on the first request, then follows its script."""
timed_out: bool = False
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
if not self.timed_out:
self.timed_out = True
raise httpx.ReadTimeout("no response")
async for response in super().generate_content_async(llm_request, stream):
yield response
async def test_a_timed_out_model_request_is_retried_in_the_same_session():
recorder = Recorder()
model = TimingOutLlm(script=[[call("delegate", note="go")]])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[recorder.tool("delegate", TurnEnded("delegate"))],
)
assert outcome == TurnEnded("delegate")
assert "timed out" in model.requests[0].contents[-1].parts[0].text
async def test_model_timeout_is_set_on_every_request():
model = ScriptedLlm(script=[[call("delegate")]])
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
)
await runtime.run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[Recorder().tool("delegate", TurnEnded("delegate"))],
)
assert model.requests[0].config.http_options.timeout == 90_000
async def test_a_role_with_its_own_timeout_is_given_it_over_the_default():
model = ScriptedLlm(script=[[call("delegate")]])
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
)
await runtime.run(
RoleSpec("orchestrator", "Coordinate.", budget=5, model_timeout_seconds=420),
TASK,
[Recorder().tool("delegate", TurnEnded("delegate"))],
)
assert model.requests[0].config.http_options.timeout == 420_000
class MeteredLlm(ScriptedLlm):
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
async for response in super().generate_content_async(llm_request, stream):
response.usage_metadata = types.GenerateContentResponseUsageMetadata(
prompt_token_count=100,
cached_content_token_count=40,
candidates_token_count=10,
thoughts_token_count=5,
total_token_count=115,
)
yield response
async def test_token_usage_is_summed_per_role_and_model():
model = MeteredLlm(script=[[text("")], [call("delegate")]])
role = RoleSpec("orchestrator", "Coordinate.", budget=5, model="gemini/x")
runtime = runtime_with(model)
await runtime.run(role, TASK, [Recorder().tool("delegate", TurnEnded("delegate"))])
usage = runtime.usage[("orchestrator", "gemini/x")]
assert (usage.requests, usage.prompt, usage.cached, usage.output) == (
2,
200,
80,
20,
)
assert (usage.thinking, usage.total) == (10, 230)
async def test_task_message_names_the_roles_skills():
model = ScriptedLlm(script=[[text("done")]] * 3)
await runtime_with(model).run(ROLE, TASK, [])
first = model.requests[0].contents[0].parts[0].text
assert "Skills available to you (load with load_skill): toy-design." in first
async def test_needs_input_outcome_passes_through():
needs = NeedsInput(("What region?",))
model = ScriptedLlm(script=[[call("needs_input")]])
outcome = await runtime_with(model).run(
ROLE, TASK, [Recorder().tool("needs_input", needs)]
)
assert outcome == needs
async def test_each_run_starts_with_a_fresh_session():
model = ScriptedLlm(script=[[text("first")], [text("second")]])
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, max_nudges=0
)
await runtime.run(ROLE, TASK, [])
await runtime.run(ROLE, TASK, [])
assert len(model.requests[1].contents) == 1
def test_default_model_factory_uses_native_gemini_and_litellm_for_others():
gemini = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-3.8-flash")
)
other = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="anthropic/claude-sonnet-5")
)
assert isinstance(gemini, Gemini)
assert gemini.model == "gemini-3.8-flash"
assert gemini.retry_options.attempts == 5
assert isinstance(other, LiteLlm)
assert other.model == "anthropic/claude-sonnet-5"
with pytest.raises(ValueError, match="model"):
default_model_factory(RoleSpec("diagrammer", "", budget=1))
def test_unknown_skill_is_rejected_when_the_runtime_is_built():
with pytest.raises(ValueError, match="no-such-skill"):
AdkAgentRuntime(
skill_dirs={"no-such-skill": Path("/nonexistent")},
model_factory=lambda role: ScriptedLlm(),
)
def _function_responses(request: LlmRequest) -> dict:
responses = {}
for content in request.contents:
for part in content.parts or []:
if part.function_response:
responses[part.function_response.name] = part.function_response.response
return responses
def test_a_data_file_is_written_for_the_model_as_data_not_as_escaped_text():
from execution_engine.adk.adk_runtime import _render_task
from execution_engine.adk.runtime import AgentTask
task = AgentTask(
"iac-author",
"Produce.",
{
"dependencies": {
"target_architecture": {
"content": "prose",
"attachments": {
"target-architecture.json": '{"elements": [{"id": "E-1"}]}',
"architecture.mmd": 'graph TD\n A["x"]',
"broken.json": "{not json",
},
}
}
},
)
text = _render_task(task)
assert '"target-architecture.json":{"elements":[{"id":"E-1"}]}' in text
assert '\\"elements\\"' not in text
# Anything that is not JSON, or does not parse, is passed as it is.
assert '"architecture.mmd":"graph TD\\n A[\\"x\\"]"' in text
assert '"broken.json":"{not json"' in text