"""A scripted AgentRuntime for deterministic tests and evaluation dry runs. Each role has a queue of turns; a turn is a list of tool calls, or a function of the task that returns one. Calls go through the real ToolSpecs, so every rule the tools enforce is exercised exactly as a model would meet it. """ from __future__ import annotations import json from collections import deque from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any from app.adk.runtime import ( AgentTask, Answer, BudgetExhausted, Outcome, Question, RoleSpec, Stopped, ToolSpec, ) from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind @dataclass(frozen=True) class Call: tool: str args: Mapping[str, Any] = field(default_factory=dict) Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]] # One small, coherent Requirements chain, so sample submissions pass every link. SAMPLE_DATA: dict[str, str] = { "requirements.json": json.dumps( [ {"id": "FR-1", "kind": "functional", "text": "Accept events."}, {"id": "NFR-1", "kind": "non_functional", "text": "99.9% availability."}, ] ), "components.json": json.dumps( [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1", "NFR-1"]}] ), "products.json": json.dumps([{"component": "C-1", "product": "Example Run"}]), "recommendations.json": json.dumps( [ { "id": "DR-1", "pillar": "security", "text": "Use a dedicated identity.", "components": ["C-1"], } ] ), "target-architecture.json": json.dumps( { "elements": [ { "id": "E-1", "name": "Ingress service", "component": "C-1", "product": "Example Run", "resource_type": "example_run_service", }, { "id": "E-2", "name": "Ingress identity", "product": "Example IAM", "resource_type": "example_service_account", }, ], "relationships": [ {"id": "R-1", "from": "E-1", "to": "E-2", "kind": "runs as"} ], "checks": [ { "id": "K-1", "applies_to": "E-1", "statement": "The service runs at least two instances.", "source": "NFR-1", }, { "id": "K-2", "applies_to": "E-2", "statement": "The identity is used by this service only.", "source": "DR-1", }, ], } ), "implementation.json": json.dumps( [ {"check": "K-1", "file": "terraform/main.tf", "address": "example.run"}, {"check": "K-2", "file": "terraform/main.tf", "address": "example.sa"}, ] ), "verdicts.json": json.dumps( { "dry_run": {"verdict": "pass", "evidence": "terraform validate passed."}, "checks": [ {"check": "K-1", "verdict": "pass", "evidence": "min instances is 2."}, {"check": "K-2", "verdict": "pass", "evidence": "One binding."}, ], "proposed_additions": [], } ), } def sample_submission(kind: DeliverableKind, note: str = "") -> Call: """A `submit_deliverable` call that meets the Deliverable's contract.""" spec = DEPENDENCY_GRAPH[kind] parts = [f"# {kind}", note or "Summary."] parts += [f"## {section}\n{note or 'Details.'}" for section in spec.sections] attachments = { name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments } return Call( "submit_deliverable", {"content": "\n".join(parts) + "\n", "attachments": attachments}, ) @dataclass(frozen=True) class RunRecord: role: str task: AgentTask tools: tuple[str, ...] results: tuple[tuple[str, str], ...] # (tool name, message) outcome: Outcome class ScriptExhausted(AssertionError): """A role was run more times than the test scripted.""" class FakeAgentRuntime: def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None: self._turns = {role: deque(role_turns) for role, role_turns in turns.items()} self.runs: list[RunRecord] = [] def remaining(self, role: str) -> int: return len(self._turns.get(role, ())) async def run( self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec] ) -> Outcome: queue = self._turns.get(role.name) if not queue: raise ScriptExhausted(f"No scripted turn left for role {role.name!r}") turn = queue.popleft() calls = turn(task) if callable(turn) else turn by_name = {tool.name: tool for tool in tools} results: list[tuple[str, str]] = [] outcome: Outcome = Stopped( "The script ended without a tool that ends the turn." ) for index, call in enumerate(calls): if index >= role.budget: outcome = BudgetExhausted( f"{role.name} used all {role.budget} tool calls before finishing." ) break if call.tool not in by_name: raise ValueError( f"{role.name} called {call.tool!r}, which it was not offered: " f"{sorted(by_name)}" ) result = await by_name[call.tool].handler(**call.args) results.append((call.tool, result.message)) if result.outcome is not None: outcome = result.outcome break self.runs.append( RunRecord(role.name, task, tuple(by_name), tuple(results), outcome) ) return outcome class FakeDecider: """A scripted Decider: answers by question id, or declines so callers fall back. `answers` maps a question id to an Answer, or to a function of (state, question) returning one. An id with no entry makes the whole decision unavailable. """ def __init__(self, answers: Mapping[str, Any] | None = None) -> None: self.answers = dict(answers or {}) self.asked: list[tuple[str, dict[str, Question], str]] = [] async def decide( self, state: str, questions: Mapping[str, Question], *, scope: str = "" ) -> Mapping[str, Answer] | None: self.asked.append((state, dict(questions), scope)) decided = {} for name, question in questions.items(): if name not in self.answers: return None answer = self.answers[name] decided[name] = answer(state, question) if callable(answer) else answer return decided