"""A scripted AgentRuntime for deterministic tests and evaluation dry runs. Each role has a queue of turns; a turn is a list of tool calls, or a function of the task that returns one. Calls go through the real ToolSpecs, so every rule the tools enforce is exercised exactly as a model would meet it. """ from __future__ import annotations import json from collections import deque from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any from app.engine.runtime import ( AgentTask, Answer, BudgetExhausted, Outcome, Question, RoleSpec, Stopped, ToolSpec, ) from app.core.deliverables import DEPENDENCY_GRAPH, DeliverableKind, ProviderContract # What a sample carries of a Cloud provider: the source's six pillars and two code files. SAMPLE_CONTRACT = ProviderContract( ( "Security, privacy, and compliance", "Reliability", "Operational excellence", "Cost optimization", "Performance efficiency", "Sustainability", ), ("terraform/main.tf", "terraform/variables.tf"), ) @dataclass(frozen=True) class Call: tool: str args: Mapping[str, Any] = field(default_factory=dict) Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]] # One small, coherent Requirements chain, so sample submissions pass every link. SAMPLE_DATA: dict[str, str] = { "scope.yaml": "engagement:\n kind: workload\nintent:\n statement: Accept events.\n", "scope.json": json.dumps( { "kind": "workload", "stated": {"intent": {"statement": "Accept events."}}, "to_decide": [], "to_ask": [], "not_applicable": [], } ), "requirements.json": json.dumps( [ {"id": "FR-1", "kind": "functional", "text": "Accept events."}, { "id": "NFR-1", "kind": "non_functional", "text": "99.9% availability.", "conditions": ["The service stays available when one instance fails."], }, ] ), "components.json": json.dumps( [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1", "NFR-1"]}] ), "products.json": json.dumps([{"component": "C-1", "product": "Example Run"}]), "recommendations.json": json.dumps( [ { "id": "DR-1", "pillar": "security", "text": "Use a dedicated identity.", "components": ["C-1"], "conditions": ["The service runs as an identity nothing else uses."], } ] ), "target-architecture.json": json.dumps( { "elements": [ { "id": "E-1", "name": "Ingress service", "component": "C-1", "product": "Example Run", "resource_type": "example_run_service", }, { "id": "E-2", "name": "Ingress identity", "product": "Example IAM", "resource_type": "example_service_account", }, ], "relationships": [ {"id": "R-1", "from": "E-1", "to": "E-2", "kind": "runs as"} ], "checks": [ { "id": "K-1", "applies_to": "E-1", "statement": "The service runs at least two instances.", "source": "NFR-1", }, { "id": "K-2", "applies_to": "E-2", "statement": "The identity is used by this service only.", "source": "DR-1", }, ], } ), "implementation.json": json.dumps( [ {"check": "K-1", "file": "terraform/main.tf", "address": "example.run"}, {"check": "K-2", "file": "terraform/main.tf", "address": "example.sa"}, ] ), "work-breakdown.json": json.dumps( { "work_packages": [ { "id": "WP-1", "name": "Ingress", "parent": None, "elements": ["E-1", "E-2"], "scope": "The service and its identity.", "produces": ["a running service"], "skills": ["platform engineering"], } ], "activities": [ { "id": "A-1", "work_package": "WP-1", "name": "Create the identity", "days": 1, "after": [], }, { "id": "A-2", "work_package": "WP-1", "name": "Deploy the service", "days": 2, "after": ["A-1"], }, ], } ), "verdicts.json": json.dumps( { "dry_run": {"verdict": "pass", "evidence": "terraform validate passed."}, "checks": [ {"check": "K-1", "verdict": "pass", "evidence": "min instances is 2."}, {"check": "K-2", "verdict": "pass", "evidence": "One binding."}, ], "proposed_additions": [], } ), } def sample_submission(kind: DeliverableKind, note: str = "") -> Call: """A `submit_deliverable` call that meets the Deliverable's contract.""" spec = DEPENDENCY_GRAPH[kind] parts = [f"# {kind}", note or "Summary."] parts += [f"## {section}\n{note or 'Details.'}" for section in SAMPLE_CONTRACT.sections(kind)] attachments = { name: SAMPLE_DATA.get(name, f"# {name}\n") for name in SAMPLE_CONTRACT.attachments(kind) } return Call( "submit_deliverable", {"content": "\n".join(parts) + "\n", "attachments": attachments}, ) @dataclass(frozen=True) class RunRecord: role: str task: AgentTask tools: tuple[str, ...] results: tuple[tuple[str, str], ...] # (tool name, message) outcome: Outcome class ScriptExhausted(AssertionError): """A role was run more times than the test scripted.""" class FakeAgentRuntime: def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None: self._turns = {role: deque(role_turns) for role, role_turns in turns.items()} self.runs: list[RunRecord] = [] def remaining(self, role: str) -> int: return len(self._turns.get(role, ())) async def run( self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec] ) -> Outcome: queue = self._turns.get(role.name) if not queue: raise ScriptExhausted(f"No scripted turn left for role {role.name!r}") turn = queue.popleft() calls = turn(task) if callable(turn) else turn by_name = {tool.name: tool for tool in tools} results: list[tuple[str, str]] = [] outcome: Outcome = Stopped( "The script ended without a tool that ends the turn." ) for index, call in enumerate(calls): if index >= role.budget: outcome = BudgetExhausted( f"{role.name} used all {role.budget} tool calls before finishing." ) break if call.tool not in by_name: raise ValueError( f"{role.name} called {call.tool!r}, which it was not offered: " f"{sorted(by_name)}" ) result = await by_name[call.tool].handler(**call.args) results.append((call.tool, result.message)) if result.outcome is not None: outcome = result.outcome break self.runs.append( RunRecord(role.name, task, tuple(by_name), tuple(results), outcome) ) return outcome class FakeDecider: """A scripted Decider: answers by question id, or declines so callers fall back. `answers` maps a question id to an Answer, or to a function of (state, question) returning one. An id with no entry makes the whole decision unavailable. """ def __init__(self, answers: Mapping[str, Any] | None = None) -> None: self.answers = dict(answers or {}) self.asked: list[tuple[str, dict[str, Question], str]] = [] async def decide( self, state: str, questions: Mapping[str, Question], *, scope: str = "" ) -> Mapping[str, Answer] | None: self.asked.append((state, dict(questions), scope)) decided = {} for name, question in questions.items(): if name not in self.answers: return None answer = self.answers[name] decided[name] = answer(state, question) if callable(answer) else answer return decided