Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
183 lines
5.9 KiB
Python
183 lines
5.9 KiB
Python
"""A scripted AgentRuntime for deterministic tests and evaluation dry runs.
|
|
|
|
Each role has a queue of turns; a turn is a list of tool calls, or a function of the
|
|
task that returns one. Calls go through the real ToolSpecs, so every rule the tools
|
|
enforce is exercised exactly as a model would meet it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import deque
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
AgentTask,
|
|
BudgetExhausted,
|
|
Outcome,
|
|
RoleSpec,
|
|
Stopped,
|
|
ToolSpec,
|
|
)
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Call:
|
|
tool: str
|
|
args: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]]
|
|
|
|
|
|
# One small, coherent Requirements chain, so sample submissions pass every link.
|
|
SAMPLE_DATA: dict[str, str] = {
|
|
"requirements.json": json.dumps(
|
|
[
|
|
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
|
|
{"id": "NFR-1", "kind": "non_functional", "text": "99.9% availability."},
|
|
]
|
|
),
|
|
"components.json": json.dumps(
|
|
[{"id": "C-1", "name": "Ingress", "requirements": ["FR-1", "NFR-1"]}]
|
|
),
|
|
"products.json": json.dumps([{"component": "C-1", "product": "Example Run"}]),
|
|
"recommendations.json": json.dumps(
|
|
[
|
|
{
|
|
"id": "DR-1",
|
|
"pillar": "security",
|
|
"text": "Use a dedicated identity.",
|
|
"components": ["C-1"],
|
|
}
|
|
]
|
|
),
|
|
"target-architecture.json": json.dumps(
|
|
{
|
|
"elements": [
|
|
{
|
|
"id": "E-1",
|
|
"name": "Ingress service",
|
|
"component": "C-1",
|
|
"product": "Example Run",
|
|
"resource_type": "example_run_service",
|
|
},
|
|
{
|
|
"id": "E-2",
|
|
"name": "Ingress identity",
|
|
"product": "Example IAM",
|
|
"resource_type": "example_service_account",
|
|
},
|
|
],
|
|
"relationships": [
|
|
{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "runs as"}
|
|
],
|
|
"checks": [
|
|
{
|
|
"id": "K-1",
|
|
"applies_to": "E-1",
|
|
"statement": "The service runs at least two instances.",
|
|
"source": "NFR-1",
|
|
},
|
|
{
|
|
"id": "K-2",
|
|
"applies_to": "E-2",
|
|
"statement": "The identity is used by this service only.",
|
|
"source": "DR-1",
|
|
},
|
|
],
|
|
}
|
|
),
|
|
"implementation.json": json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "example.run"},
|
|
{"check": "K-2", "file": "terraform/main.tf", "address": "example.sa"},
|
|
]
|
|
),
|
|
"verdicts.json": json.dumps(
|
|
{
|
|
"dry_run": {"verdict": "pass", "evidence": "terraform validate passed."},
|
|
"checks": [
|
|
{"check": "K-1", "verdict": "pass", "evidence": "min instances is 2."},
|
|
{"check": "K-2", "verdict": "pass", "evidence": "One binding."},
|
|
],
|
|
"proposed_additions": [],
|
|
}
|
|
),
|
|
}
|
|
|
|
|
|
def sample_submission(kind: DeliverableKind, note: str = "") -> Call:
|
|
"""A `submit_deliverable` call that meets the Deliverable's contract."""
|
|
spec = DEPENDENCY_GRAPH[kind]
|
|
parts = [f"# {kind}", note or "Summary."]
|
|
parts += [f"## {section}\n{note or 'Details.'}" for section in spec.sections]
|
|
attachments = {
|
|
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
|
}
|
|
return Call(
|
|
"submit_deliverable",
|
|
{"content": "\n".join(parts) + "\n", "attachments": attachments},
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunRecord:
|
|
role: str
|
|
task: AgentTask
|
|
tools: tuple[str, ...]
|
|
results: tuple[tuple[str, str], ...] # (tool name, message)
|
|
outcome: Outcome
|
|
|
|
|
|
class ScriptExhausted(AssertionError):
|
|
"""A role was run more times than the test scripted."""
|
|
|
|
|
|
class FakeAgentRuntime:
|
|
def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None:
|
|
self._turns = {role: deque(role_turns) for role, role_turns in turns.items()}
|
|
self.runs: list[RunRecord] = []
|
|
|
|
def remaining(self, role: str) -> int:
|
|
return len(self._turns.get(role, ()))
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
queue = self._turns.get(role.name)
|
|
if not queue:
|
|
raise ScriptExhausted(f"No scripted turn left for role {role.name!r}")
|
|
turn = queue.popleft()
|
|
calls = turn(task) if callable(turn) else turn
|
|
by_name = {tool.name: tool for tool in tools}
|
|
|
|
results: list[tuple[str, str]] = []
|
|
outcome: Outcome = Stopped(
|
|
"The script ended without a tool that ends the turn."
|
|
)
|
|
for index, call in enumerate(calls):
|
|
if index >= role.budget:
|
|
outcome = BudgetExhausted(
|
|
f"{role.name} used all {role.budget} tool calls before finishing."
|
|
)
|
|
break
|
|
if call.tool not in by_name:
|
|
raise ValueError(
|
|
f"{role.name} called {call.tool!r}, which it was not offered: "
|
|
f"{sorted(by_name)}"
|
|
)
|
|
result = await by_name[call.tool].handler(**call.args)
|
|
results.append((call.tool, result.message))
|
|
if result.outcome is not None:
|
|
outcome = result.outcome
|
|
break
|
|
|
|
self.runs.append(
|
|
RunRecord(role.name, task, tuple(by_name), tuple(results), outcome)
|
|
)
|
|
return outcome
|