After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
7.1 KiB
Python
215 lines
7.1 KiB
Python
"""A scripted AgentRuntime for deterministic tests and evaluation dry runs.
|
|
|
|
Each role has a queue of turns; a turn is a list of tool calls, or a function of the
|
|
task that returns one. Calls go through the real ToolSpecs, so every rule the tools
|
|
enforce is exercised exactly as a model would meet it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import deque
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
AgentTask,
|
|
Answer,
|
|
BudgetExhausted,
|
|
Outcome,
|
|
Question,
|
|
RoleSpec,
|
|
Stopped,
|
|
ToolSpec,
|
|
)
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Call:
|
|
tool: str
|
|
args: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
Turn = Sequence[Call] | Callable[[AgentTask], Sequence[Call]]
|
|
|
|
|
|
# One small, coherent Requirements chain, so sample submissions pass every link.
|
|
SAMPLE_DATA: dict[str, str] = {
|
|
"requirements.json": json.dumps(
|
|
[
|
|
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
|
|
{
|
|
"id": "NFR-1",
|
|
"kind": "non_functional",
|
|
"text": "99.9% availability.",
|
|
"conditions": ["The service stays available when one instance fails."],
|
|
},
|
|
]
|
|
),
|
|
"components.json": json.dumps(
|
|
[{"id": "C-1", "name": "Ingress", "requirements": ["FR-1", "NFR-1"]}]
|
|
),
|
|
"products.json": json.dumps([{"component": "C-1", "product": "Example Run"}]),
|
|
"recommendations.json": json.dumps(
|
|
[
|
|
{
|
|
"id": "DR-1",
|
|
"pillar": "security",
|
|
"text": "Use a dedicated identity.",
|
|
"components": ["C-1"],
|
|
"conditions": ["The service runs as an identity nothing else uses."],
|
|
}
|
|
]
|
|
),
|
|
"target-architecture.json": json.dumps(
|
|
{
|
|
"elements": [
|
|
{
|
|
"id": "E-1",
|
|
"name": "Ingress service",
|
|
"component": "C-1",
|
|
"product": "Example Run",
|
|
"resource_type": "example_run_service",
|
|
},
|
|
{
|
|
"id": "E-2",
|
|
"name": "Ingress identity",
|
|
"product": "Example IAM",
|
|
"resource_type": "example_service_account",
|
|
},
|
|
],
|
|
"relationships": [
|
|
{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "runs as"}
|
|
],
|
|
"checks": [
|
|
{
|
|
"id": "K-1",
|
|
"applies_to": "E-1",
|
|
"statement": "The service runs at least two instances.",
|
|
"source": "NFR-1",
|
|
},
|
|
{
|
|
"id": "K-2",
|
|
"applies_to": "E-2",
|
|
"statement": "The identity is used by this service only.",
|
|
"source": "DR-1",
|
|
},
|
|
],
|
|
}
|
|
),
|
|
"implementation.json": json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "example.run"},
|
|
{"check": "K-2", "file": "terraform/main.tf", "address": "example.sa"},
|
|
]
|
|
),
|
|
"verdicts.json": json.dumps(
|
|
{
|
|
"dry_run": {"verdict": "pass", "evidence": "terraform validate passed."},
|
|
"checks": [
|
|
{"check": "K-1", "verdict": "pass", "evidence": "min instances is 2."},
|
|
{"check": "K-2", "verdict": "pass", "evidence": "One binding."},
|
|
],
|
|
"proposed_additions": [],
|
|
}
|
|
),
|
|
}
|
|
|
|
|
|
def sample_submission(kind: DeliverableKind, note: str = "") -> Call:
|
|
"""A `submit_deliverable` call that meets the Deliverable's contract."""
|
|
spec = DEPENDENCY_GRAPH[kind]
|
|
parts = [f"# {kind}", note or "Summary."]
|
|
parts += [f"## {section}\n{note or 'Details.'}" for section in spec.sections]
|
|
attachments = {
|
|
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
|
}
|
|
return Call(
|
|
"submit_deliverable",
|
|
{"content": "\n".join(parts) + "\n", "attachments": attachments},
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunRecord:
|
|
role: str
|
|
task: AgentTask
|
|
tools: tuple[str, ...]
|
|
results: tuple[tuple[str, str], ...] # (tool name, message)
|
|
outcome: Outcome
|
|
|
|
|
|
class ScriptExhausted(AssertionError):
|
|
"""A role was run more times than the test scripted."""
|
|
|
|
|
|
class FakeAgentRuntime:
|
|
def __init__(self, turns: Mapping[str, Sequence[Turn]]) -> None:
|
|
self._turns = {role: deque(role_turns) for role, role_turns in turns.items()}
|
|
self.runs: list[RunRecord] = []
|
|
|
|
def remaining(self, role: str) -> int:
|
|
return len(self._turns.get(role, ()))
|
|
|
|
async def run(
|
|
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]
|
|
) -> Outcome:
|
|
queue = self._turns.get(role.name)
|
|
if not queue:
|
|
raise ScriptExhausted(f"No scripted turn left for role {role.name!r}")
|
|
turn = queue.popleft()
|
|
calls = turn(task) if callable(turn) else turn
|
|
by_name = {tool.name: tool for tool in tools}
|
|
|
|
results: list[tuple[str, str]] = []
|
|
outcome: Outcome = Stopped(
|
|
"The script ended without a tool that ends the turn."
|
|
)
|
|
for index, call in enumerate(calls):
|
|
if index >= role.budget:
|
|
outcome = BudgetExhausted(
|
|
f"{role.name} used all {role.budget} tool calls before finishing."
|
|
)
|
|
break
|
|
if call.tool not in by_name:
|
|
raise ValueError(
|
|
f"{role.name} called {call.tool!r}, which it was not offered: "
|
|
f"{sorted(by_name)}"
|
|
)
|
|
result = await by_name[call.tool].handler(**call.args)
|
|
results.append((call.tool, result.message))
|
|
if result.outcome is not None:
|
|
outcome = result.outcome
|
|
break
|
|
|
|
self.runs.append(
|
|
RunRecord(role.name, task, tuple(by_name), tuple(results), outcome)
|
|
)
|
|
return outcome
|
|
|
|
|
|
class FakeDecider:
|
|
"""A scripted Decider: answers by question id, or declines so callers fall back.
|
|
|
|
`answers` maps a question id to an Answer, or to a function of (state, question)
|
|
returning one. An id with no entry makes the whole decision unavailable.
|
|
"""
|
|
|
|
def __init__(self, answers: Mapping[str, Any] | None = None) -> None:
|
|
self.answers = dict(answers or {})
|
|
self.asked: list[tuple[str, dict[str, Question], str]] = []
|
|
|
|
async def decide(
|
|
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
|
|
) -> Mapping[str, Answer] | None:
|
|
self.asked.append((state, dict(questions), scope))
|
|
decided = {}
|
|
for name, question in questions.items():
|
|
if name not in self.answers:
|
|
return None
|
|
answer = self.answers[name]
|
|
decided[name] = answer(state, question) if callable(answer) else answer
|
|
return decided
|