Files
gcp_solution_architecture_a…/app/agent.py
Jonathan Boniface 58e778ca7a feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07:

- The validator records verdicts a few Checks at a time (verdict_put) and
  they are attached on submission. It timed out five times writing them in
  one reply. Verdicts on the same code survive a run cut short; verdicts on
  changed code are dropped. Its model timeout is 300 s.
- A schema review at submission, in code: every setting a Check names is
  looked up on its Element's resource type, and an unknown resource type is
  reported. Two Revisions in that run were for Checks no code could meet.
  On the two designs that passed validation it raises 2 of 96 Checks, both
  one metric name used as a value, so it is returned once and never blocks.
- Reviews run together; the Jev coverage review stays off.

Plans record both steps and the cancelled run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 09:39:13 +01:00

359 lines
13 KiB
Python

"""The agent: the ExecutionEngine that starts and resumes Executions, and its assembly.
ExecutionEngine is the one interface the A2A server, /generate and the evaluation
harness use. Each call runs the Execution until it needs the user or finishes, and
returns an Interaction describing which.
open_assembly is the composition root: the only place concrete runtimes, tools,
checkpointers and recorders are chosen, from settings and the provider bundle.
"""
from __future__ import annotations
import dataclasses
import uuid
from collections.abc import AsyncIterator, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from langchain_core.runnables import RunnableConfig
from langgraph.checkpoint.base import BaseCheckpointSaver
from langgraph.types import Command
from app.adk.roles import load_roles
from app.adk.runtime import (
AgentRuntime,
Decider,
NoDecider,
RoleSpec,
RunLimiter,
SubmissionReview,
ToolFactory,
)
from app.checkpointing import open_checkpointer
from app.config import Settings
from app.customer_case import record_intent
from app.nodes.router import (
REPLY_WRITER,
OrchestratorPolicy,
Router,
RuntimeReplyWriter,
Thresholds,
)
from app.providers import ProviderBundle, load_provider
from app.recorder import DatabaseRecorder, ExecutionRecorder
from app.skills.agent_skills import skill_version
from app.states.execution_state import check_customer_case
from app.states.ledger import Ledger
from app.tools.coverage import CoverageReview
from app.tools.execution import ORCHESTRATOR
from app.tools.jev import JevDecider, JevLimits
from app.tools.local import TerraformRunner, local_tools
from app.tools.mcp import McpCatalog, auth_from_config
from app.tools.schema_review import Reviews, SchemaReview
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.execution_graph import build_execution_graph
_RECURSION_LIMIT = 10_000 # loops are bounded by max_orchestrator_turns instead
@dataclass(frozen=True)
class Interaction:
execution_id: str
status: str # awaiting_user | completed | completed_with_warnings
prompt: Mapping[str, Any] | None # questions | approval | permission
state: Mapping[str, Any]
class ExecutionNotAwaitingUser(RuntimeError):
pass
class ExecutionEngine:
def __init__(
self,
*,
runtime: AgentRuntime,
roles: Mapping[str, RoleSpec],
recorder: ExecutionRecorder,
checkpointer: BaseCheckpointSaver,
output_root: Path,
max_revisions: int = 3,
max_orchestrator_turns: int = 100,
tool_factory: ToolFactory | None = None,
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
) -> None:
required = {
ORCHESTRATOR,
*(spec.specialist for spec in DEPENDENCY_GRAPH.values()),
}
missing = sorted(required - set(roles))
if missing:
raise ValueError(f"Missing roles: {', '.join(missing)}")
self._max_revisions = max_revisions
self._output_root = output_root
self._graph = build_execution_graph(
runtime=runtime,
roles=roles,
recorder=recorder,
output_root=output_root,
max_orchestrator_turns=max_orchestrator_turns,
tool_factory=tool_factory,
limiter=limiter,
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
).compile(checkpointer=checkpointer)
async def start(
self,
workflow_request: str,
*,
session_id: str | None = None,
customer_case: str | None = None,
pre_approved: bool = False,
allow_cloud_read: bool | None = None,
execution_id: str | None = None,
scripted_answers: Sequence[tuple[str, str]] = (),
) -> Interaction:
"""Start an Execution. `scripted_answers` are (regex, answer) pairs a
pre-approved Execution uses to answer questions, for evaluation runs."""
customer_case = check_customer_case(customer_case)
execution_id = execution_id or str(uuid.uuid4())
if await self._exists(execution_id):
raise ValueError(f"Execution {execution_id} already exists")
if pre_approved and allow_cloud_read is None:
allow_cloud_read = False
session_id = session_id or str(uuid.uuid4())
intent_version = record_intent(
self._output_root,
customer_case,
execution_id=execution_id,
session_id=session_id,
workflow_request=workflow_request,
)
initial: dict[str, Any] = {
"execution_id": execution_id,
"session_id": session_id,
"customer_case": customer_case,
"intent_version": intent_version,
"workflow_request": workflow_request,
"pre_approved": pre_approved,
"allow_cloud_read": allow_cloud_read,
"ledger": Ledger(
pre_approved=pre_approved, max_revisions=self._max_revisions
).to_dict(),
"deliverables": {},
"conversation": [{"from": "user", "text": workflow_request}],
"notes": [],
"pending_questions": [],
"awaiting_questions": [],
"answers": {},
"drafts": {},
"working": {},
"pending_permission": None,
"prompt": None,
"last_reply": None,
"pending_change": None,
"awaiting_gap_decision": False,
"validation_settled": 0,
"decisions": [],
"outcomes": [],
"refusals": [],
"outcomes_applied": 0,
"events_recorded": 0,
"turns": 0,
"status": "running",
"warnings": [],
"scripted_answers": [
[pattern, answer] for pattern, answer in scripted_answers
],
}
await self._graph.ainvoke(initial, self._config(execution_id))
return await self.interaction(execution_id)
async def resume(self, execution_id: str, reply: Mapping[str, Any]) -> Interaction:
"""Answer the Execution's current prompt: {"reply"} | {"approval"} | {"permission"}."""
current = await self.interaction(execution_id)
if current.status != "awaiting_user":
raise ExecutionNotAwaitingUser(
f"Execution {execution_id} is {current.status}, not awaiting the user"
)
await self._graph.ainvoke(
Command(resume=dict(reply)), self._config(execution_id)
)
return await self.interaction(execution_id)
async def interaction(self, execution_id: str) -> Interaction:
snapshot = await self._graph.aget_state(self._config(execution_id))
if not snapshot.values:
raise KeyError(f"Unknown Execution {execution_id}")
if snapshot.interrupts:
return Interaction(
execution_id,
"awaiting_user",
snapshot.interrupts[0].value,
snapshot.values,
)
return Interaction(
execution_id,
snapshot.values.get("status", "running"),
None,
snapshot.values,
)
async def _exists(self, execution_id: str) -> bool:
snapshot = await self._graph.aget_state(self._config(execution_id))
return bool(snapshot.values)
@staticmethod
def _config(execution_id: str) -> RunnableConfig:
return {
"configurable": {"thread_id": execution_id},
"recursion_limit": _RECURSION_LIMIT,
}
# --- assembly ----------------------------------------------------------------------
AGENTS_DIR = Path(__file__).resolve().parent.parent / "agents"
@dataclass(frozen=True)
class Assembly:
engine: ExecutionEngine
bundle: ProviderBundle
roles: dict[str, RoleSpec]
mcp_gaps: dict[str, str]
decider: Decider = field(default_factory=NoDecider)
def open_decider(settings: Settings) -> Decider:
"""The Jev client when a key is configured; otherwise a decider that always
sends its caller to the fallback."""
if not settings.TYPESAFE_API_KEY:
return NoDecider()
return JevDecider(
api_key=settings.TYPESAFE_API_KEY,
base_url=settings.TYPESAFE_BASE_URL,
model=settings.TYPESAFE_DEFAULT_MODEL,
limits=dataclasses.replace(
JevLimits(),
request_timeout=settings.JEV_TIMEOUT_SECONDS,
requests_per_second=settings.JEV_REQUESTS_PER_SECOND,
max_in_flight=settings.JEV_MAX_IN_FLIGHT,
budget_calls=settings.JEV_BUDGET_CALLS,
budget_tokens=settings.JEV_BUDGET_TOKENS,
),
)
@asynccontextmanager
async def open_assembly(
settings: Settings,
*,
runtime: AgentRuntime | None = None,
recorder: ExecutionRecorder | None = None,
output_root: Path | None = None,
) -> AsyncIterator[Assembly]:
bundle = load_provider(settings.CLOUD_PROVIDER)
roles = load_roles(
AGENTS_DIR,
bundle,
orchestrator_budget=settings.ORCHESTRATOR_BUDGET,
specialist_budget=settings.SPECIALIST_BUDGET,
include_preview_mcp=settings.MCP_ENABLE_PREVIEW,
)
if runtime is None:
from app.adk.adk_runtime import AdkAgentRuntime
runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
catalog = McpCatalog()
if settings.MCP_ENABLED:
used = {server_id for role in roles.values() for server_id in role.mcp_servers}
catalog = await McpCatalog.load(
[bundle.mcp_servers[server_id] for server_id in sorted(used)],
auth_from_config(bundle.mcp_auth, settings.MCP_QUOTA_PROJECT),
timeout=settings.MCP_TIMEOUT_SECONDS,
)
runner = TerraformRunner(
binary=settings.TERRAFORM_BINARY,
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
plugin_cache_dir=settings.terraform_plugin_cache(),
providers=bundle.terraform_providers,
)
offered = [name for names in bundle.tools.values() for name in names]
# One cap for Specialists and the pillar advisors they fan out to.
limiter = RunLimiter(settings.MAX_PARALLEL_SPECIALISTS)
tool_factory = make_tool_factory(
runtime,
roles,
bundle.pillar_skills,
local_tools=lambda working_copy: local_tools(offered, runner, working_copy),
catalog=catalog,
limiter=limiter,
)
decider = open_decider(settings)
orchestrator: OrchestratorPolicy | None = None
if settings.ORCHESTRATOR == "router":
# The Orchestrator routes: Jev reads, the playbook decides, and a model only
# writes what the user reads. With no Jev key the playbook runs on its rules.
orchestrator = Router(
decider,
RuntimeReplyWriter(runtime, roles[REPLY_WRITER]),
Thresholds(
act=settings.JEV_ACT_THRESHOLD,
close=settings.JEV_CLOSE_THRESHOLD,
confidence=settings.JEV_CONFIDENCE_THRESHOLD,
),
)
elif settings.ORCHESTRATOR != "model":
raise ValueError(
f"ORCHESTRATOR must be router or model, not {settings.ORCHESTRATOR!r}"
)
# What is looked at in a design before it is submitted. Each raises what it finds
# once and never blocks.
reviews: list[SubmissionReview] = []
if bundle.terraform_providers:
# In code: every setting a Check names exists on its resource type.
reviews.append(SchemaReview(runner))
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider):
# Optional, and off by default: Jev says which Check verifies each condition.
reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD))
async with open_checkpointer(settings) as checkpointer:
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=recorder or DatabaseRecorder(),
checkpointer=checkpointer,
output_root=output_root or settings.BASE_DIR,
max_revisions=settings.MAX_REVISIONS,
max_orchestrator_turns=settings.MAX_ORCHESTRATOR_TURNS,
tool_factory=tool_factory,
limiter=limiter,
skill_versions={
name: skill_version(meta.path) for name, meta in bundle.skills.items()
},
orchestrator=orchestrator,
submission_review=Reviews(reviews) if reviews else None,
)
try:
yield Assembly(engine, bundle, roles, dict(catalog.gaps), decider)
finally:
if isinstance(decider, JevDecider):
await decider.aclose()