Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
505 lines
19 KiB
Python
505 lines
19 KiB
Python
"""The agent: the ExecutionEngine that starts and resumes Executions, and its assembly.
|
|
|
|
ExecutionEngine is the one interface the A2A server, /generate and the evaluation
|
|
harness use. Each call runs the Execution until it needs the user or finishes, and
|
|
returns an Interaction describing which.
|
|
|
|
open_assembly is the composition root: the only place concrete runtimes, tools,
|
|
checkpointers and recorders are chosen, from settings and the provider bundle.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import uuid
|
|
from collections.abc import AsyncIterator, Mapping, Sequence
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from langchain_core.runnables import RunnableConfig
|
|
from langgraph.checkpoint.base import BaseCheckpointSaver
|
|
from langgraph.types import Command
|
|
|
|
from app.adk.roles import load_roles
|
|
from app.adk.runtime import (
|
|
AgentRuntime,
|
|
Decider,
|
|
NoDecider,
|
|
RoleSpec,
|
|
RunLimiter,
|
|
SubmissionReview,
|
|
ToolFactory,
|
|
)
|
|
from app.baseline import load_baseline
|
|
from app.checkpointing import open_checkpointer
|
|
from app.config import Settings
|
|
from app.customer_case import record_intent
|
|
from app.nodes.intake import Intake
|
|
from app.nodes.router import (
|
|
REPLY_WRITER,
|
|
OrchestratorPolicy,
|
|
Router,
|
|
RuntimeReplyWriter,
|
|
Thresholds,
|
|
)
|
|
from app.providers import ProviderBundle, load_provider
|
|
from app.recorder import DatabaseRecorder, ExecutionRecorder
|
|
from app.skills.agent_skills import discover_skills, skill_version
|
|
from app.states.execution_state import check_customer_case
|
|
from app.states.ledger import Ledger
|
|
from app.tools.coverage import CoverageReview
|
|
from app.tools.execution import ORCHESTRATOR
|
|
from app.tools.jev import JevDecider, JevLimits
|
|
from app.tools.local import TerraformRunner, local_tools
|
|
from app.tools.mcp import McpCatalog, auth_from_config
|
|
from app.tools.requirements_review import RequirementsReview
|
|
from app.tools.schema_review import Reviews, SchemaReview
|
|
from app.tools.skill_routing import routable
|
|
from app.tools.toolkit import make_tool_factory
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
|
from app.workflows.execution_graph import build_execution_graph
|
|
|
|
_RECURSION_LIMIT = 10_000 # loops are bounded by max_orchestrator_turns instead
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Interaction:
|
|
execution_id: str
|
|
status: str # awaiting_user | completed | completed_with_warnings
|
|
prompt: Mapping[str, Any] | None # questions | approval | permission
|
|
state: Mapping[str, Any]
|
|
|
|
|
|
class ExecutionNotAwaitingUser(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ExecutionEngine:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
runtime: AgentRuntime,
|
|
roles: Mapping[str, RoleSpec],
|
|
recorder: ExecutionRecorder,
|
|
checkpointer: BaseCheckpointSaver,
|
|
output_root: Path,
|
|
max_revisions: int = 3,
|
|
max_orchestrator_turns: int = 100,
|
|
tool_factory: ToolFactory | None = None,
|
|
limiter: RunLimiter | None = None,
|
|
skill_versions: Mapping[str, str] | None = None,
|
|
orchestrator: OrchestratorPolicy | None = None,
|
|
submission_review: SubmissionReview | None = None,
|
|
confirm_drafts: bool = False,
|
|
decider: Decider | None = None,
|
|
pillar_threshold: float = 1.5,
|
|
reworded_threshold: float = 0.25,
|
|
overlays: Mapping[str, str] | None = None,
|
|
default_overlays: Sequence[str] = (),
|
|
guard_user_text: bool = False,
|
|
guard_threshold: float = 0.75,
|
|
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
|
|
skill_threshold: float = 1.25,
|
|
assess_at_end: bool = False,
|
|
) -> None:
|
|
required = {
|
|
ORCHESTRATOR,
|
|
*(spec.specialist for spec in DEPENDENCY_GRAPH.values()),
|
|
}
|
|
missing = sorted(required - set(roles))
|
|
if missing:
|
|
raise ValueError(f"Missing roles: {', '.join(missing)}")
|
|
self._max_revisions = max_revisions
|
|
self._output_root = output_root
|
|
# Overlay Skills an Execution may name, with the version of each (ADR-0004).
|
|
self._overlays = dict(overlays or {})
|
|
self._default_overlays = tuple(default_overlays)
|
|
unknown = sorted(set(self._default_overlays) - set(self._overlays))
|
|
if unknown:
|
|
raise ValueError(f"Unknown default overlay Skills: {', '.join(unknown)}")
|
|
self._graph = build_execution_graph(
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=recorder,
|
|
output_root=output_root,
|
|
max_orchestrator_turns=max_orchestrator_turns,
|
|
tool_factory=tool_factory,
|
|
limiter=limiter,
|
|
skill_versions=skill_versions,
|
|
orchestrator=orchestrator,
|
|
submission_review=submission_review,
|
|
confirm_drafts=confirm_drafts,
|
|
decider=decider,
|
|
pillar_threshold=pillar_threshold,
|
|
reworded_threshold=reworded_threshold,
|
|
guard_user_text=guard_user_text,
|
|
guard_threshold=guard_threshold,
|
|
skill_references=skill_references,
|
|
skill_threshold=skill_threshold,
|
|
assess_at_end=assess_at_end,
|
|
).compile(checkpointer=checkpointer)
|
|
|
|
async def start(
|
|
self,
|
|
workflow_request: str,
|
|
*,
|
|
session_id: str | None = None,
|
|
customer_case: str | None = None,
|
|
pre_approved: bool = False,
|
|
allow_cloud_read: bool | None = None,
|
|
execution_id: str | None = None,
|
|
scripted_answers: Sequence[Sequence[str]] = (),
|
|
baseline: str | None = None,
|
|
overlay_skills: Sequence[str] = (),
|
|
) -> Interaction:
|
|
"""Start an Execution. `scripted_answers` are (regex, answer[, asker]) entries a
|
|
pre-approved Execution uses to answer questions, for evaluation runs.
|
|
|
|
`baseline` names a completed Execution of the same Customer case and makes
|
|
this a Change request (ADR-0006): the request says what has changed, and the
|
|
Execution starts from that Baseline's Deliverables. It is refused, before
|
|
anything runs or is recorded, when the Baseline cannot be one.
|
|
|
|
`overlay_skills` names Skills that carry an organisation's standards or
|
|
this engagement's constraints. Each must be one the deployment has
|
|
configured; an unknown name refuses the start. Every Specialist loads them
|
|
with the ones the deployment loads by default.
|
|
"""
|
|
customer_case = check_customer_case(customer_case)
|
|
execution_id = execution_id or str(uuid.uuid4())
|
|
if await self._exists(execution_id):
|
|
raise ValueError(f"Execution {execution_id} already exists")
|
|
if isinstance(overlay_skills, str) or not all(
|
|
isinstance(name, str) for name in overlay_skills
|
|
):
|
|
raise ValueError("overlay_skills must be a list of Skill names")
|
|
chosen = list(dict.fromkeys([*self._default_overlays, *overlay_skills]))
|
|
unknown = sorted(set(chosen) - set(self._overlays))
|
|
if unknown:
|
|
raise ValueError(
|
|
f"Unknown overlay Skills: {', '.join(unknown)}. Available: "
|
|
f"{', '.join(sorted(self._overlays)) or 'none'}"
|
|
)
|
|
loaded = (
|
|
load_baseline(self._output_root, customer_case, baseline)
|
|
if baseline
|
|
else None
|
|
)
|
|
if pre_approved and allow_cloud_read is None:
|
|
allow_cloud_read = False
|
|
session_id = session_id or str(uuid.uuid4())
|
|
intent_version = record_intent(
|
|
self._output_root,
|
|
customer_case,
|
|
execution_id=execution_id,
|
|
session_id=session_id,
|
|
workflow_request=workflow_request,
|
|
baseline=baseline or None,
|
|
)
|
|
initial: dict[str, Any] = {
|
|
"execution_id": execution_id,
|
|
"session_id": session_id,
|
|
"customer_case": customer_case,
|
|
"intent_version": intent_version,
|
|
"workflow_request": workflow_request,
|
|
# The Baseline of a Change request, as read from its files; else None.
|
|
"baseline": loaded,
|
|
# Overlay Skills this Execution loads, each with the version it had.
|
|
"overlay_skills": {name: self._overlays[name] for name in chosen},
|
|
"risks": [],
|
|
"pre_approved": pre_approved,
|
|
"allow_cloud_read": allow_cloud_read,
|
|
"ledger": Ledger(
|
|
pre_approved=pre_approved, max_revisions=self._max_revisions
|
|
).to_dict(),
|
|
"deliverables": {},
|
|
"conversation": [{"from": "user", "text": workflow_request}],
|
|
"notes": [],
|
|
"pending_questions": [],
|
|
"awaiting_questions": [],
|
|
"answers": {},
|
|
"drafts": {},
|
|
"drafts_shown": [],
|
|
"working": {},
|
|
"pending_permission": None,
|
|
"prompt": None,
|
|
"last_reply": None,
|
|
"pending_change": None,
|
|
"awaiting_gap_decision": False,
|
|
"validation_settled": 0,
|
|
"awaiting_limit_decision": False,
|
|
"revision_limit_asked": [],
|
|
"idle_asks": 0,
|
|
"ask_fingerprint": "",
|
|
"decisions": [],
|
|
"outcomes": [],
|
|
"refusals": [],
|
|
"outcomes_applied": 0,
|
|
"events_recorded": 0,
|
|
"turns": 0,
|
|
"status": "running",
|
|
"warnings": [],
|
|
"scripted_answers": [list(entry) for entry in scripted_answers],
|
|
}
|
|
await self._graph.ainvoke(initial, self._config(execution_id))
|
|
return await self.interaction(execution_id)
|
|
|
|
async def resume(self, execution_id: str, reply: Mapping[str, Any]) -> Interaction:
|
|
"""Answer the Execution's current prompt: {"reply"} | {"approval"} | {"permission"}."""
|
|
current = await self.interaction(execution_id)
|
|
if current.status != "awaiting_user":
|
|
raise ExecutionNotAwaitingUser(
|
|
f"Execution {execution_id} is {current.status}, not awaiting the user"
|
|
)
|
|
await self._graph.ainvoke(
|
|
Command(resume=dict(reply)), self._config(execution_id)
|
|
)
|
|
return await self.interaction(execution_id)
|
|
|
|
async def interaction(self, execution_id: str) -> Interaction:
|
|
snapshot = await self._graph.aget_state(self._config(execution_id))
|
|
if not snapshot.values:
|
|
raise KeyError(f"Unknown Execution {execution_id}")
|
|
if snapshot.interrupts:
|
|
return Interaction(
|
|
execution_id,
|
|
"awaiting_user",
|
|
snapshot.interrupts[0].value,
|
|
snapshot.values,
|
|
)
|
|
return Interaction(
|
|
execution_id,
|
|
snapshot.values.get("status", "running"),
|
|
None,
|
|
snapshot.values,
|
|
)
|
|
|
|
async def _exists(self, execution_id: str) -> bool:
|
|
snapshot = await self._graph.aget_state(self._config(execution_id))
|
|
return bool(snapshot.values)
|
|
|
|
@staticmethod
|
|
def _config(execution_id: str) -> RunnableConfig:
|
|
return {
|
|
"configurable": {"thread_id": execution_id},
|
|
"recursion_limit": _RECURSION_LIMIT,
|
|
}
|
|
|
|
|
|
# --- assembly ----------------------------------------------------------------------
|
|
|
|
AGENTS_DIR = Path(__file__).resolve().parent.parent / "agents"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Assembly:
|
|
engine: ExecutionEngine
|
|
bundle: ProviderBundle
|
|
roles: dict[str, RoleSpec]
|
|
mcp_gaps: dict[str, str]
|
|
decider: Decider = field(default_factory=NoDecider)
|
|
intake: Intake | None = None # reads a message that starts something
|
|
|
|
|
|
def discover_overlays(settings: Settings, bundle: ProviderBundle) -> dict[str, Any]:
|
|
"""The overlay Skills this deployment has configured, by name (ADR-0004).
|
|
|
|
They come only from the folders the deployment names, pass the same validation
|
|
as the provider's Skills, and may not reuse one of the provider's names.
|
|
"""
|
|
folders = [folder for folder in settings.overlay_dirs() if folder.is_dir()]
|
|
overlays = discover_skills(folders)
|
|
clash = sorted(set(overlays) & set(bundle.skills))
|
|
if clash:
|
|
raise ValueError(
|
|
f"Overlay Skills reuse the names of provider Skills: {', '.join(clash)}"
|
|
)
|
|
return overlays
|
|
|
|
|
|
def skill_dirs(settings: Settings, bundle: ProviderBundle) -> dict[str, Path]:
|
|
"""Every Skill a runtime may be asked to load: the provider's and the overlays."""
|
|
return {
|
|
**{name: meta.path for name, meta in bundle.skills.items()},
|
|
**{
|
|
name: meta.path
|
|
for name, meta in discover_overlays(settings, bundle).items()
|
|
},
|
|
}
|
|
|
|
|
|
def role_references(
|
|
roles: Mapping[str, RoleSpec], bundle: ProviderBundle
|
|
) -> dict[str, dict[str, dict[str, str]]]:
|
|
"""For each role, its Skills that hold enough references to choose among, with
|
|
what each reference covers (iteration 5 step 8)."""
|
|
worth = routable({name: meta.path for name, meta in bundle.skills.items()})
|
|
by_role = {
|
|
name: {skill: worth[skill] for skill in role.skills if skill in worth}
|
|
for name, role in roles.items()
|
|
}
|
|
return {name: skills for name, skills in by_role.items() if skills}
|
|
|
|
|
|
def open_decider(settings: Settings) -> Decider:
|
|
"""The Jev client when a key is configured; otherwise a decider that always
|
|
sends its caller to the fallback."""
|
|
if not settings.TYPESAFE_API_KEY:
|
|
return NoDecider()
|
|
return JevDecider(
|
|
api_key=settings.TYPESAFE_API_KEY,
|
|
base_url=settings.TYPESAFE_BASE_URL,
|
|
model=settings.TYPESAFE_DEFAULT_MODEL,
|
|
limits=dataclasses.replace(
|
|
JevLimits(),
|
|
request_timeout=settings.JEV_TIMEOUT_SECONDS,
|
|
requests_per_second=settings.JEV_REQUESTS_PER_SECOND,
|
|
max_in_flight=settings.JEV_MAX_IN_FLIGHT,
|
|
budget_calls=settings.JEV_BUDGET_CALLS,
|
|
budget_tokens=settings.JEV_BUDGET_TOKENS,
|
|
),
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def open_assembly(
|
|
settings: Settings,
|
|
*,
|
|
runtime: AgentRuntime | None = None,
|
|
recorder: ExecutionRecorder | None = None,
|
|
output_root: Path | None = None,
|
|
) -> AsyncIterator[Assembly]:
|
|
bundle = load_provider(settings.CLOUD_PROVIDER)
|
|
roles = load_roles(
|
|
AGENTS_DIR,
|
|
bundle,
|
|
orchestrator_budget=settings.ORCHESTRATOR_BUDGET,
|
|
specialist_budget=settings.SPECIALIST_BUDGET,
|
|
include_preview_mcp=settings.MCP_ENABLE_PREVIEW,
|
|
lessons=settings.LESSONS_ENABLED,
|
|
)
|
|
overlays = discover_overlays(settings, bundle)
|
|
if runtime is None:
|
|
from app.adk.adk_runtime import AdkAgentRuntime
|
|
|
|
runtime = AdkAgentRuntime(
|
|
skill_dirs=skill_dirs(settings, bundle),
|
|
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
|
)
|
|
|
|
catalog = McpCatalog()
|
|
if settings.MCP_ENABLED:
|
|
used = {server_id for role in roles.values() for server_id in role.mcp_servers}
|
|
catalog = await McpCatalog.load(
|
|
[bundle.mcp_servers[server_id] for server_id in sorted(used)],
|
|
auth_from_config(bundle.mcp_auth, settings.MCP_QUOTA_PROJECT),
|
|
timeout=settings.MCP_TIMEOUT_SECONDS,
|
|
)
|
|
|
|
runner = TerraformRunner(
|
|
binary=settings.TERRAFORM_BINARY,
|
|
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
|
plugin_cache_dir=settings.terraform_plugin_cache(),
|
|
providers=bundle.terraform_providers,
|
|
)
|
|
offered = [name for names in bundle.tools.values() for name in names]
|
|
# One cap for Specialists and the pillar advisors they fan out to.
|
|
limiter = RunLimiter(settings.MAX_PARALLEL_SPECIALISTS)
|
|
decider = open_decider(settings)
|
|
tool_factory = make_tool_factory(
|
|
runtime,
|
|
roles,
|
|
bundle.pillar_skills,
|
|
local_tools=lambda working_copy: local_tools(offered, runner, working_copy),
|
|
catalog=catalog,
|
|
limiter=limiter,
|
|
guard=decider
|
|
if settings.JEV_GUARDRAIL and isinstance(decider, JevDecider)
|
|
else None,
|
|
withhold_at=settings.JEV_GUARDRAIL_THRESHOLD,
|
|
)
|
|
|
|
orchestrator: OrchestratorPolicy | None = None
|
|
if settings.ORCHESTRATOR == "router":
|
|
# The Orchestrator routes: Jev reads, the playbook decides, and a model only
|
|
# writes what the user reads. With no Jev key the playbook runs on its rules.
|
|
orchestrator = Router(
|
|
decider,
|
|
RuntimeReplyWriter(runtime, roles[REPLY_WRITER]),
|
|
Thresholds(
|
|
act=settings.JEV_ACT_THRESHOLD,
|
|
close=settings.JEV_CLOSE_THRESHOLD,
|
|
confidence=settings.JEV_CONFIDENCE_THRESHOLD,
|
|
),
|
|
)
|
|
elif settings.ORCHESTRATOR != "model":
|
|
raise ValueError(
|
|
f"ORCHESTRATOR must be router or model, not {settings.ORCHESTRATOR!r}"
|
|
)
|
|
|
|
# What is looked at in a design before it is submitted. Each raises what it finds
|
|
# once and never blocks.
|
|
reviews: list[SubmissionReview] = []
|
|
if bundle.terraform_providers:
|
|
# In code: every setting a Check names exists on its resource type.
|
|
reviews.append(SchemaReview(runner))
|
|
if settings.JEV_REQUIREMENTS_REVIEW and isinstance(decider, JevDecider):
|
|
# Optional, and off by default: Jev says whether a requirement names a product.
|
|
reviews.append(
|
|
RequirementsReview(decider, act=settings.JEV_REQUIREMENTS_THRESHOLD)
|
|
)
|
|
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider):
|
|
# Optional, and off by default: Jev says which Check verifies each condition.
|
|
reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD))
|
|
|
|
async with open_checkpointer(settings) as checkpointer:
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=recorder or DatabaseRecorder(),
|
|
checkpointer=checkpointer,
|
|
output_root=output_root or settings.BASE_DIR,
|
|
max_revisions=settings.MAX_REVISIONS,
|
|
max_orchestrator_turns=settings.MAX_ORCHESTRATOR_TURNS,
|
|
tool_factory=tool_factory,
|
|
limiter=limiter,
|
|
skill_versions={
|
|
name: skill_version(meta.path) for name, meta in bundle.skills.items()
|
|
},
|
|
orchestrator=orchestrator,
|
|
submission_review=Reviews(reviews) if reviews else None,
|
|
confirm_drafts=settings.CONFIRM_DRAFTS,
|
|
decider=decider,
|
|
pillar_threshold=settings.JEV_PILLAR_THRESHOLD,
|
|
reworded_threshold=settings.JEV_REWORDED_THRESHOLD,
|
|
guard_user_text=settings.JEV_GUARDRAIL,
|
|
guard_threshold=settings.JEV_GUARDRAIL_THRESHOLD,
|
|
skill_references=role_references(roles, bundle)
|
|
if settings.JEV_SKILL_ROUTING
|
|
else None,
|
|
skill_threshold=settings.JEV_SKILL_THRESHOLD,
|
|
assess_at_end=settings.JEV_ASSESS_EXECUTIONS,
|
|
overlays={
|
|
name: skill_version(meta.path) for name, meta in overlays.items()
|
|
},
|
|
default_overlays=settings.default_overlays(),
|
|
)
|
|
try:
|
|
yield Assembly(
|
|
engine,
|
|
bundle,
|
|
roles,
|
|
dict(catalog.gaps),
|
|
decider,
|
|
Intake(
|
|
decider,
|
|
output_root or settings.BASE_DIR,
|
|
act=settings.JEV_ACT_THRESHOLD,
|
|
),
|
|
)
|
|
finally:
|
|
if isinstance(decider, JevDecider):
|
|
await decider.aclose()
|