Files
orchestration_engine/execution_engine/agent.py

686 lines
28 KiB
Python

"""The agent: the ExecutionEngine that starts and resumes Executions, and its assembly.
ExecutionEngine is the one interface the A2A server, /generate and the evaluation
harness use. Each call runs the Execution until it needs the user or finishes, and
returns an Interaction describing which.
open_assembly is the composition root: the only place concrete runtimes, tools,
checkpointers and recorders are chosen, from settings and the provider bundle.
"""
from __future__ import annotations
import dataclasses
import hashlib
import uuid
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from langchain_core.runnables import RunnableConfig
from langgraph.checkpoint.base import BaseCheckpointSaver
from langgraph.types import Command
from execution_engine.adk.roles import load_roles
from execution_engine.adk.runtime import (
AgentRuntime,
Decider,
NoDecider,
RoleSpec,
RunLimiter,
SubmissionReview,
ToolFactory,
)
from execution_engine.baseline import load_baseline
from execution_engine.checkpointing import open_checkpointer
from execution_engine.config import Settings
from execution_engine.customer_case import record_intent
from execution_engine.nodes.intake import DeliveredAnswers, Intake
from execution_engine.nodes.router import (
REPLY_WRITER,
OrchestratorPolicy,
Router,
RuntimeReplyWriter,
Thresholds,
)
from execution_engine.providers import ProviderBundle, load_provider
from execution_engine.recorder import DatabaseRecorder, ExecutionRecorder
from execution_engine.scope import (
ESSENTIAL,
Catalogue,
Scope,
merged,
read_scope,
slug,
)
from execution_engine.skills.agent_skills import discover_skills, skill_version
from execution_engine.states.execution_state import check_customer_case
from execution_engine.states.ledger import Ledger
from execution_engine.tools.execution import ORCHESTRATOR
from execution_engine.tools.jev import JevDecider, JevLimits
from execution_engine.tools.mcp import McpCatalog, auth_from_config
from execution_engine.tools.scoping import Scoping, to_working
from execution_engine.tools.skill_routing import routable
from execution_engine.tools.toolkit import make_tool_factory
from execution_engine.tools.workspace import safe_relative
from execution_engine.workflows.execution_graph import build_execution_graph
from execution_engine.workflows.workflow import Assembling, Digested, Workflow
# How much of a source code did not read is given to a Specialist to read.
_DOCUMENT_LIMIT = 60_000
_RECURSION_LIMIT = 10_000 # loops are bounded by max_orchestrator_turns instead
@dataclass(frozen=True)
class Interaction:
execution_id: str
status: str # awaiting_user | completed | completed_with_warnings
prompt: Mapping[str, Any] | None # questions | approval | permission
state: Mapping[str, Any]
class ExecutionNotAwaitingUser(RuntimeError):
pass
class ExecutionEngine:
def __init__(
self,
*,
workflow: Workflow,
runtime: AgentRuntime,
roles: Mapping[str, RoleSpec],
recorder: ExecutionRecorder,
checkpointer: BaseCheckpointSaver,
output_root: Path,
max_revisions: int = 3,
max_orchestrator_turns: int = 100,
tool_factory: ToolFactory | None = None,
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
decider: Decider | None = None,
reworded_threshold: float = 0.25,
overlays: Mapping[str, str] | None = None,
default_overlays: Sequence[str] = (),
guard_user_text: bool = False,
guard_threshold: float = 0.75,
skill_references: Mapping[str, Mapping[str, Mapping[str, str]]] | None = None,
skill_threshold: float = 1.25,
assess_at_end: bool = False,
scope_catalogue: Catalogue | None = None,
scope_values: Mapping[str, Sequence[str]] | None = None,
scope_skill: str | None = None,
) -> None:
required = {
ORCHESTRATOR,
*(spec.specialist for spec in workflow.deliverables.values()),
}
missing = sorted(required - set(roles))
if missing:
raise ValueError(f"Missing roles: {', '.join(missing)}")
self.workflow = workflow
self._max_revisions = max_revisions
# The catalogue a scope spec is read against (ADR-0008); None when this
# deployment's Cloud provider has none.
self._scope_catalogue = scope_catalogue
self._scope_values = dict(scope_values or {})
self._scope_skill = scope_skill
self._output_root = output_root
# Overlay Skills an Execution may name, with the version of each (ADR-0004).
self._overlays = dict(overlays or {})
self._default_overlays = tuple(default_overlays)
unknown = sorted(set(self._default_overlays) - set(self._overlays))
if unknown:
raise ValueError(f"Unknown default overlay Skills: {', '.join(unknown)}")
self._graph = build_execution_graph(
workflow=workflow,
runtime=runtime,
roles=roles,
recorder=recorder,
output_root=output_root,
max_orchestrator_turns=max_orchestrator_turns,
tool_factory=tool_factory,
limiter=limiter,
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
confirm_drafts=confirm_drafts,
decider=decider,
reworded_threshold=reworded_threshold,
guard_user_text=guard_user_text,
guard_threshold=guard_threshold,
skill_references=skill_references,
skill_threshold=skill_threshold,
assess_at_end=assess_at_end,
scoping=Scoping(scope_catalogue, dict(scope_values or {}), scope_skill)
if scope_catalogue is not None
else None,
).compile(checkpointer=checkpointer)
async def start(
self,
workflow_request: str,
*,
session_id: str | None = None,
customer_case: str | None = None,
pre_approved: bool | None = None,
allow_cloud_read: bool | None = None,
execution_id: str | None = None,
scripted_answers: Sequence[Sequence[str]] = (),
baseline: str | None = None,
overlay_skills: Sequence[str] = (),
scope: str | Mapping[str, Any] | None = None,
sources: Mapping[str, str] | None = None,
**options: Any,
) -> Interaction:
"""Start an Execution. `scripted_answers` are (regex, answer[, asker]) entries a
pre-approved Execution uses to answer questions, for evaluation runs.
`baseline` names a completed Execution of the same Customer case and makes
this a Change request (ADR-0006): the request says what has changed, and the
Execution starts from that Baseline's Deliverables. It is refused, before
anything runs or is recorded, when the Baseline cannot be one.
`overlay_skills` names Skills that carry an organisation's standards or
this engagement's constraints. Each must be one the deployment has
configured; an unknown name refuses the start. Every Specialist loads them
with the ones the deployment loads by default.
`sources` are files to digest into a spec, by name: what another agent
delivered, or documents that describe what is wanted. The Workflow reads
the ones it knows in code. Any other is given to the Specialist that
completes the spec, to read. A `scope` given with them is laid over what
was digested, field by field. A source is refused when its name is not a
safe relative path, or when the Workflow knows its kind and cannot read it.
`options` are the choices this agent's Workflow lets a caller make for one
Execution, by name. One left unset is taken from the spec when the spec
states it; then a Change request does as its Baseline did, and any other
Execution takes the Workflow's default. A name the Workflow does not have,
or a value it does not allow, refuses the start.
`scope` is a scope spec (ADR-0008), as YAML text or already parsed. It may
be complete or partly filled. The start is refused when it names a field
the catalogue does not have or holds a value of the wrong type, and, for a
pre-approved Execution, when an essential field is not settled: nobody is
there to ask. What the spec states under `run` applies where the caller
gave no value of their own. Its statement of intent is the request when
none is given.
"""
unknown_options = sorted(set(options) - set(self.workflow.options))
if unknown_options:
raise ValueError(
f"Unknown options: {', '.join(unknown_options)}. Available: "
f"{', '.join(sorted(self.workflow.options)) or 'none'}"
)
digested = self._digest(sources) if sources else None
if digested is not None and digested.spec:
if self._scope_catalogue is None:
raise ValueError("This deployment has no scope spec catalogue.")
read: Scope | None = merged(
digested.spec,
scope,
self._scope_catalogue,
provider_values=self._scope_values,
notes=digested.notes,
)
else:
read = self._read_scope(scope) if scope is not None else None
if read is not None:
run = read.values
if pre_approved is None:
pre_approved = bool(run.get("run.pre_approved", False))
if allow_cloud_read is None and "run.allow_cloud_read" in run:
allow_cloud_read = bool(run["run.allow_cloud_read"])
for name, option in self.workflow.options.items():
if (
options.get(name) is None
and option.spec_field
and option.spec_field in run
):
options[name] = run[option.spec_field]
overlay_skills = [*overlay_skills, *run.get("run.overlay_skills", [])]
if (
customer_case
and read.customer_case
and slug(customer_case) != (read.customer_case)
):
raise ValueError(
f"The scope spec is for {read.customer_case}, and this was "
f"started for {customer_case}."
)
customer_case = customer_case or read.customer_case
# The spec's statement is the customer's; it is what gets recorded.
workflow_request = read.statement or workflow_request
missing = read.gaps(ESSENTIAL)
if pre_approved and missing:
raise ValueError(
"The scope spec leaves essential fields unsettled, and a "
"pre-approved Execution has nobody to ask: "
+ ", ".join(gap["field"] for gap in missing)
+ ". Give each a value, or `decide` to leave it to the agent."
)
pre_approved = bool(pre_approved)
if not workflow_request.strip():
raise ValueError(
"A request is needed: text, or a scope spec with a statement."
)
customer_case = check_customer_case(customer_case)
execution_id = execution_id or str(uuid.uuid4())
if await self._exists(execution_id):
raise ValueError(f"Execution {execution_id} already exists")
if isinstance(overlay_skills, str) or not all(
isinstance(name, str) for name in overlay_skills
):
raise ValueError("overlay_skills must be a list of Skill names")
chosen = list(dict.fromkeys([*self._default_overlays, *overlay_skills]))
unknown = sorted(set(chosen) - set(self._overlays))
if unknown:
raise ValueError(
f"Unknown overlay Skills: {', '.join(unknown)}. Available: "
f"{', '.join(sorted(self._overlays)) or 'none'}"
)
loaded = (
load_baseline(self._output_root, customer_case, baseline)
if baseline
else None
)
chosen_options: dict[str, Any] = {}
for name, option in self.workflow.options.items():
value = options.get(name)
if value is None:
before = (loaded or {}).get("options") or {}
value = before.get(name, option.default)
refused = option.check(value) if option.check else None
if refused:
# A ValueError is what callers turn into a refused start.
raise ValueError(refused)
chosen_options[name] = value
if pre_approved and allow_cloud_read is None:
allow_cloud_read = False
session_id = session_id or str(uuid.uuid4())
intent_version = record_intent(
self._output_root,
customer_case,
execution_id=execution_id,
session_id=session_id,
workflow_request=workflow_request,
baseline=baseline or None,
scope_supplied=scope if isinstance(scope, str) else None,
scope_completed=read.completed(read.status(pre_approved)) if read else None,
sources=sources,
)
initial: dict[str, Any] = {
"execution_id": execution_id,
"session_id": session_id,
"customer_case": customer_case,
"intent_version": intent_version,
"workflow_request": workflow_request,
# The Baseline of a Change request, as read from its files; else None.
"baseline": loaded,
# Overlay Skills this Execution loads, each with the version it had.
"overlay_skills": {name: self._overlays[name] for name in chosen},
"options": chosen_options,
# What the customer's scope spec settles and leaves open; None without one.
"scope": {
"skill": self._scope_skill,
"status": read.status(pre_approved),
"readiness": read.readiness(),
"context": read.for_context(),
# What the scope analyst starts from and completes.
"working": to_working(read),
}
if read
else None,
# What the spec was digested from: each source's hash, and whether code
# read it. The ones code did not read are documents for the Specialist
# that completes the spec.
"sources": {
name: {
"sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(),
"digested": bool(digested and name in digested.used),
}
for name, content in (sources or {}).items()
},
"source_documents": {
name: content[:_DOCUMENT_LIMIT]
for name, content in (sources or {}).items()
if not (digested and name in digested.used)
},
"risks": [],
"pre_approved": pre_approved,
"allow_cloud_read": allow_cloud_read,
"ledger": Ledger(
self.workflow,
pre_approved=pre_approved,
max_revisions=self._max_revisions,
).to_dict(),
"deliverables": {},
"conversation": [{"from": "user", "text": workflow_request}],
"notes": [],
"pending_questions": [],
"awaiting_questions": [],
"answers": {},
"drafts": {},
"drafts_shown": [],
"working": {},
"pending_permission": None,
"prompt": None,
"last_reply": None,
"pending_change": None,
"awaiting_gap_decision": False,
"validation_settled": 0,
"awaiting_limit_decision": False,
"revision_limit_asked": [],
"idle_asks": 0,
"ask_fingerprint": "",
"decisions": [],
"outcomes": [],
"refusals": [],
"outcomes_applied": 0,
"events_recorded": 0,
"turns": 0,
"status": "running",
"warnings": [],
"scripted_answers": [list(entry) for entry in scripted_answers],
}
await self._graph.ainvoke(initial, self._config(execution_id))
return await self.interaction(execution_id)
def _digest(self, sources: Mapping[str, str]) -> Digested:
"""What the Workflow reads of the sources, after checking they are files."""
if not isinstance(sources, Mapping) or not all(
isinstance(name, str) and isinstance(content, str)
for name, content in sources.items()
):
raise ValueError("sources must be files by name: {name: content}")
unsafe = sorted(name for name in sources if not safe_relative(name))
if unsafe:
raise ValueError(
"source names must be relative paths without '..': " + ", ".join(unsafe)
)
if self.workflow.digest is None:
return Digested({})
return self.workflow.digest(sources)
def _read_scope(self, supplied: str | Mapping[str, Any]) -> Scope:
if self._scope_catalogue is None:
raise ValueError("This deployment has no scope spec catalogue.")
return read_scope(
supplied, self._scope_catalogue, provider_values=self._scope_values
)
async def resume(self, execution_id: str, reply: Mapping[str, Any]) -> Interaction:
"""Answer the Execution's current prompt: {"reply"} | {"approval"} | {"permission"}."""
current = await self.interaction(execution_id)
if current.status != "awaiting_user":
raise ExecutionNotAwaitingUser(
f"Execution {execution_id} is {current.status}, not awaiting the user"
)
await self._graph.ainvoke(
Command(resume=dict(reply)), self._config(execution_id)
)
return await self.interaction(execution_id)
async def interaction(self, execution_id: str) -> Interaction:
snapshot = await self._graph.aget_state(self._config(execution_id))
if not snapshot.values:
raise KeyError(f"Unknown Execution {execution_id}")
if snapshot.interrupts:
return Interaction(
execution_id,
"awaiting_user",
snapshot.interrupts[0].value,
snapshot.values,
)
return Interaction(
execution_id,
snapshot.values.get("status", "running"),
None,
snapshot.values,
)
async def _exists(self, execution_id: str) -> bool:
snapshot = await self._graph.aget_state(self._config(execution_id))
return bool(snapshot.values)
@staticmethod
def _config(execution_id: str) -> RunnableConfig:
return {
"configurable": {"thread_id": execution_id},
"recursion_limit": _RECURSION_LIMIT,
}
# --- assembly ----------------------------------------------------------------------
@dataclass(frozen=True)
class Assembly:
engine: ExecutionEngine
bundle: ProviderBundle
roles: dict[str, RoleSpec]
mcp_gaps: dict[str, str]
decider: Decider = field(default_factory=NoDecider)
intake: Intake | None = None # reads a message that starts something
# Answers a question about delivered work from its files.
answers: DeliveredAnswers | None = None
# The runtime the agents run on, for a caller that reports what they used.
runtime: AgentRuntime | None = None
def discover_overlays(
settings: Settings, bundle: ProviderBundle, root: Path | None = None
) -> dict[str, Any]:
"""The overlay Skills this deployment has configured, by name (ADR-0004).
They come only from the folders the deployment names, pass the same validation
as the provider's Skills, and may not reuse one of the provider's names.
"""
folders = [folder for folder in settings.overlay_dirs(root) if folder.is_dir()]
overlays = discover_skills(folders)
clash = sorted(set(overlays) & set(bundle.skills))
if clash:
raise ValueError(
f"Overlay Skills reuse the names of provider Skills: {', '.join(clash)}"
)
return overlays
def skill_dirs(
settings: Settings, bundle: ProviderBundle, root: Path | None = None
) -> dict[str, Path]:
"""Every Skill a runtime may be asked to load: the provider's and the overlays."""
return {
**{name: meta.path for name, meta in bundle.skills.items()},
**{
name: meta.path
for name, meta in discover_overlays(settings, bundle, root).items()
},
}
def role_references(
roles: Mapping[str, RoleSpec], bundle: ProviderBundle
) -> dict[str, dict[str, dict[str, str]]]:
"""For each role, its Skills that hold enough references to choose among, with
what each reference covers (iteration 5 step 8)."""
worth = routable({name: meta.path for name, meta in bundle.skills.items()})
by_role = {
name: {skill: worth[skill] for skill in role.skills if skill in worth}
for name, role in roles.items()
}
return {name: skills for name, skills in by_role.items() if skills}
def open_decider(settings: Settings) -> Decider:
"""The Jev client when a key is configured; otherwise a decider that always
sends its caller to the fallback."""
if not settings.TYPESAFE_API_KEY:
return NoDecider()
return JevDecider(
api_key=settings.TYPESAFE_API_KEY,
base_url=settings.TYPESAFE_BASE_URL,
model=settings.TYPESAFE_DEFAULT_MODEL,
limits=dataclasses.replace(
JevLimits(),
request_timeout=settings.JEV_TIMEOUT_SECONDS,
requests_per_second=settings.JEV_REQUESTS_PER_SECOND,
max_in_flight=settings.JEV_MAX_IN_FLIGHT,
budget_calls=settings.JEV_BUDGET_CALLS,
budget_tokens=settings.JEV_BUDGET_TOKENS,
),
)
@asynccontextmanager
async def open_assembly(
settings: Settings,
workflow: Workflow | Callable[[Settings, ProviderBundle], Workflow],
*,
root: Path,
runtime: AgentRuntime | None = None,
recorder: ExecutionRecorder | None = None,
output_root: Path | None = None,
) -> AsyncIterator[Assembly]:
"""Assemble an agent: the engine, and the Workflow that makes it about a subject.
`root` is the agent's own folder: its provider bundles are under `providers/`,
its role files under `agents/`, and its Skills where the bundle says. `workflow`
is the Workflow, or a function that builds it for the settings and the bundle
when parts of it depend on them.
"""
bundle = load_provider(settings.CLOUD_PROVIDER, root)
if not isinstance(workflow, Workflow):
workflow = workflow(settings, bundle)
roles = load_roles(
root / "agents",
bundle,
workflow,
orchestrator_budget=settings.ORCHESTRATOR_BUDGET,
specialist_budget=settings.SPECIALIST_BUDGET,
include_preview_mcp=settings.MCP_ENABLE_PREVIEW,
lessons=settings.LESSONS_ENABLED,
model_tiers=settings.MODEL_TIERS,
)
overlays = discover_overlays(settings, bundle, root)
if runtime is None:
from execution_engine.adk.adk_runtime import AdkAgentRuntime
runtime = AdkAgentRuntime(
skill_dirs=skill_dirs(settings, bundle, root),
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
catalog = McpCatalog()
if settings.MCP_ENABLED:
used = {server_id for role in roles.values() for server_id in role.mcp_servers}
catalog = await McpCatalog.load(
[bundle.mcp_servers[server_id] for server_id in sorted(used)],
auth_from_config(bundle.mcp_auth, settings.MCP_QUOTA_PROJECT),
timeout=settings.MCP_TIMEOUT_SECONDS,
)
# One cap for Specialists and the agents they fan out to.
limiter = RunLimiter(settings.MAX_PARALLEL_SPECIALISTS)
decider = open_decider(settings)
assembling = Assembling(settings, bundle, runtime, roles, decider)
tool_factory = make_tool_factory(
workflow,
runtime,
roles,
local_tools=workflow.local_tools(assembling) if workflow.local_tools else (),
catalog=catalog,
limiter=limiter,
guard=decider
if settings.JEV_GUARDRAIL and isinstance(decider, JevDecider)
else None,
withhold_at=settings.JEV_GUARDRAIL_THRESHOLD,
)
orchestrator: OrchestratorPolicy | None = None
if settings.ORCHESTRATOR == "router":
# The Orchestrator routes: Jev reads, the playbook decides, and a model only
# writes what the user reads. With no Jev key the playbook runs on its rules.
orchestrator = Router(
decider,
RuntimeReplyWriter(runtime, roles[REPLY_WRITER]),
Thresholds(
act=settings.JEV_ACT_THRESHOLD,
close=settings.JEV_CLOSE_THRESHOLD,
confidence=settings.JEV_CONFIDENCE_THRESHOLD,
),
)
elif settings.ORCHESTRATOR != "model":
raise ValueError(
f"ORCHESTRATOR must be router or model, not {settings.ORCHESTRATOR!r}"
)
# What is looked at in a submission before it is accepted. Each review raises
# what it finds once and never blocks.
submission_review = workflow.reviews(assembling) if workflow.reviews else None
async with open_checkpointer(settings) as checkpointer:
engine = ExecutionEngine(
workflow=workflow,
runtime=runtime,
roles=roles,
recorder=recorder or DatabaseRecorder(),
checkpointer=checkpointer,
output_root=output_root or root,
max_revisions=settings.MAX_REVISIONS,
max_orchestrator_turns=settings.MAX_ORCHESTRATOR_TURNS,
tool_factory=tool_factory,
limiter=limiter,
skill_versions={
name: skill_version(meta.path) for name, meta in bundle.skills.items()
},
orchestrator=orchestrator,
submission_review=submission_review,
confirm_drafts=settings.CONFIRM_DRAFTS,
decider=decider,
reworded_threshold=settings.JEV_REWORDED_THRESHOLD,
guard_user_text=settings.JEV_GUARDRAIL,
guard_threshold=settings.JEV_GUARDRAIL_THRESHOLD,
skill_references=role_references(roles, bundle)
if settings.JEV_SKILL_ROUTING
else None,
skill_threshold=settings.JEV_SKILL_THRESHOLD,
assess_at_end=settings.JEV_ASSESS_EXECUTIONS,
scope_catalogue=bundle.scope_catalogue() if bundle.scope_skill else None,
scope_values=bundle.scope_values,
scope_skill=bundle.scope_skill,
overlays={
name: skill_version(meta.path) for name, meta in overlays.items()
},
default_overlays=settings.default_overlays(),
)
try:
yield Assembly(
engine,
bundle,
roles,
dict(catalog.gaps),
decider,
Intake(
decider,
output_root or root,
act=settings.JEV_ACT_THRESHOLD,
),
DeliveredAnswers(
runtime,
roles[REPLY_WRITER],
output_root or root,
workflow.final,
),
runtime,
)
finally:
if isinstance(decider, JevDecider):
await decider.aclose()