Files
orchestration_engine/execution_engine/workflows/workflow.py

536 lines
23 KiB
Python

"""The Workflow: everything about a subject, as one value (ADR-0001).
An agent is the Execution engine and one Workflow. The engine names no Deliverable,
no Phase and no subject; it reads them from the Workflow it was assembled with.
"""
from __future__ import annotations
import dataclasses
import json
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass, field
from typing import Any
from execution_engine.adk.runtime import (
AgentRuntime,
Decider,
RoleSpec,
RunLimiter,
SubmissionReview,
ToolSpec,
)
from execution_engine.workflows.deliverables import (
BaselineCheck,
DataCheck,
DeliverableSpec,
Upstream,
deliverables_in,
dependents_of,
inputs_of,
missing_from,
)
class WorkflowError(ValueError):
"""A Workflow that does not hold together; the message says where."""
def validation_outcome(verdicts_text: str) -> dict[str, Any]:
"""What a validation decided, from its data file.
The file gives a verdict on each Check, an overall run of whatever the Workflow
runs (`dry_run`), and anything the validator proposes adding. A failed Check is
a fault in what was validated. A Check that could not be verified and a proposed
addition are gaps in the design it was validated against.
"""
try:
data = json.loads(verdicts_text)
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
checks = [item for item in data.get("checks", []) if isinstance(item, dict)]
def with_verdict(verdict: str) -> list[dict[str, Any]]:
return [
{"check": item.get("check"), "evidence": item.get("evidence")}
for item in checks
if item.get("verdict") == verdict
]
dry_run = data.get("dry_run") if isinstance(data.get("dry_run"), dict) else {}
return {
"dry_run": dry_run,
"passed": [
item.get("check") for item in checks if item.get("verdict") == "pass"
],
"failed": with_verdict("fail"),
"not_verifiable": with_verdict("not_verifiable"),
"proposed_additions": [
item
for item in data.get("proposed_additions", [])
if isinstance(item, dict)
],
}
@dataclass(frozen=True)
class ValidationPolicy:
"""Which Deliverable is validation, and what its result reopens."""
kind: str # the validation Deliverable
data_file: str # its verdicts, read by `read`
implementation: str # reopened when a Check fails
design: str # reopened when a Check fails twice running, or a gap matters
read: Callable[[str], dict[str, Any]] = validation_outcome
# The ids of the Checks a validation must give a verdict on, from what it
# builds on. Without it the validator is not told how many are left.
check_ids: Callable[[Upstream | None], Sequence[str]] | None = None
# Checks the implementation's author reported cannot be met as the design
# words them, from the Deliverables: [{check, reason}]. They reopen the design.
unimplementable: Callable[[Upstream], list[dict[str, str]]] | None = None
# The ids a proposed addition may name as what it serves. An addition that
# names one is a gap worth a Revision when nothing judges it.
known_ids: Callable[[Upstream], set[str]] | None = None
@dataclass(frozen=True)
class Digested:
"""A spec read by code from the sources a caller supplied."""
spec: Mapping[str, Any] # the spec's fields, nested as a spec file is
used: tuple[str, ...] = () # the sources it was read from, by name
# What is said of a field in the completed spec: where it was read from.
notes: Mapping[str, str] = field(default_factory=dict)
@dataclass(frozen=True)
class CardText:
"""How an agent describes itself on its A2A card."""
description: str
skill_id: str
skill_name: str
tags: tuple[str, ...] = ()
examples: tuple[str, ...] = ()
@dataclass(frozen=True)
class OptionSpec:
"""A choice a caller may make for one Execution, beyond what every Execution has.
Left unset, a Change request does as its Baseline did, and any other Execution
takes `default`.
"""
default: Any = None
# The field of the spec that states it, such as "run.terraform_modules".
spec_field: str | None = None
# Why a value is refused, or None when it is allowed.
check: Callable[[Any], str | None] | None = None
@dataclass(frozen=True)
class Assembling:
"""What an agent has been assembled from, for a Workflow to build its own parts."""
settings: Any
bundle: Any # the provider bundle
runtime: AgentRuntime
roles: Mapping[str, RoleSpec]
decider: Decider
# The local tools, built on a working copy of files when the Deliverable has one.
LocalToolBuilder = Callable[[MutableMapping[str, str] | None], Sequence[ToolSpec]]
@dataclass(frozen=True)
class ChangePolicy:
"""What a Change request changed against its Baseline, worked out from the two
versions' data files and never written by an agent."""
file: str # where the account is written, among the Execution's files
# (the Baseline's Deliverables, this Execution's) -> what was added, changed
# and removed.
summary: Callable[[Upstream | None, Upstream], dict[str, Any]]
is_empty: Callable[[Mapping[str, Any]], bool]
# The summary's changes to what the customer asked for, in words: what a
# Specialist started on a Change request is told its task is.
in_words: Callable[[Mapping[str, Any]], str]
# Tells a rewording from a change where a decision is available:
# (decider, summary, before, after, unchanged_at, scope) -> the summary.
settle: Callable[..., Awaitable[dict[str, Any]]] | None = None
@dataclass(frozen=True)
class Hints:
"""What a Workflow is given to judge what the Specialists being started need."""
state: Mapping[str, Any] # the Execution's state
feedback: Mapping[str, Sequence[str]] # kind -> why it was revised
deliverables: Upstream
started: Mapping[str, int] # kind -> attempt
decider: Decider | None
changed: str # in a Change request, what changed, in words
@dataclass(frozen=True)
class Bench:
"""What a Workflow's own code is given when a Deliverable is submitted."""
kind: str
working: MutableMapping[str, Any] # the Deliverable's working memory
upstream: Upstream | None # the Deliverables it builds on
baseline: Mapping[str, Any] | None # its Baseline version, in a Change request
# Adds to an accepted submission what code writes from it: a view of its data that
# must not be able to disagree with it. Returns the content and the attachments.
SubmitHook = Callable[[Bench, str, dict[str, str]], tuple[str, dict[str, str]]]
# What an Approval shows beyond the Phase's summary: (phase, the Deliverables
# submitted so far, the Baseline in a Change request) -> entries for the prompt.
ApprovalDetails = Callable[
[str, Mapping[str, Mapping[str, Any]], Mapping[str, Any] | None], dict[str, Any]
]
@dataclass(frozen=True)
class RoleTools:
"""What a Workflow needs to give a role tools of its own, such as consulting
further agents on one Deliverable."""
runtime: AgentRuntime
roles: Mapping[str, RoleSpec]
role: RoleSpec
deliverable: str | None
working: MutableMapping[str, Any]
limiter: RunLimiter | None
# A role's tools from the remote MCP servers its bundle assigns it, checked for
# instructions addressed to the agent.
mcp_tools: Callable[[RoleSpec], list[ToolSpec]]
@dataclass(frozen=True)
class Workflow:
name: str
# What a caller is told this agent does, when a message starts nothing.
about: str
phases: tuple[str, ...]
deliverables: Mapping[str, DeliverableSpec]
# The Deliverable handed over as the result. It is assembled from every other
# Deliverable, so it is shown the sections written for readers.
final: str
# What each data file along the chain must hold, checked at submission:
# kind -> (the data file, its check).
data_checks: Mapping[str, tuple[str, DataCheck]] = field(default_factory=dict)
# In a Change request, the account of what changed: kind -> (file, check).
baseline_checks: Mapping[str, tuple[str, BaselineCheck]] = field(
default_factory=dict
)
# Sections code appends to a Deliverable for the person reading it. A Specialist
# working from that Deliverable has the same facts as data, and is not shown them.
for_readers: Mapping[str, str] = field(default_factory=dict)
# The Deliverable that holds the agreed spec. It is in every Specialist's
# context as data, so it is not shown again as a dependency.
spec_deliverable: str | None = None
validation: ValidationPolicy | None = None
# What code adds to an accepted submission, by kind.
on_submit: Mapping[str, SubmitHook] = field(default_factory=dict)
approval_details: ApprovalDetails | None = None
# Facts of this Workflow's own that an Approval summary is written from:
# (phase, the Deliverables) -> facts. Data, never Deliverable prose.
approval_facts: Callable[[str, Upstream], dict[str, Any]] | None = None
# How those facts read when no writer is available: (phase, facts) -> sentences.
summary_lines: Callable[[str, Mapping[str, Any]], Sequence[str]] | None = None
# What is agreed so far, as text: shown to the decider beside a user's message,
# so a message that restates it is not read as a change.
agreed: Callable[[Upstream], str] | None = None
changes: ChangePolicy | None = None
# What the work is, to finish the sentence "the x Deliverable of ...".
subject: str = "this work"
# The design so far in a few lines, for judging which of a Skill's references
# a task needs. "" when there is none yet.
describe_design: Callable[[Upstream], str] | None = None
# A role as one Execution runs it: (role, the Specialist's task input) -> role.
prepare_role: Callable[[RoleSpec, Mapping[str, Any]], RoleSpec] | None = None
# Entries of this Workflow's own in a Specialist's task: (state, kind) -> the
# task input's entries and its context's.
task_entries: (
Callable[[Mapping[str, Any], str], tuple[dict[str, Any], dict[str, Any]]] | None
) = None
# Context for the Specialists being started that needs a judgement:
# kind -> entries for its context.
task_hints: Callable[[Hints], Awaitable[Mapping[str, Mapping[str, Any]]]] | None = (
None
)
# Working memory kept with a completed Execution, for a Change request to start
# from: the keys kept, of each Deliverable's working memory.
kept_for_changes: tuple[str, ...] = ()
# The file the final Deliverable is also written as, at the top of an
# Execution's folder.
final_file: str = "result.md"
# What this Workflow adds when an Execution's files are written. It is given
# the state, the Deliverables and the files so far, which it may add to or
# amend, and returns its entries for the manifest.
outputs: (
Callable[[Mapping[str, Any], Upstream, dict[str, str]], dict[str, Any]] | None
) = None
# Reads a spec from sources of a kind this Workflow knows, such as another
# agent's files: {name: content} -> the spec. It raises ValueError, naming what
# is wrong, when the sources are of that kind and cannot be read. A source it
# does not use is given to the Specialist that completes the spec, to read.
digest: Callable[[Mapping[str, str]], Digested] | None = None
card: CardText | None = None
# What a caller is told when an Execution completes.
done: str = "The work is complete and approved."
# What an Approval's own details say, in the text a user reads: the entries
# `approval_details` added -> a paragraph, ending in a blank line, or "".
describe_approval: Callable[[Mapping[str, Any]], str] | None = None
# The reply to a one-shot request, when it is not the engine's:
# (the Interaction, the roles) -> the response body.
response: Callable[[Any, Mapping[str, RoleSpec]], dict[str, Any]] | None = None
# Choices a caller may make for one Execution, by name.
options: Mapping[str, OptionSpec] = field(default_factory=dict)
# The tools that run on the agent's own machine, for the roles whose files name
# them; and what looks at a submission's meaning before it is accepted.
local_tools: Callable[[Assembling], LocalToolBuilder] | None = None
reviews: Callable[[Assembling], SubmissionReview | None] | None = None
# A judgement of the finished work, kept in the Execution's record:
# (decider, deliverables, scope) -> the record's entry.
assess: Callable[..., Awaitable[dict[str, Any] | None]] | None = None
# Tools a role is given beyond its role file's and its bundle's.
role_tools: Callable[[RoleTools], Sequence[ToolSpec]] | None = None
# Roles this Workflow needs beyond the engine's own and one per Deliverable,
# and roles it can use when their file is present.
roles: tuple[str, ...] = ()
optional_roles: tuple[str, ...] = ()
# A last change to the loaded roles from the provider bundle, such as giving
# a Skill to every role that has a certain tool: (roles, bundle) -> roles.
adjust_roles: (
Callable[[Mapping[str, RoleSpec], Any], Mapping[str, RoleSpec]] | None
) = None
# A passing attachment by file name, for the scripted runtime's sample
# submissions. A file with no sample is given a placeholder.
samples: Mapping[str, str] = field(default_factory=dict)
def __post_init__(self) -> None:
self._plain_names()
problems = []
if len(set(self.phases)) != len(self.phases):
problems.append("a Phase is listed twice")
specialists: dict[str, str] = {}
for kind, spec in self.deliverables.items():
if spec.kind != kind:
problems.append(f"{kind}: its spec is for {spec.kind}")
if spec.phase not in self.phases:
problems.append(f"{kind}: Phase {spec.phase} is not listed")
named = (
*spec.depends_on,
*(spec.reads_in_full or ()),
*spec.not_shown,
*((spec.working_copy,) if spec.working_copy else ()),
)
for other in named:
if other not in self.deliverables:
problems.append(f"{kind}: names unknown Deliverable {other}")
if spec.specialist in specialists:
problems.append(
f"{kind}: {spec.specialist} already produces "
f"{specialists[spec.specialist]}"
)
specialists[spec.specialist] = kind
for phase in self.phases:
if not deliverables_in(self.deliverables, phase):
problems.append(f"Phase {phase} has no Deliverable")
for kind, spec in self.deliverables.items():
for dependency in spec.depends_on:
other = self.deliverables.get(dependency)
if other and self.phases.index(other.phase) > self.phases.index(
spec.phase
):
problems.append(
f"{kind}: depends on {dependency}, in a later Phase"
)
if self.final not in self.deliverables:
problems.append(f"final Deliverable {self.final} is not declared")
named_kinds = [
*self.data_checks,
*self.baseline_checks,
*self.for_readers,
*self.on_submit,
]
if self.spec_deliverable:
named_kinds.append(self.spec_deliverable)
if self.validation:
policy = self.validation
named_kinds += [policy.kind, policy.implementation, policy.design]
for kind in named_kinds:
if kind not in self.deliverables:
problems.append(f"names unknown Deliverable {kind}")
if problems:
raise WorkflowError(f"Workflow {self.name}: " + "; ".join(problems))
def _plain_names(self) -> None:
"""Hold every kind and Phase as a plain string.
A Workflow may declare them as members of a StrEnum. They go into an
Execution's checkpointed state, which is plain data, so the engine keeps
the strings and never the members.
"""
def names(values: tuple[Any, ...] | None) -> tuple[str, ...] | None:
return None if values is None else tuple(str(value) for value in values)
def keys(mapping: Mapping[Any, Any]) -> dict[str, Any]:
return {str(key): value for key, value in mapping.items()}
graph = {
str(kind): dataclasses.replace(
spec,
kind=str(spec.kind),
phase=str(spec.phase),
depends_on=names(spec.depends_on) or (),
reads_in_full=names(spec.reads_in_full),
not_shown=names(spec.not_shown) or (),
working_copy=None
if spec.working_copy is None
else str(spec.working_copy),
)
for kind, spec in self.deliverables.items()
}
plain: dict[str, Any] = {
"phases": names(self.phases),
"deliverables": graph,
"final": str(self.final),
"data_checks": keys(self.data_checks),
"baseline_checks": keys(self.baseline_checks),
"for_readers": keys(self.for_readers),
"on_submit": keys(self.on_submit),
"spec_deliverable": None
if self.spec_deliverable is None
else str(self.spec_deliverable),
}
if self.validation:
policy = self.validation
plain["validation"] = dataclasses.replace(
policy,
kind=str(policy.kind),
implementation=str(policy.implementation),
design=str(policy.design),
)
for name, value in plain.items():
object.__setattr__(self, name, value)
# --- names -------------------------------------------------------------------
def kind(self, value: object) -> str:
"""`value` as one of this Workflow's Deliverable kinds; ValueError otherwise."""
if not isinstance(value, str) or value not in self.deliverables:
raise ValueError(f"{value!r} is not a Deliverable of {self.name}")
return str(value)
def phase(self, value: object) -> str:
"""`value` as one of this Workflow's Phases; ValueError otherwise."""
if not isinstance(value, str) or value not in self.phases:
raise ValueError(f"{value!r} is not a Phase of {self.name}")
return str(value)
def specialists(self) -> dict[str, str]:
"""The role that produces each Deliverable: {role: kind}."""
return {spec.specialist: kind for kind, spec in self.deliverables.items()}
# --- the graph ---------------------------------------------------------------
def deliverables_in(self, phase: str) -> tuple[str, ...]:
return deliverables_in(self.deliverables, phase)
def inputs_of(self, kind: str) -> tuple[str, ...]:
return inputs_of(self.deliverables, self.phases, kind)
def dependents_of(self, kind: str) -> tuple[str, ...]:
return dependents_of(self.deliverables, kind)
def dependencies_for(
self, kind: str, deliverables: Mapping[str, Mapping[str, object]]
) -> dict[str, dict[str, object]]:
"""What a Specialist is given of each Deliverable it works from.
A Deliverable read in full arrives as submitted. Any other arrives as its data
file alone, which is all the chain needs of it.
"""
spec = self.deliverables[kind]
in_full = spec.depends_on if spec.reads_in_full is None else spec.reads_in_full
given: dict[str, dict[str, object]] = {}
for dependency in self.inputs_of(kind):
deliverable = deliverables[dependency]
if dependency in in_full or dependency not in self.data_checks:
given[dependency] = dict(deliverable)
continue
name = self.data_checks[dependency][0]
attachments = deliverable.get("attachments")
files = attachments if isinstance(attachments, Mapping) else {}
given[dependency] = {
"attachments": {name: files[name]} if name in files else {}
}
return given
def shown(
self, kind: str, given: Mapping[str, Mapping[str, object]]
) -> dict[str, dict[str, object]]:
"""What of `given` a Specialist is shown. Nothing it uses is left out.
- A Deliverable its spec lists under `not_shown` is omitted.
- The agreed spec is in every Specialist's context as data, so it is not
shown again as a dependency; a Specialist that reads it whole sees its prose.
- A section written for a reader is left off, except for the final
Deliverable, which assembles the document those sections belong in.
"""
spec = self.deliverables[kind]
view: dict[str, dict[str, object]] = {}
for name, deliverable in given.items():
if name in spec.not_shown:
continue
entry = dict(deliverable)
if name == self.spec_deliverable:
if name not in (*spec.depends_on, *(spec.reads_in_full or ())):
continue
entry = {"content": entry.get("content", "")}
heading = self.for_readers.get(name)
content = entry.get("content")
if heading and isinstance(content, str) and kind != self.final:
entry["content"] = content.split(f"\n## {heading}")[0].rstrip() + "\n"
view[name] = entry
return view
def check_submission(
self,
kind: str,
content: str,
attachments: Mapping[str, str],
upstream: Upstream | None = None,
baseline: Mapping[str, object] | None = None,
) -> tuple[str, ...]:
"""Return what a submission lacks against its contract; empty means it's complete.
`upstream` is the Deliverables this one builds on, by kind. With it, the links
along the chain are checked too. `baseline` is the Baseline's version of this
Deliverable in a Change request; with it, the account of what changed is
checked.
"""
if not content.strip():
return ("content: must not be empty",)
missing = missing_from(self.deliverables[kind], content, attachments)
if kind in self.data_checks:
name, check = self.data_checks[kind]
if attachments.get(name, "").strip():
missing.extend(check(attachments, upstream))
if baseline is not None and kind in self.baseline_checks and not missing:
name, compare = self.baseline_checks[kind]
if attachments.get(name, "").strip():
missing.extend(compare(attachments, baseline))
return tuple(missing)