fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

779
app/nodes/router.py Normal file
View File

@@ -0,0 +1,779 @@
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
1. Readings facts about the turn. Most come from the Ledger; the ones that need
judgement (what is the user asking for? does this gap matter?) are
typed answers from the decision model.
2. Disposition code turns a judged reading into act, queue or close, by threshold.
3. Playbook ordered groups. The first group that applies is taken, and within
it the strongest action whose conditions hold. When no group
applies, escalate to the user.
The decision model reads and never acts: every action goes through the Orchestrator's
tools, so the Ledger refuses anything that is not allowed, whatever was decided.
"""
from __future__ import annotations
import logging
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any, Protocol
from app.adk.runtime import (
AgentRuntime,
AgentTask,
Decider,
Outcome,
Question,
RoleSpec,
Stopped,
Submitted,
ToolResult,
ToolSpec,
)
from app.states.ledger import DeliverableStatus, Ledger, PhaseStatus
from app.workflows.chain import (
IMPLEMENTATION_FILE,
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
VERDICTS_FILE,
read_graph,
read_list,
unimplementable_checks,
validation_outcome,
)
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
logger = logging.getLogger(__name__)
ORCHESTRATOR = "orchestrator"
REPLY_WRITER = "reply-writer"
MAX_STEPS = 12 # actions in one turn; a turn that needs more is not making progress
ACT, QUEUE, CLOSE = "act", "queue", "close"
@dataclass(frozen=True)
class Thresholds:
"""Where a probability becomes a disposition."""
act: float = 0.75 # at or above: do it
close: float = 0.15 # at or below: there is nothing to do
confidence: float = 0.6 # a choice below this is not relied on
def disposition(self, probability: float) -> str:
if probability >= self.act:
return ACT
if probability <= self.close:
return CLOSE
return QUEUE
@dataclass(frozen=True)
class TurnView:
"""What one Orchestrator turn can see."""
execution_id: str
pre_approved: bool
ledger: Ledger
deliverables: Mapping[str, Mapping[str, Any]]
pending_questions: Sequence[Mapping[str, Any]]
outcomes: Sequence[Mapping[str, Any]]
last_reply: Mapping[str, Any] | None = None # the user's message, not yet routed
pending_change: Mapping[str, Any] | None = None # a change we asked them to confirm
awaiting_gap_decision: bool = False
validation_settled: int = 0 # the validation attempt already decided on
context: Mapping[str, Any] = field(default_factory=dict) # for a model Orchestrator
@dataclass
class TurnResult:
outcome: Outcome
decisions: list[dict[str, Any]] = field(default_factory=list)
updates: dict[str, Any] = field(default_factory=dict)
class OrchestratorPolicy(Protocol):
async def turn(self, view: TurnView, tools: Sequence[ToolSpec]) -> TurnResult: ...
class ReplyWriter(Protocol):
async def approval_summary(
self, phase: str, facts: Mapping[str, Any]
) -> str | None: ...
# --- readings ------------------------------------------------------------------------
@dataclass
class Readings:
"""Everything the playbook decides from."""
# From the Ledger and the Execution's data.
questions_waiting: bool = False
phases_ready: tuple[PhaseId, ...] = ()
phases_awaiting: tuple[PhaseId, ...] = () # asked for Approval, not yet decided
ready_work: tuple[DeliverableKind, ...] = ()
can_finish: bool = False
validation_attempt: int = 0 # 0 when no validation stands undecided
failed: list[dict[str, Any]] = field(default_factory=list)
repeat_failed: list[dict[str, Any]] = field(default_factory=list)
unverifiable: list[dict[str, Any]] = field(default_factory=list)
additions: list[dict[str, Any]] = field(default_factory=list)
dry_run_failed: bool = False
# Checks the code's author reported cannot be met as the Target architecture
# words them.
unimplementable: list[dict[str, str]] = field(default_factory=list)
# Judged: what the user's message asks for.
change: str = CLOSE # act | queue | close
change_target: DeliverableKind | None = None
change_reason: str = ""
# Judged: whether a gap in the Target architecture is worth a Revision.
gap: str = CLOSE
judged: dict[str, Any] = field(default_factory=dict) # what the decider answered
def _attempt(ledger: Ledger, kind: DeliverableKind) -> int:
return int(ledger.to_dict()["deliverables"][kind.value].get("attempt", 0))
def _submitted(ledger: Ledger) -> list[DeliverableKind]:
return [
kind
for kind in DEPENDENCY_GRAPH
if ledger.status(kind) is DeliverableStatus.SUBMITTED
]
def _mentioned(
text: str, candidates: Sequence[DeliverableKind]
) -> list[DeliverableKind]:
"""Deliverables named outright in a message, as `deployment_guidance` or in words."""
lowered = re.sub(r"[\s_\-]+", " ", text.lower())
return [kind for kind in candidates if kind.value.replace("_", " ") in lowered]
def _failed_before(outcomes: Sequence[Mapping[str, Any]]) -> set[str]:
"""Check ids that failed in the validation before the one that stands now."""
validations = [
record
for record in outcomes
if record.get("type") == "submitted"
and record.get("kind") == DeliverableKind.VALIDATION_RESULTS.value
]
if len(validations) < 2:
return set()
previous = validation_outcome(
validations[-2].get("attachments", {}).get(VERDICTS_FILE, "")
)
return {item["check"] for item in previous["failed"]}
class Router:
"""Decides an Orchestrator turn from readings and a playbook, with no language
model choosing the action. A model, when one is given, only writes what the user
reads."""
def __init__(
self,
decider: Decider,
writer: ReplyWriter | None = None,
thresholds: Thresholds | None = None,
) -> None:
self._decider = decider
self._writer = writer
self._thresholds = thresholds or Thresholds()
async def turn(self, view: TurnView, tools: Sequence[ToolSpec]) -> TurnResult:
by_name = {tool.name: tool for tool in tools}
result = TurnResult(Stopped("The router took no action."))
# A user message is routed once; a confirmation we asked for is consumed.
result.updates = {
"last_reply": None,
"pending_change": None,
"awaiting_gap_decision": False,
}
readings = await self._read(view, result)
blocked: set[str] = set()
settled = view.validation_settled
async def call(tool: str, **arguments: Any) -> ToolResult:
return await by_name[tool].handler(**arguments)
for _ in range(MAX_STEPS):
group, action = self._choose(view, readings, blocked, settled)
result.decisions.append(
{"group": group, "action": action, "judged": dict(readings.judged)}
)
readings.judged = {}
reply = await self._perform(action, view, readings, call, result)
if action == "proceed":
settled = readings.validation_attempt
result.updates["validation_settled"] = settled
self._read_ledger(view, readings, settled)
continue
if reply is None:
return result
if reply.outcome is not None:
result.outcome = reply.outcome
return result
if "revision limit" in reply.message and not view.pre_approved:
# Past the Revision limit the user decides.
limit = await call(
"ask_user", questions=[reply.message.split(" Use ")[0]]
)
if limit.outcome is not None:
result.outcome = limit.outcome
return result
if reply.message.startswith("Refused") or "revision limit" in reply.message:
# Permission is the Ledger's: take the next action that is allowed.
blocked.add(action)
# The action changed the Execution; read the Ledger's facts again.
self._read_ledger(view, readings, settled)
if action in {"revise_requested", "confirm_change"}:
readings.change = CLOSE
result.outcome = Stopped("The router made no progress in this turn.")
return result
# --- readings ---------------------------------------------------------------------
async def _read(self, view: TurnView, result: TurnResult) -> Readings:
readings = Readings()
self._read_ledger(view, readings, view.validation_settled)
await self._read_message(view, readings)
await self._read_gap(view, readings)
return readings
def _read_ledger(self, view: TurnView, readings: Readings, settled: int) -> None:
ledger = view.ledger
readings.questions_waiting = bool(view.pending_questions)
readings.phases_ready = ledger.phases_ready_for_approval()
readings.phases_awaiting = tuple(
phase
for phase in PhaseId
if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL
)
readings.ready_work = ledger.ready_work()
readings.can_finish = ledger.can_finish()
readings.validation_attempt = 0
readings.failed, readings.repeat_failed = [], []
readings.unverifiable, readings.additions = [], []
readings.dry_run_failed = False
code = DeliverableKind.DEPLOYMENT_GUIDANCE
readings.unimplementable = (
unimplementable_checks(
view.deliverables.get(code.value, {})
.get("attachments", {})
.get(IMPLEMENTATION_FILE, "")
)
if ledger.status(code) is DeliverableStatus.SUBMITTED
else []
)
validation = DeliverableKind.VALIDATION_RESULTS
phase = DEPENDENCY_GRAPH[validation].phase
attempt = _attempt(ledger, validation)
if (
ledger.status(validation) is not DeliverableStatus.SUBMITTED
or ledger.phase_status(phase) is PhaseStatus.APPROVED
or attempt == settled
):
return
outcome = validation_outcome(
view.deliverables.get(validation.value, {})
.get("attachments", {})
.get(VERDICTS_FILE, "")
)
before = _failed_before(view.outcomes)
readings.validation_attempt = attempt
readings.failed = outcome["failed"]
readings.repeat_failed = [f for f in outcome["failed"] if f["check"] in before]
readings.unverifiable = outcome["not_verifiable"]
readings.additions = outcome["proposed_additions"]
readings.dry_run_failed = outcome["dry_run"].get("verdict") == "fail"
async def _read_message(self, view: TurnView, readings: Readings) -> None:
"""Triage of the user's message: is it asking for a change, to what, and how
clearly?"""
reply = view.last_reply
if not reply or view.awaiting_gap_decision:
return
text = str(reply.get("text", "")).strip()
pending = view.pending_change
if pending:
text = f"{pending.get('text', '')}\nThe user then said: {text}"
candidates = _submitted(view.ledger)
if not text or not candidates:
return
readings.change_reason = text
# A structured request for changes is known to be one, and so is the reply to
# our own question about a change they already asked for.
structured = reply.get("kind") == "changes" or bool(pending)
questions = {
"change_target": Question(
"choice",
"Which submitted Deliverable would have to change first to do what "
"the user asks?",
tuple(kind.value for kind in candidates),
),
"change_clarity": Question(
"score",
"How specific is the user about what should change?",
("vague", "partial", "specific"),
),
}
if not structured:
questions["asks_change"] = Question(
"noul",
"Is the user asking for a change to work that has already been "
"submitted, as opposed to answering a question or adding information?",
)
state = (
f"Submitted Deliverables: {', '.join(kind.value for kind in candidates)}.\n"
f"Questions waiting for the user: {len(view.pending_questions)}.\n"
f"The user's message: {text}"
)
answers = await self._decider.decide(state, questions, scope=view.execution_id)
named = _mentioned(text, candidates)
if answers is None:
# No judgement available: act only on what is certain.
if structured:
target = (
candidates[0]
if len(candidates) == 1
else named[0]
if len(named) == 1
else None
)
readings.change = ACT if target else QUEUE
readings.change_target = target
return
readings.judged = {
name: {"value": answer.value, "confidence": answer.confidence}
for name, answer in answers.items()
}
probability = 1.0 if structured else float(answers["asks_change"].value)
disposition = self._thresholds.disposition(probability)
target_answer = answers["change_target"]
target = DeliverableKind(str(target_answer.value))
readings.change_target = target
if disposition == ACT and (
target_answer.confidence < self._thresholds.confidence
or float(answers["change_clarity"].value) < 0.5
):
# They want a change, but which, or what, is not clear enough to act on.
disposition = QUEUE
if disposition == QUEUE and pending:
# Asked once already: act if they named it, otherwise let it go.
disposition = ACT if len(named) == 1 else CLOSE
readings.change_target = named[0] if named else target
readings.change = disposition
async def _read_gap(self, view: TurnView, readings: Readings) -> None:
"""Whether a gap in the Target architecture is worth revising it for."""
if (
readings.failed
or readings.dry_run_failed
or not readings.validation_attempt
):
return
if not (readings.additions or readings.unverifiable):
return
items = "\n".join(
[
f"- proposed addition on {item.get('applies_to')}: "
f"{item.get('statement')} (serves {item.get('reason')})"
for item in readings.additions
]
+ [
f"- Check {item.get('check')} could not be verified: "
f"{item.get('evidence')}"
for item in readings.unverifiable
]
)
reply = view.last_reply if view.awaiting_gap_decision else None
if reply:
question = Question(
"noul",
"Is the user asking for the Target architecture to be revised to "
"close these gaps?",
)
state = f"Gaps found by validation:\n{items}\nThe user's reply: {reply.get('text', '')}"
name = "wants_revision"
else:
question = Question(
"noul",
"Would leaving these gaps open mean a stated requirement or design "
"recommendation is not met?",
)
state = f"Gaps found by validation:\n{items}"
name = "gap_material"
answers = await self._decider.decide(
state, {name: question}, scope=view.execution_id
)
if answers is None:
readings.gap = self._gap_by_rule(view, readings) if not reply else CLOSE
return
readings.judged[name] = {
"value": answers[name].value,
"confidence": answers[name].confidence,
}
disposition = self._thresholds.disposition(float(answers[name].value))
if reply and disposition == QUEUE:
disposition = CLOSE # asked once already; an unclear answer means go on
readings.gap = disposition
@staticmethod
def _gap_by_rule(view: TurnView, readings: Readings) -> str:
"""Without a judgement: a Check that cannot be verified must be fixed, and an
addition counts when it names a real requirement or recommendation."""
if readings.unverifiable:
return ACT
known = {
item["id"]
for name, kind in (
(REQUIREMENTS_FILE, "requirements"),
(RECOMMENDATIONS_FILE, "design_recommendations"),
)
for item in read_list(view.deliverables, kind, name)
if "id" in item
}
for item in readings.additions:
words = set(re.findall(r"[A-Za-z]+-\d+", str(item.get("reason", ""))))
if words & known:
return ACT
return CLOSE
# --- the playbook -----------------------------------------------------------------
def _choose(
self, view: TurnView, r: Readings, blocked: set[str], settled: int
) -> tuple[str, str]:
"""The first group that applies, and within it the strongest action whose
conditions hold. No group applies: escalate."""
interactive = not view.pre_approved
playbook: list[tuple[str, bool, list[tuple[str, bool]]]] = [
(
"the user asked for a change",
r.change in {ACT, QUEUE},
[
(
"revise_requested",
r.change == ACT and r.change_target is not None,
),
("confirm_change", r.change == QUEUE and interactive),
],
),
(
"questions are waiting",
r.questions_waiting,
[("relay_questions", True)],
),
(
# Known as soon as the code is submitted, so no validation is spent
# on a Check that cannot pass.
"the code cannot meet a Check",
bool(r.unimplementable),
[("revise_unimplementable", True)],
),
(
"validation decided",
bool(r.validation_attempt),
[
(
"revise_target_architecture",
bool(r.repeat_failed) or (not r.failed and r.gap == ACT),
),
("revise_code", bool(r.failed) or r.dry_run_failed),
("ask_about_gaps", r.gap == QUEUE and interactive),
("proceed", True),
],
),
(
"a Phase is complete",
bool(r.phases_ready or r.phases_awaiting),
# An Approval is granted only by a structured decision, so a Phase
# still awaiting one is asked for again.
[("request_approval", True)],
),
("work is ready", bool(r.ready_work), [("delegate", True)]),
("packaging is approved", r.can_finish, [("finish", True)]),
]
for group, applies, actions in playbook:
if not applies:
continue
for action, holds in actions:
if holds and action not in blocked:
return group, action
return "no group applies", "escalate"
async def _perform(
self,
action: str,
view: TurnView,
r: Readings,
call: Callable[..., Awaitable[ToolResult]],
result: TurnResult,
) -> ToolResult | None:
if action == "proceed":
return ToolResult("Proceeding.")
if action == "revise_requested":
assert r.change_target is not None
return await call(
"reopen", kind=r.change_target.value, reason=r.change_reason
)
if action == "confirm_change":
guess = r.change_target.value if r.change_target else None
result.updates["pending_change"] = {"text": r.change_reason, "guess": guess}
options = ", ".join(kind.value for kind in _submitted(view.ledger))
question = (
f"Do you want {guess} changed? Reply naming the Deliverable to change "
f"and what should be different. Submitted so far: {options}."
if guess
else "Which Deliverable should change, and what should be different? "
f"Submitted so far: {options}."
)
return await call("ask_user", questions=[question])
if action == "relay_questions":
return await call("ask_user", questions=[])
if action == "revise_code":
items = r.failed or [
{"check": "dry-run", "evidence": "the deployment dry-run failed"}
]
reason = "; ".join(f"{i['check']} failed: {i['evidence']}" for i in items)
return await call(
"reopen", kind=DeliverableKind.DEPLOYMENT_GUIDANCE.value, reason=reason
)
if action == "revise_unimplementable":
reason = " ".join(
f"Check {item['check']} cannot be implemented as written "
f"({item['reason']}). Rewrite it so it can be met, or remove it with "
"a reason."
for item in r.unimplementable
)
return await call(
"reopen", kind=DeliverableKind.TARGET_ARCHITECTURE.value, reason=reason
)
if action == "revise_target_architecture":
return await call(
"reopen",
kind=DeliverableKind.TARGET_ARCHITECTURE.value,
reason=_graph_reason(r),
)
if action == "ask_about_gaps":
result.updates["awaiting_gap_decision"] = True
listed = "; ".join(
[str(i.get("statement")) for i in r.additions]
+ [
f"Check {i.get('check')} could not be verified"
for i in r.unverifiable
]
)
question = (
"Validation passed every Check it could, and found gaps in the "
f"Target architecture: {listed}. Should the Target architecture "
"be revised to close them, or shall I proceed as it stands?"
)
return await call("ask_user", questions=[question])
if action == "request_approval":
phase = (r.phases_ready or r.phases_awaiting)[0]
return await call(
"request_approval",
phase=phase.value,
summary=await self._summary(phase, view, r),
)
if action == "delegate":
return await call("delegate", kinds=[kind.value for kind in r.ready_work])
if action == "finish":
return await call("finish")
# escalate: no group applies.
if view.pre_approved:
result.outcome = Stopped(
"No next step applies and no user is available to decide."
)
return None
return await call(
"ask_user",
questions=["I have no next step to take. What would you like me to do?"],
)
async def _summary(self, phase: PhaseId, view: TurnView, r: Readings) -> str:
facts = approval_facts(phase, view, r)
if self._writer is not None:
try:
written = await self._writer.approval_summary(phase.value, facts)
except Exception: # a summary must never stop an Approval
logger.exception("The reply writer failed; using the assembled summary")
written = None
if written and written.strip():
return written.strip()
return assembled_summary(phase, facts)
def _graph_reason(r: Readings) -> str:
parts = [
f"Check {item['check']} failed in two validations in a row and may not be "
f"implementable as written ({item['evidence']}). Rewrite it so it can be met, "
"or remove it with a reason."
for item in r.repeat_failed
]
if not r.repeat_failed:
parts += [
f"Check {item.get('check')} could not be verified from the code "
f"({item.get('evidence')}). Rewrite it so it can be."
for item in r.unverifiable
]
parts += [
f"Add a Check on {item.get('applies_to')}: {item.get('statement')} "
f"(serves {item.get('reason')})."
for item in r.additions
]
return " ".join(parts)
def approval_facts(phase: PhaseId, view: TurnView, r: Readings) -> dict[str, Any]:
"""What an Approval summary is written from: data, never Deliverable prose."""
kinds = [kind for kind, spec in DEPENDENCY_GRAPH.items() if spec.phase is phase]
facts: dict[str, Any] = {
"phase": phase.value,
"deliverables": [kind.value for kind in kinds],
"revisions": {
kind.value: view.ledger.revisions(kind)
for kind in kinds
if view.ledger.revisions(kind)
},
}
counts = {
"requirements": len(
read_list(view.deliverables, "requirements", REQUIREMENTS_FILE)
),
"recommendations": len(
read_list(view.deliverables, "design_recommendations", RECOMMENDATIONS_FILE)
),
}
facts["counts"] = {name: count for name, count in counts.items() if count}
graph = read_graph(view.deliverables)
if graph is not None:
facts["target_architecture"] = graph.summary()
validation = view.deliverables.get(DeliverableKind.VALIDATION_RESULTS.value)
if (
validation
and phase is DEPENDENCY_GRAPH[DeliverableKind.VALIDATION_RESULTS].phase
):
outcome = validation_outcome(
validation.get("attachments", {}).get(VERDICTS_FILE, "")
)
facts["validation"] = {
"passed": len(outcome["passed"]),
"failed": [item["check"] for item in outcome["failed"]],
"not_verifiable": [item["check"] for item in outcome["not_verifiable"]],
"open_gaps": [
item.get("statement") for item in outcome["proposed_additions"]
],
}
return facts
def assembled_summary(phase: PhaseId, facts: Mapping[str, Any]) -> str:
"""An Approval summary from the facts alone, used when no writer is available."""
lines = [
(
f"The {phase.value} Phase is ready: "
f"{', '.join(facts['deliverables'])} submitted."
)
]
counts = facts.get("counts") or {}
if counts:
lines.append(
", ".join(f"{count} {name}" for name, count in counts.items()) + "."
)
graph = facts.get("target_architecture")
if graph:
lines.append(
f"Target architecture: {graph['elements']} Elements, "
f"{graph['relationships']} Relationships, {graph['checks']} Checks."
)
validation = facts.get("validation")
if validation:
lines.append(
f"Validation: {validation['passed']} Checks passed, "
f"{len(validation['failed'])} failed, "
f"{len(validation['not_verifiable'])} not verifiable."
)
if validation["open_gaps"]:
lines.append(
"Open gaps: " + "; ".join(map(str, validation["open_gaps"])) + "."
)
if facts.get("revisions"):
lines.append(
"Revised: "
+ ", ".join(
f"{kind} × {count}" for kind, count in facts["revisions"].items()
)
+ "."
)
return " ".join(lines)
# --- the two ways an Orchestrator turn can be driven ---------------------------------
class ModelOrchestrator:
"""The Orchestrator as a reasoning agent: a language model chooses the tool calls.
Kept so the Ledger's rules can be driven in any order by a scripted runtime in
tests, and so an evaluation can compare it with the router.
"""
def __init__(self, runtime: AgentRuntime, role: RoleSpec) -> None:
self._runtime = runtime
self._role = role
async def turn(self, view: TurnView, tools: Sequence[ToolSpec]) -> TurnResult:
task = AgentTask(
role=ORCHESTRATOR,
prompt="Decide the next step for this Execution using your tools.",
context=view.context,
)
return TurnResult(await self._runtime.run(self._role, task, tools))
class RuntimeReplyWriter:
"""Writes what the user reads, on a language model. It is given a decision that
has already been made and the facts, and it decides nothing."""
def __init__(self, runtime: AgentRuntime, role: RoleSpec) -> None:
self._runtime = runtime
self._role = role
async def approval_summary(
self, phase: str, facts: Mapping[str, Any]
) -> str | None:
async def write_reply(text: str) -> ToolResult:
if not text.strip():
return ToolResult("Not submitted: the reply is empty.")
return ToolResult("Written.", Submitted(text))
tool = ToolSpec(
"write_reply",
"Give the text the user will read. Ends your turn.",
{
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
},
write_reply,
tier="execution",
)
task = AgentTask(
role=REPLY_WRITER,
prompt=f"Write the summary the user will approve the {phase} Phase from.",
context={"phase": phase, "facts": dict(facts)},
)
outcome = await self._runtime.run(self._role, task, [tool])
return outcome.content if isinstance(outcome, Submitted) else None