fix: read only the user's own words, and stop the router looping at a limit

Case 04, the first interactive evaluation, stopped at 100 Orchestrator
turns on two router faults:

- A reply that quoted the agent's question back was judged on that question.
  A reply now reaches the router with quoted questions and Q:/A: labels
  removed, and the questions are given to the decider as background.
- At a Revision limit the reply was read as another change request for the
  same Deliverable, and the refusal repeated. The user is now told once,
  the reply is taken as the answer, that Deliverable is not reopened again,
  and the Execution carries on with it as it stands.
- Any run that asks the user four times running with no work done in
  between now ends with a warning.

The validator is told to confirm with the user in an interactive Execution.

Plans record the fixes and the rerun, which did not finish: both fixes
held, and a third fault appeared that is not fixed here. A scripted answer
restating a requirement, given to a draft-confirmation question, reopens
requirements.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 15:05:03 +01:00
parent c9f59be274
commit d41607f505
11 changed files with 321 additions and 9 deletions

View File

@@ -84,6 +84,10 @@ class TurnView:
pending_change: Mapping[str, Any] | None = None # a change we asked them to confirm
awaiting_gap_decision: bool = False
validation_settled: int = 0 # the validation attempt already decided on
# The user's reply answers "this Deliverable reached its Revision limit".
awaiting_limit_decision: bool = False
# Deliverables at their Revision limit that the user has been told about.
revision_limit_asked: Sequence[str] = ()
context: Mapping[str, Any] = field(default_factory=dict) # for a model Orchestrator
@@ -135,6 +139,25 @@ class Readings:
judged: dict[str, Any] = field(default_factory=dict) # what the decider answered
# The actions that reopen a Deliverable, and which one.
_REOPENS = (
"revise_requested",
"revise_code",
"revise_target_architecture",
"revise_unimplementable",
)
def _reopened_by(action: str, readings: Readings) -> DeliverableKind | None:
if action == "revise_requested":
return readings.change_target
if action == "revise_code":
return DeliverableKind.DEPLOYMENT_GUIDANCE
if action in {"revise_target_architecture", "revise_unimplementable"}:
return DeliverableKind.TARGET_ARCHITECTURE
return None
def _attempt(ledger: Ledger, kind: DeliverableKind) -> int:
return int(ledger.to_dict()["deliverables"][kind.value].get("attempt", 0))
@@ -194,16 +217,25 @@ class Router:
"last_reply": None,
"pending_change": None,
"awaiting_gap_decision": False,
"awaiting_limit_decision": False,
}
readings = await self._read(view, result)
blocked: set[str] = set()
settled = view.validation_settled
# Told once, the user is not asked again: a Deliverable at its limit is no
# longer reopened, and the Execution carries on with it as it stands.
at_limit = set(view.revision_limit_asked)
async def call(tool: str, **arguments: Any) -> ToolResult:
return await by_name[tool].handler(**arguments)
for _ in range(MAX_STEPS):
group, action = self._choose(view, readings, blocked, settled)
barred = blocked | {
name
for name in _REOPENS
if (kind := _reopened_by(name, readings)) and kind.value in at_limit
}
group, action = self._choose(view, readings, barred, settled)
result.decisions.append(
{"group": group, "action": action, "judged": dict(readings.judged)}
)
@@ -220,9 +252,20 @@ class Router:
result.outcome = reply.outcome
return result
if "revision limit" in reply.message and not view.pre_approved:
# Past the Revision limit the user decides.
# Past the Revision limit the user is told, once. Their reply is
# read as the answer to this, never as a new request for changes.
kind = _reopened_by(action, readings)
if kind:
at_limit.add(kind.value)
result.updates["revision_limit_asked"] = sorted(at_limit)
result.updates["awaiting_limit_decision"] = True
limit = await call(
"ask_user", questions=[reply.message.split(" Use ")[0]]
"ask_user",
questions=[
reply.message.split(" Use ")[0]
+ " It will not be revised again in this Execution; I will "
"carry on with it as it stands. Reply to continue."
],
)
if limit.outcome is not None:
result.outcome = limit.outcome
@@ -298,7 +341,7 @@ class Router:
"""Triage of the user's message: is it asking for a change, to what, and how
clearly?"""
reply = view.last_reply
if not reply or view.awaiting_gap_decision:
if not reply or view.awaiting_gap_decision or view.awaiting_limit_decision:
return
text = str(reply.get("text", "")).strip()
pending = view.pending_change
@@ -308,6 +351,7 @@ class Router:
if not text or not candidates:
return
readings.change_reason = text
asked = [str(question) for question in reply.get("asked") or ()]
# A structured request for changes is known to be one, and so is the reply to
# our own question about a change they already asked for.
structured = reply.get("kind") == "changes" or bool(pending)
@@ -334,7 +378,16 @@ class Router:
state = (
f"Submitted Deliverables: {', '.join(kind.value for kind in candidates)}.\n"
f"Questions waiting for the user: {len(view.pending_questions)}.\n"
f"The user's message: {text}"
# What we asked is background. Only the user's own words are their
# message: our question echoed back is not something they said.
+ (
"The user was replying to these questions from the agents:\n"
+ "\n".join(f"- {question}" for question in asked)
+ "\n"
if asked
else ""
)
+ f"The user's message: {text}"
)
answers = await self._decider.decide(state, questions, scope=view.execution_id)