fix: read only the user's own words, and stop the router looping at a limit
Case 04, the first interactive evaluation, stopped at 100 Orchestrator turns on two router faults: - A reply that quoted the agent's question back was judged on that question. A reply now reaches the router with quoted questions and Q:/A: labels removed, and the questions are given to the decider as background. - At a Revision limit the reply was read as another change request for the same Deliverable, and the refusal repeated. The user is now told once, the reply is taken as the answer, that Deliverable is not reopened again, and the Execution carries on with it as it stands. - Any run that asks the user four times running with no work done in between now ends with a warning. The validator is told to confirm with the user in an interactive Execution. Plans record the fixes and the rerun, which did not finish: both fixes held, and a third fault appeared that is not fixed here. A scripted answer restating a requirement, given to a draft-confirmation question, reopens requirements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -84,6 +84,10 @@ class TurnView:
|
||||
pending_change: Mapping[str, Any] | None = None # a change we asked them to confirm
|
||||
awaiting_gap_decision: bool = False
|
||||
validation_settled: int = 0 # the validation attempt already decided on
|
||||
# The user's reply answers "this Deliverable reached its Revision limit".
|
||||
awaiting_limit_decision: bool = False
|
||||
# Deliverables at their Revision limit that the user has been told about.
|
||||
revision_limit_asked: Sequence[str] = ()
|
||||
context: Mapping[str, Any] = field(default_factory=dict) # for a model Orchestrator
|
||||
|
||||
|
||||
@@ -135,6 +139,25 @@ class Readings:
|
||||
judged: dict[str, Any] = field(default_factory=dict) # what the decider answered
|
||||
|
||||
|
||||
# The actions that reopen a Deliverable, and which one.
|
||||
_REOPENS = (
|
||||
"revise_requested",
|
||||
"revise_code",
|
||||
"revise_target_architecture",
|
||||
"revise_unimplementable",
|
||||
)
|
||||
|
||||
|
||||
def _reopened_by(action: str, readings: Readings) -> DeliverableKind | None:
|
||||
if action == "revise_requested":
|
||||
return readings.change_target
|
||||
if action == "revise_code":
|
||||
return DeliverableKind.DEPLOYMENT_GUIDANCE
|
||||
if action in {"revise_target_architecture", "revise_unimplementable"}:
|
||||
return DeliverableKind.TARGET_ARCHITECTURE
|
||||
return None
|
||||
|
||||
|
||||
def _attempt(ledger: Ledger, kind: DeliverableKind) -> int:
|
||||
return int(ledger.to_dict()["deliverables"][kind.value].get("attempt", 0))
|
||||
|
||||
@@ -194,16 +217,25 @@ class Router:
|
||||
"last_reply": None,
|
||||
"pending_change": None,
|
||||
"awaiting_gap_decision": False,
|
||||
"awaiting_limit_decision": False,
|
||||
}
|
||||
readings = await self._read(view, result)
|
||||
blocked: set[str] = set()
|
||||
settled = view.validation_settled
|
||||
# Told once, the user is not asked again: a Deliverable at its limit is no
|
||||
# longer reopened, and the Execution carries on with it as it stands.
|
||||
at_limit = set(view.revision_limit_asked)
|
||||
|
||||
async def call(tool: str, **arguments: Any) -> ToolResult:
|
||||
return await by_name[tool].handler(**arguments)
|
||||
|
||||
for _ in range(MAX_STEPS):
|
||||
group, action = self._choose(view, readings, blocked, settled)
|
||||
barred = blocked | {
|
||||
name
|
||||
for name in _REOPENS
|
||||
if (kind := _reopened_by(name, readings)) and kind.value in at_limit
|
||||
}
|
||||
group, action = self._choose(view, readings, barred, settled)
|
||||
result.decisions.append(
|
||||
{"group": group, "action": action, "judged": dict(readings.judged)}
|
||||
)
|
||||
@@ -220,9 +252,20 @@ class Router:
|
||||
result.outcome = reply.outcome
|
||||
return result
|
||||
if "revision limit" in reply.message and not view.pre_approved:
|
||||
# Past the Revision limit the user decides.
|
||||
# Past the Revision limit the user is told, once. Their reply is
|
||||
# read as the answer to this, never as a new request for changes.
|
||||
kind = _reopened_by(action, readings)
|
||||
if kind:
|
||||
at_limit.add(kind.value)
|
||||
result.updates["revision_limit_asked"] = sorted(at_limit)
|
||||
result.updates["awaiting_limit_decision"] = True
|
||||
limit = await call(
|
||||
"ask_user", questions=[reply.message.split(" Use ")[0]]
|
||||
"ask_user",
|
||||
questions=[
|
||||
reply.message.split(" Use ")[0]
|
||||
+ " It will not be revised again in this Execution; I will "
|
||||
"carry on with it as it stands. Reply to continue."
|
||||
],
|
||||
)
|
||||
if limit.outcome is not None:
|
||||
result.outcome = limit.outcome
|
||||
@@ -298,7 +341,7 @@ class Router:
|
||||
"""Triage of the user's message: is it asking for a change, to what, and how
|
||||
clearly?"""
|
||||
reply = view.last_reply
|
||||
if not reply or view.awaiting_gap_decision:
|
||||
if not reply or view.awaiting_gap_decision or view.awaiting_limit_decision:
|
||||
return
|
||||
text = str(reply.get("text", "")).strip()
|
||||
pending = view.pending_change
|
||||
@@ -308,6 +351,7 @@ class Router:
|
||||
if not text or not candidates:
|
||||
return
|
||||
readings.change_reason = text
|
||||
asked = [str(question) for question in reply.get("asked") or ()]
|
||||
# A structured request for changes is known to be one, and so is the reply to
|
||||
# our own question about a change they already asked for.
|
||||
structured = reply.get("kind") == "changes" or bool(pending)
|
||||
@@ -334,7 +378,16 @@ class Router:
|
||||
state = (
|
||||
f"Submitted Deliverables: {', '.join(kind.value for kind in candidates)}.\n"
|
||||
f"Questions waiting for the user: {len(view.pending_questions)}.\n"
|
||||
f"The user's message: {text}"
|
||||
# What we asked is background. Only the user's own words are their
|
||||
# message: our question echoed back is not something they said.
|
||||
+ (
|
||||
"The user was replying to these questions from the agents:\n"
|
||||
+ "\n".join(f"- {question}" for question in asked)
|
||||
+ "\n"
|
||||
if asked
|
||||
else ""
|
||||
)
|
||||
+ f"The user's message: {text}"
|
||||
)
|
||||
answers = await self._decider.decide(state, questions, scope=view.execution_id)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user