fix: read only the user's own words, and stop the router looping at a limit
Case 04, the first interactive evaluation, stopped at 100 Orchestrator turns on two router faults: - A reply that quoted the agent's question back was judged on that question. A reply now reaches the router with quoted questions and Q:/A: labels removed, and the questions are given to the decider as background. - At a Revision limit the reply was read as another change request for the same Deliverable, and the refusal repeated. The user is now told once, the reply is taken as the answer, that Deliverable is not reopened again, and the Execution carries on with it as it stands. - Any run that asks the user four times running with no work done in between now ends with a warning. The validator is told to confirm with the user in an interactive Execution. Plans record the fixes and the rerun, which did not finish: both fixes held, and a third fault appeared that is not fixed here. A scripted answer restating a requirement, given to a draft-confirmation question, reopens requirements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -64,6 +64,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# Working-memory key: the last submission of a Deliverable with what it was written from.
|
||||
STANDING = "standing"
|
||||
# Questions put to the user running, with no work done between, before stopping.
|
||||
MAX_IDLE_ASKS = 3
|
||||
|
||||
NO_USER_REPLY = (
|
||||
"No user is available. State your assumption, record it in the Deliverable under "
|
||||
@@ -266,6 +268,8 @@ def make_execution_nodes(
|
||||
pending_change=state.get("pending_change"),
|
||||
awaiting_gap_decision=bool(state.get("awaiting_gap_decision")),
|
||||
validation_settled=int(state.get("validation_settled", 0)),
|
||||
awaiting_limit_decision=bool(state.get("awaiting_limit_decision")),
|
||||
revision_limit_asked=tuple(state.get("revision_limit_asked", ())),
|
||||
context=_orchestrator_context(
|
||||
state, ledger, deliverables, pending_questions, notes
|
||||
),
|
||||
@@ -332,6 +336,22 @@ def make_execution_nodes(
|
||||
return ask({"approval": toolbox.approval})
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "ask_user":
|
||||
# Asking again and again with nothing done in between is a loop, not a
|
||||
# conversation: stop it, whatever caused it.
|
||||
fingerprint = _progress(ledger)
|
||||
idle = (
|
||||
state.get("idle_asks", 0) + 1
|
||||
if fingerprint == state.get("ask_fingerprint")
|
||||
else 0
|
||||
)
|
||||
update.update(idle_asks=idle, ask_fingerprint=fingerprint)
|
||||
if idle >= MAX_IDLE_ASKS:
|
||||
warnings.append(
|
||||
f"Stopped: the user was asked {idle + 1} times running with "
|
||||
"no work done in between."
|
||||
)
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
update["events_recorded"] = await record_progress(
|
||||
state, ledger, None if pre_approved else "awaiting_user"
|
||||
)
|
||||
@@ -519,7 +539,9 @@ def make_execution_nodes(
|
||||
else:
|
||||
text = _reply_text(reply)
|
||||
conversation.append({"from": "user", "text": text})
|
||||
said = {"kind": "reply", "text": text}
|
||||
put = [item["question"] for item in prompt.get("questions", [])]
|
||||
# What is routed is what the user said, not our questions quoted back.
|
||||
said = {"kind": "reply", "text": _own_words(text, put), "asked": put}
|
||||
answers = {
|
||||
kind: list(items) for kind, items in state.get("answers", {}).items()
|
||||
}
|
||||
@@ -775,6 +797,35 @@ def _reply_text(reply: object) -> str:
|
||||
return str(reply)
|
||||
|
||||
|
||||
def _own_words(text: str, asked: Sequence[str]) -> str:
|
||||
"""A reply without the questions it quotes, and without Q: and A: labels."""
|
||||
for question in sorted(asked, key=len, reverse=True):
|
||||
text = text.replace(question, "")
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
line = re.sub(r"^\s*(?:Q|A)\s*:\s*", "", line).strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _progress(ledger: Ledger) -> str:
|
||||
"""The work done so far, as one comparable value: each Deliverable's status and
|
||||
attempt, and each Phase's status."""
|
||||
data = ledger.to_dict()
|
||||
return json.dumps(
|
||||
{
|
||||
"deliverables": {
|
||||
kind: [record.get("status"), record.get("attempt")]
|
||||
for kind, record in data["deliverables"].items()
|
||||
},
|
||||
"phases": data.get("phases"),
|
||||
},
|
||||
sort_keys=True,
|
||||
default=str,
|
||||
)
|
||||
|
||||
|
||||
def _render_prompt(prompt: Mapping[str, Any]) -> str:
|
||||
if "approval" in prompt:
|
||||
approval = prompt["approval"]
|
||||
|
||||
Reference in New Issue
Block a user