fix: read only the user's own words, and stop the router looping at a limit
Case 04, the first interactive evaluation, stopped at 100 Orchestrator turns on two router faults: - A reply that quoted the agent's question back was judged on that question. A reply now reaches the router with quoted questions and Q:/A: labels removed, and the questions are given to the decider as background. - At a Revision limit the reply was read as another change request for the same Deliverable, and the refusal repeated. The user is now told once, the reply is taken as the answer, that Deliverable is not reopened again, and the Execution carries on with it as it stands. - Any run that asks the user four times running with no work done in between now ends with a warning. The validator is told to confirm with the user in an interactive Execution. Plans record the fixes and the rerun, which did not finish: both fixes held, and a third fault appeared that is not fixed here. A scripted answer restating a requirement, given to a draft-confirmation question, reopens requirements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -596,3 +596,162 @@ async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
||||
"deployment_guidance has reached its revision limit (1)"
|
||||
in (question["question"])
|
||||
)
|
||||
|
||||
|
||||
# --- the two faults of evaluation case 04 (2026-10-08) -----------------------------------
|
||||
|
||||
|
||||
async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on(
|
||||
tmp_path,
|
||||
):
|
||||
"""The user is told once. Their reply is not read as a new request for changes,
|
||||
the Deliverable is not reopened again, and nothing is asked a second time."""
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
||||
# A decider that would call any reply a clear request to change the code.
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.95, 0.9),
|
||||
"change_target": Answer("deployment_guidance", 0.95),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
[question] = result.prompt["questions"]
|
||||
assert "I will carry on with it as it stands" in question["question"]
|
||||
|
||||
result = await engine.resume(
|
||||
result.execution_id, {"reply": "Please change the code again."}
|
||||
)
|
||||
|
||||
# It moves on to the Approval of validation; the code was reopened once only.
|
||||
assert result.prompt["approval"]["phase"] == "solution_validation"
|
||||
assert len(reopened(result)) == 1
|
||||
assert result.state["revision_limit_asked"] == ["deployment_guidance"]
|
||||
assert decider.asked == [] # the reply was never put to the decider
|
||||
|
||||
|
||||
async def test_only_the_users_own_words_are_read_as_their_message(tmp_path):
|
||||
"""A reply that quotes our question back must not be judged on our words."""
|
||||
question = "Please confirm the requirements draft. Shall I change anything?"
|
||||
turns = once()
|
||||
turns[specialist(K.REQUIREMENTS)] = [
|
||||
[Call("needs_input", {"questions": [question]})],
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
]
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.05, 0.9),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.2, 0.5),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
result = await engine.resume(
|
||||
first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."}
|
||||
)
|
||||
# Nothing has been submitted yet, so nothing is judged on that turn. Let the
|
||||
# next reply, to the decomposer's approval prompt, be a text one.
|
||||
result = await engine.resume(result.execution_id, {"reply": "Looks fine."})
|
||||
|
||||
assert result.state["last_reply"] is None
|
||||
state, _, _ = decider.asked[-1]
|
||||
assert state.endswith("The user's message: Looks fine.")
|
||||
|
||||
|
||||
def test_a_quoted_question_is_removed_from_what_the_user_said():
|
||||
from app.nodes.execution_nodes import _own_words
|
||||
|
||||
asked = ["Please confirm the draft. Shall I change the design?", "Which region?"]
|
||||
reply = (
|
||||
"Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n"
|
||||
"Q: Which region?\nA: europe-west2"
|
||||
)
|
||||
|
||||
assert _own_words(reply, asked) == "Yes, approved.\neurope-west2"
|
||||
assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years."
|
||||
|
||||
|
||||
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
|
||||
from app.nodes.router import TurnView
|
||||
from app.states.ledger import Ledger
|
||||
|
||||
ledger = Ledger()
|
||||
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.05, 0.9),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.2, 0.5),
|
||||
}
|
||||
)
|
||||
router = Router(decider, None, Thresholds())
|
||||
view = TurnView(
|
||||
execution_id="e1",
|
||||
pre_approved=False,
|
||||
ledger=ledger,
|
||||
deliverables={},
|
||||
pending_questions=[],
|
||||
outcomes=[],
|
||||
last_reply={
|
||||
"kind": "reply",
|
||||
"text": "Yes, approved.",
|
||||
"asked": ["Shall I change the design?"],
|
||||
},
|
||||
)
|
||||
from app.nodes.router import Readings
|
||||
|
||||
await router._read_message(view, Readings())
|
||||
|
||||
state, _, _ = decider.asked[0]
|
||||
assert "replying to these questions from the agents:\n- Shall I change" in state
|
||||
assert state.endswith("The user's message: Yes, approved.")
|
||||
|
||||
|
||||
async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
||||
tmp_path,
|
||||
):
|
||||
"""Whatever causes it, a run that only asks is stopped, not left to the turn limit."""
|
||||
from app.nodes.execution_nodes import MAX_IDLE_ASKS
|
||||
from app.nodes.router import TurnResult
|
||||
|
||||
class OnlyAsks:
|
||||
"""An Orchestrator that asks the user something on every turn."""
|
||||
|
||||
async def turn(self, view, tools):
|
||||
ask = next(tool for tool in tools if tool.name == "ask_user")
|
||||
reply = await ask.handler(questions=["What now?"])
|
||||
return TurnResult(reply.outcome)
|
||||
|
||||
engine = ExecutionEngine(
|
||||
runtime=FakeAgentRuntime({}),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=OnlyAsks(),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
asked = 1
|
||||
while result.status == "awaiting_user" and asked < 20:
|
||||
result = await engine.resume(result.execution_id, {"reply": "Carry on."})
|
||||
asked += 1
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert asked == MAX_IDLE_ASKS + 1
|
||||
assert "asked" in result.state["warnings"][-1]
|
||||
assert "no work done in between" in result.state["warnings"][-1]
|
||||
|
||||
Reference in New Issue
Block a user