Case 04, the first interactive evaluation, stopped at 100 Orchestrator turns on two router faults: - A reply that quoted the agent's question back was judged on that question. A reply now reaches the router with quoted questions and Q:/A: labels removed, and the questions are given to the decider as background. - At a Revision limit the reply was read as another change request for the same Deliverable, and the refusal repeated. The user is now told once, the reply is taken as the answer, that Deliverable is not reopened again, and the Execution carries on with it as it stands. - Any run that asks the user four times running with no work done in between now ends with a warning. The validator is told to confirm with the user in an interactive Execution. Plans record the fixes and the rerun, which did not finish: both fixes held, and a third fault appeared that is not fixed here. A scripted answer restating a requirement, given to a draft-confirmation question, reopens requirements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
758 lines
25 KiB
Python
758 lines
25 KiB
Python
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
|
|
|
|
No language model chooses an action here. Specialists are scripted; the Orchestrator
|
|
is the Router, with a scripted decider standing in for Jev, or none at all.
|
|
"""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import (
|
|
SAMPLE_DATA,
|
|
Call,
|
|
FakeAgentRuntime,
|
|
FakeDecider,
|
|
sample_submission,
|
|
)
|
|
from app.adk.runtime import Answer, NoDecider, RoleSpec
|
|
from app.agent import ExecutionEngine
|
|
from app.nodes.router import (
|
|
REPLY_WRITER,
|
|
Router,
|
|
RuntimeReplyWriter,
|
|
Thresholds,
|
|
)
|
|
from app.recorder import InMemoryRecorder
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
K = DeliverableKind
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
|
|
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def once() -> dict:
|
|
return {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
|
|
|
|
def validation(**changes) -> Call:
|
|
"""A validation submission whose verdicts differ from the all-pass sample."""
|
|
call = sample_submission(K.VALIDATION_RESULTS)
|
|
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
verdicts.update(changes)
|
|
return Call(
|
|
call.tool,
|
|
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
|
)
|
|
|
|
|
|
def failing(check: str = "K-1") -> Call:
|
|
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
|
|
for item in checks:
|
|
if item["check"] == check:
|
|
item.update(verdict="fail", evidence="min instances is 1")
|
|
return validation(checks=checks)
|
|
|
|
|
|
def addition(reason: str) -> Call:
|
|
return validation(
|
|
proposed_additions=[
|
|
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
|
|
]
|
|
)
|
|
|
|
|
|
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
|
|
runtime = FakeAgentRuntime(turns)
|
|
router = Router(
|
|
decider or NoDecider(),
|
|
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
|
|
Thresholds(),
|
|
)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=router,
|
|
**options,
|
|
)
|
|
return engine, runtime
|
|
|
|
|
|
def actions(result) -> list[str]:
|
|
return [decision["action"] for decision in result.state["decisions"]]
|
|
|
|
|
|
def reopened(result) -> list[tuple[str, str]]:
|
|
return [
|
|
(event["subject"], event["detail"])
|
|
for event in result.state["ledger"]["events"]
|
|
if event["type"] == "reopened"
|
|
]
|
|
|
|
|
|
def approve(phase: str) -> dict:
|
|
return {"approval": {"phase": phase, "decision": "approve"}}
|
|
|
|
|
|
# --- the playbook on rules alone -------------------------------------------------------
|
|
|
|
|
|
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
|
|
engine, runtime = engine_for(tmp_path, once())
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert "orchestrator" not in {run.role for run in runtime.runs}
|
|
assert actions(result) == [
|
|
"delegate", # requirements
|
|
"delegate", # technical decomposition
|
|
"request_approval",
|
|
"delegate", # product selection
|
|
"delegate", # design recommendations
|
|
"delegate", # Target architecture
|
|
"request_approval",
|
|
"delegate", # description and deployment guidance, together
|
|
"request_approval",
|
|
"delegate", # validation
|
|
"proceed",
|
|
"request_approval",
|
|
"delegate", # solution guide
|
|
"request_approval",
|
|
"finish",
|
|
]
|
|
groups = {decision["group"] for decision in result.state["decisions"]}
|
|
assert groups == {
|
|
"work is ready",
|
|
"a Phase is complete",
|
|
"validation decided",
|
|
"packaging is approved",
|
|
}
|
|
|
|
|
|
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once())
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
first = result.state["decisions"][0]
|
|
assert first == {
|
|
"turn": 1,
|
|
"group": "work is ready",
|
|
"action": "delegate",
|
|
"judged": {},
|
|
}
|
|
assert [d["turn"] for d in result.state["decisions"]] == sorted(
|
|
d["turn"] for d in result.state["decisions"]
|
|
)
|
|
|
|
|
|
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
|
|
tmp_path,
|
|
):
|
|
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
|
|
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
# The Specialist's question is relayed, answered automatically, and it stops
|
|
# again; the router never invents work.
|
|
assert result.status == "completed_with_warnings"
|
|
|
|
|
|
# --- Approvals: the summary is written from data, and only data approves ----------------
|
|
|
|
|
|
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once())
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
|
|
|
|
assert first.prompt["approval"]["summary"] == (
|
|
"The requirements_discovery Phase is ready: requirements, "
|
|
"technical_decomposition submitted. 2 requirements."
|
|
)
|
|
summary = design.prompt["approval"]["summary"]
|
|
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
|
|
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
|
|
|
|
|
|
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
|
|
turns = {
|
|
**once(),
|
|
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
|
|
}
|
|
engine, runtime = engine_for(tmp_path, turns, writer=True)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
|
|
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
|
|
assert writer_run.tools == ("write_reply",)
|
|
assert writer_run.task.context["facts"]["deliverables"] == [
|
|
"requirements",
|
|
"technical_decomposition",
|
|
]
|
|
|
|
|
|
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
assert first.prompt["approval"]["summary"].startswith(
|
|
"The requirements_discovery Phase is ready"
|
|
)
|
|
|
|
|
|
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.03, 0.94),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.0, 0.9),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, once(), decider)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
|
|
|
|
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
|
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
|
|
|
|
|
|
# --- validation: the code, the graph, or proceed ----------------------------------------
|
|
|
|
|
|
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[failing("K-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, runtime = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == [
|
|
("deployment_guidance", "K-1 failed: min instances is 1")
|
|
]
|
|
assert "revise_code" in actions(result)
|
|
architect_runs = [
|
|
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
|
|
]
|
|
assert len(architect_runs) == 1
|
|
|
|
|
|
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
|
|
tmp_path,
|
|
):
|
|
"""The code's author found the Check names a setting the resource lacks. No
|
|
validation is spent on it: the Target architecture is revised first."""
|
|
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
|
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
|
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
|
|
turns = once()
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
|
[
|
|
Call(
|
|
cannot.tool,
|
|
{
|
|
**cannot.args,
|
|
"attachments": {
|
|
**cannot.args["attachments"],
|
|
"implementation.json": json.dumps(implementation),
|
|
},
|
|
},
|
|
)
|
|
],
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
]
|
|
engine, runtime = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == [
|
|
(
|
|
"target_architecture",
|
|
(
|
|
"Check K-2 cannot be implemented as written (no such argument). "
|
|
"Rewrite it so it can be met, or remove it with a reason."
|
|
),
|
|
)
|
|
]
|
|
assert "the code cannot meet a Check" in {
|
|
decision["group"] for decision in result.state["decisions"]
|
|
}
|
|
validations = [
|
|
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
|
|
]
|
|
assert len(validations) == 1
|
|
|
|
|
|
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
|
|
"""The code was revised and the same Check failed again: the Check is suspect."""
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
|
|
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[failing("K-1")],
|
|
[failing("K-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
[first, second] = reopened(result)
|
|
assert first[0] == "deployment_guidance"
|
|
assert second[0] == "target_architecture"
|
|
assert "K-1 failed in two validations in a row" in second[1]
|
|
assert actions(result).count("revise_code") == 1
|
|
|
|
|
|
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
|
|
turns = once()
|
|
for kind in (
|
|
K.TARGET_ARCHITECTURE,
|
|
K.ARCHITECTURE_DESCRIPTION,
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
):
|
|
turns[specialist(kind)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[addition("NFR-1 has no Check on timeouts")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
[(kind, reason)] = reopened(result)
|
|
assert kind == "target_architecture"
|
|
assert reason == (
|
|
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
|
|
)
|
|
|
|
|
|
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == []
|
|
|
|
|
|
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
|
|
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
|
|
engine, _ = engine_for(tmp_path, turns, immaterial)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
# The rule alone would have revised the graph; the reading says it is not needed.
|
|
assert reopened(result) == []
|
|
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
|
|
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
|
|
[(state, questions, scope)] = immaterial.asked
|
|
assert "A timeout is set." in state and scope == result.execution_id
|
|
assert questions["gap_material"].kind == "noul"
|
|
|
|
|
|
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
|
|
turns = once()
|
|
for kind in (
|
|
K.TARGET_ARCHITECTURE,
|
|
K.ARCHITECTURE_DESCRIPTION,
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
):
|
|
turns[specialist(kind)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[addition("NFR-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
decider = FakeDecider(
|
|
{
|
|
"gap_material": Answer(0.5, 0.0), # the grey zone
|
|
"wants_revision": Answer(0.96, 0.92),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
|
|
[question] = result.prompt["questions"]
|
|
assert (
|
|
"found gaps in the Target architecture: A timeout is set."
|
|
in (question["question"])
|
|
)
|
|
|
|
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
|
|
|
|
assert reopened(result)[0][0] == "target_architecture"
|
|
# Reopening the graph withdrew the design Approval, so it is asked for again.
|
|
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
|
|
|
|
|
# --- the user's message: act, queue or close --------------------------------------------
|
|
|
|
|
|
def to_design_approval(engine):
|
|
async def run():
|
|
first = await engine.start("Ingest webhooks")
|
|
return await engine.resume(
|
|
first.execution_id, approve("requirements_discovery")
|
|
)
|
|
|
|
return run()
|
|
|
|
|
|
def changes(phase: str, feedback: str) -> dict:
|
|
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
|
|
|
|
|
|
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"change_target": Answer("design_recommendations", 0.88),
|
|
"change_clarity": Answer(1.8, 0.8),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
design = await to_design_approval(engine)
|
|
result = await engine.resume(
|
|
design.execution_id,
|
|
changes("solution_architecture", "Use customer-managed keys on the topic."),
|
|
)
|
|
|
|
assert reopened(result) == [
|
|
("design_recommendations", "Use customer-managed keys on the topic.")
|
|
]
|
|
# Product selection was not touched; the design is asked for again once redone.
|
|
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
|
|
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
|
[(state, questions, _)] = decider.asked
|
|
assert "The user's message: Use customer-managed keys on the topic." in state
|
|
assert questions["change_target"].options == (
|
|
"requirements",
|
|
"technical_decomposition",
|
|
"product_selection",
|
|
"design_recommendations",
|
|
"target_architecture",
|
|
)
|
|
assert "asks_change" not in questions # a structured request is known to be one
|
|
|
|
|
|
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"change_target": Answer("product_selection", 0.31), # not relied on
|
|
"change_clarity": Answer(1.5, 0.7),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
design = await to_design_approval(engine)
|
|
asked = await engine.resume(
|
|
design.execution_id, changes("solution_architecture", "Tighten it up.")
|
|
)
|
|
|
|
assert reopened(asked) == []
|
|
[question] = asked.prompt["questions"]
|
|
assert question["question"].startswith("Do you want product_selection changed?")
|
|
|
|
result = await engine.resume(
|
|
asked.execution_id,
|
|
{"reply": "No, change the target architecture: add a Check on timeouts."},
|
|
)
|
|
|
|
[(kind, reason)] = reopened(result)
|
|
assert kind == "target_architecture"
|
|
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
|
|
|
|
|
|
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.REQUIREMENTS)] = [
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
]
|
|
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.93, 0.86),
|
|
"change_target": Answer("requirements", 0.9),
|
|
"change_clarity": Answer(1.9, 0.8),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
result = await engine.resume(
|
|
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
|
|
)
|
|
|
|
assert reopened(result) == [
|
|
("requirements", "Retention must be 7 years, not 30 days.")
|
|
]
|
|
|
|
|
|
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
design = await to_design_approval(engine)
|
|
unnamed = await engine.resume(
|
|
design.execution_id, changes("solution_architecture", "Make it cheaper.")
|
|
)
|
|
|
|
assert reopened(unnamed) == []
|
|
assert unnamed.prompt["questions"][0]["question"].startswith(
|
|
"Which Deliverable should change"
|
|
)
|
|
|
|
named = await engine.resume(
|
|
unnamed.execution_id, {"reply": "The design recommendations."}
|
|
)
|
|
|
|
[(kind, reason)] = reopened(named)
|
|
assert kind == "design_recommendations"
|
|
assert reason == (
|
|
"Make it cheaper.\nThe user then said: The design recommendations."
|
|
)
|
|
|
|
|
|
# --- permission is the Ledger's, whatever was decided ------------------------------------
|
|
|
|
|
|
async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
|
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
|
|
assert len(reopened(result)) == 1
|
|
[question] = result.prompt["questions"]
|
|
assert (
|
|
"deployment_guidance has reached its revision limit (1)"
|
|
in (question["question"])
|
|
)
|
|
|
|
|
|
# --- the two faults of evaluation case 04 (2026-10-08) -----------------------------------
|
|
|
|
|
|
async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on(
|
|
tmp_path,
|
|
):
|
|
"""The user is told once. Their reply is not read as a new request for changes,
|
|
the Deliverable is not reopened again, and nothing is asked a second time."""
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
|
# A decider that would call any reply a clear request to change the code.
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.95, 0.9),
|
|
"change_target": Answer("deployment_guidance", 0.95),
|
|
"change_clarity": Answer(1.9, 0.8),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
[question] = result.prompt["questions"]
|
|
assert "I will carry on with it as it stands" in question["question"]
|
|
|
|
result = await engine.resume(
|
|
result.execution_id, {"reply": "Please change the code again."}
|
|
)
|
|
|
|
# It moves on to the Approval of validation; the code was reopened once only.
|
|
assert result.prompt["approval"]["phase"] == "solution_validation"
|
|
assert len(reopened(result)) == 1
|
|
assert result.state["revision_limit_asked"] == ["deployment_guidance"]
|
|
assert decider.asked == [] # the reply was never put to the decider
|
|
|
|
|
|
async def test_only_the_users_own_words_are_read_as_their_message(tmp_path):
|
|
"""A reply that quotes our question back must not be judged on our words."""
|
|
question = "Please confirm the requirements draft. Shall I change anything?"
|
|
turns = once()
|
|
turns[specialist(K.REQUIREMENTS)] = [
|
|
[Call("needs_input", {"questions": [question]})],
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
]
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.05, 0.9),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.2, 0.5),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
result = await engine.resume(
|
|
first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."}
|
|
)
|
|
# Nothing has been submitted yet, so nothing is judged on that turn. Let the
|
|
# next reply, to the decomposer's approval prompt, be a text one.
|
|
result = await engine.resume(result.execution_id, {"reply": "Looks fine."})
|
|
|
|
assert result.state["last_reply"] is None
|
|
state, _, _ = decider.asked[-1]
|
|
assert state.endswith("The user's message: Looks fine.")
|
|
|
|
|
|
def test_a_quoted_question_is_removed_from_what_the_user_said():
|
|
from app.nodes.execution_nodes import _own_words
|
|
|
|
asked = ["Please confirm the draft. Shall I change the design?", "Which region?"]
|
|
reply = (
|
|
"Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n"
|
|
"Q: Which region?\nA: europe-west2"
|
|
)
|
|
|
|
assert _own_words(reply, asked) == "Yes, approved.\neurope-west2"
|
|
assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years."
|
|
|
|
|
|
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
|
|
from app.nodes.router import TurnView
|
|
from app.states.ledger import Ledger
|
|
|
|
ledger = Ledger()
|
|
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
|
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.05, 0.9),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.2, 0.5),
|
|
}
|
|
)
|
|
router = Router(decider, None, Thresholds())
|
|
view = TurnView(
|
|
execution_id="e1",
|
|
pre_approved=False,
|
|
ledger=ledger,
|
|
deliverables={},
|
|
pending_questions=[],
|
|
outcomes=[],
|
|
last_reply={
|
|
"kind": "reply",
|
|
"text": "Yes, approved.",
|
|
"asked": ["Shall I change the design?"],
|
|
},
|
|
)
|
|
from app.nodes.router import Readings
|
|
|
|
await router._read_message(view, Readings())
|
|
|
|
state, _, _ = decider.asked[0]
|
|
assert "replying to these questions from the agents:\n- Shall I change" in state
|
|
assert state.endswith("The user's message: Yes, approved.")
|
|
|
|
|
|
async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
|
tmp_path,
|
|
):
|
|
"""Whatever causes it, a run that only asks is stopped, not left to the turn limit."""
|
|
from app.nodes.execution_nodes import MAX_IDLE_ASKS
|
|
from app.nodes.router import TurnResult
|
|
|
|
class OnlyAsks:
|
|
"""An Orchestrator that asks the user something on every turn."""
|
|
|
|
async def turn(self, view, tools):
|
|
ask = next(tool for tool in tools if tool.name == "ask_user")
|
|
reply = await ask.handler(questions=["What now?"])
|
|
return TurnResult(reply.outcome)
|
|
|
|
engine = ExecutionEngine(
|
|
runtime=FakeAgentRuntime({}),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=OnlyAsks(),
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
asked = 1
|
|
while result.status == "awaiting_user" and asked < 20:
|
|
result = await engine.resume(result.execution_id, {"reply": "Carry on."})
|
|
asked += 1
|
|
|
|
assert result.status == "completed_with_warnings"
|
|
assert asked == MAX_IDLE_ASKS + 1
|
|
assert "asked" in result.state["warnings"][-1]
|
|
assert "no work done in between" in result.state["warnings"][-1]
|