"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook. No language model chooses an action here. Specialists are scripted; the Orchestrator is the Router, with a scripted decider standing in for Jev, or none at all. """ import json from langgraph.checkpoint.memory import InMemorySaver from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec from execution_engine.agent import ExecutionEngine from execution_engine.nodes.router import ( REPLY_WRITER, Router, RuntimeReplyWriter, Thresholds, ) from execution_engine.recorder import InMemoryRecorder from tests.reference_workflow.workflow import ( DEPENDENCY_GRAPH, SAMPLE_DATA, WORKFLOW, DeliverableKind, sample_submission, ) from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811 from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811 from tests.scoped import scoped_ledger K = DeliverableKind ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40), REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def once() -> dict: return { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } def validation(**changes) -> Call: """A validation submission whose verdicts differ from the all-pass sample.""" call = sample_submission(K.VALIDATION_RESULTS) verdicts = json.loads(SAMPLE_DATA["verdicts.json"]) verdicts.update(changes) return Call( call.tool, {**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}}, ) def failing(check: str = "K-1") -> Call: checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"] for item in checks: if item["check"] == check: item.update(verdict="fail", evidence="min instances is 1") return validation(checks=checks) def addition(reason: str) -> Call: return validation( proposed_additions=[ {"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason} ] ) def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options): runtime = FakeAgentRuntime(turns) router = Router( decider or NoDecider(), RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None, Thresholds(), ) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=router, **options, ) return engine, runtime def actions(result) -> list[str]: return [decision["action"] for decision in result.state["decisions"]] def reopened(result) -> list[tuple[str, str]]: return [ (event["subject"], event["detail"]) for event in result.state["ledger"]["events"] if event["type"] == "reopened" ] def approve(phase: str) -> dict: return {"approval": {"phase": phase, "decision": "approve"}} # --- the playbook on rules alone ------------------------------------------------------- async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path): engine, runtime = engine_for(tmp_path, once()) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert "orchestrator" not in {run.role for run in runtime.runs} assert actions(result) == [ "delegate", # the scope "request_approval", "delegate", # requirements "delegate", # technical decomposition "request_approval", "delegate", # product selection "delegate", # design recommendations "delegate", # Target architecture "request_approval", "delegate", # description and deployment guidance, together "request_approval", "delegate", # validation "proceed", "request_approval", "delegate", # solution guide "request_approval", "finish", ] groups = {decision["group"] for decision in result.state["decisions"]} assert groups == { "work is ready", "a Phase is complete", "validation decided", "packaging is approved", } async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path): engine, _ = engine_for(tmp_path, once()) result = await engine.start("Ingest webhooks", pre_approved=True) first = result.state["decisions"][0] assert first == { "turn": 1, "group": "work is ready", "action": "delegate", "judged": {}, } assert [d["turn"] for d in result.state["decisions"]] == sorted( d["turn"] for d in result.state["decisions"] ) async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings( tmp_path, ): turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) # The Specialist's question is relayed, answered automatically, and it stops # again; the router never invents work. assert result.status == "completed_with_warnings" # --- Approvals: the summary is written from data, and only data approves ---------------- async def test_the_approval_summary_is_assembled_from_the_data(tmp_path): engine, _ = engine_for(tmp_path, once()) first = await engine.start("Ingest webhooks") design = await engine.resume(first.execution_id, approve("requirements_discovery")) assert first.prompt["approval"]["summary"] == ( "The requirements_discovery Phase is ready: requirements, " "technical_decomposition submitted. 2 requirements." ) summary = design.prompt["approval"]["summary"] assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary assert design.prompt["approval"]["target_architecture"]["checks"] == 2 async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path): turns = { **once(), REPLY_WRITER: [ [Call("write_reply", {"text": "The scope, ready."})], [Call("write_reply", {"text": "Two requirements, ready."})], ], } engine, runtime = engine_for(tmp_path, turns, writer=True) first = await engine.start("Ingest webhooks") assert first.prompt["approval"]["summary"] == "Two requirements, ready." # The writer ran once for the scope and once for the requirements. writer_run = [run for run in runtime.runs if run.role == REPLY_WRITER][-1] assert writer_run.tools == ("write_reply",) assert writer_run.task.context["facts"]["deliverables"] == [ "requirements", "technical_decomposition", ] async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path): engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted first = await engine.start("Ingest webhooks") assert first.prompt["approval"]["summary"].startswith( "The requirements_discovery Phase is ready" ) async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path): decider = FakeDecider( { "asks_change": Answer(0.03, 0.94), "change_target": Answer("requirements", 0.5), "change_clarity": Answer(0.0, 0.9), } ) engine, _ = engine_for(tmp_path, once(), decider) first = await engine.start("Ingest webhooks") again = await engine.resume(first.execution_id, {"reply": "Looks good to me."}) assert again.prompt["approval"]["phase"] == "requirements_discovery" assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved" # --- validation: the code, the graph, or proceed ---------------------------------------- async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved( tmp_path, ): turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [failing("K-1")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, runtime = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [ ("deployment_guidance", "K-1 failed: min instances is 1") ] assert "revise_code" in actions(result) architect_runs = [ run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE) ] assert len(architect_runs) == 1 async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated( tmp_path, ): """The code's author found the Check names a setting the resource lacks. No validation is spent on it: the Target architecture is revised first.""" cannot = sample_submission(K.DEPLOYMENT_GUIDANCE) implementation = json.loads(SAMPLE_DATA["implementation.json"]) implementation[1] = {"check": "K-2", "unimplementable": "no such argument"} turns = once() turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2 turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [ [ Call( cannot.tool, { **cannot.args, "attachments": { **cannot.args["attachments"], "implementation.json": json.dumps(implementation), }, }, ) ], [sample_submission(K.DEPLOYMENT_GUIDANCE)], ] engine, runtime = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [ ( "target_architecture", ( "Check K-2 cannot be implemented as written (no such argument). " "Rewrite it so it can be met, or remove it with a reason. The same " "fault may be in other Checks: read every Check against each reason " "given here and correct every one it applies to." ), ) ] assert "the code cannot meet a Check" in { decision["group"] for decision in result.state["decisions"] } validations = [ run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS) ] assert len(validations) == 1 async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path): """The code was revised and the same Check failed again: the Check is suspect.""" turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3 turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [failing("K-1")], [failing("K-1")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" [first, second] = reopened(result) assert first[0] == "deployment_guidance" assert second[0] == "target_architecture" assert "K-1 failed in two validations in a row" in second[1] assert actions(result).count("revise_code") == 1 async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path): turns = once() for kind in ( K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE, ): turns[specialist(kind)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [addition("NFR-1 has no Check on timeouts")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" [(kind, reason)] = reopened(result) assert kind == "target_architecture" assert reason == ( "Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)." ) async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path): turns = once() turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [] async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path): turns = once() turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]] immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)}) engine, _ = engine_for(tmp_path, turns, immaterial) result = await engine.start("Ingest webhooks", pre_approved=True) # The rule alone would have revised the graph; the reading says it is not needed. assert reopened(result) == [] judged = next(d for d in result.state["decisions"] if d["action"] == "proceed") assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}} [(state, questions, scope)] = immaterial.asked assert "A timeout is set." in state and scope == result.execution_id assert questions["gap_material"].kind == "noul" async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path): turns = once() for kind in ( K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE, ): turns[specialist(kind)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [addition("NFR-1")], [sample_submission(K.VALIDATION_RESULTS)], ] decider = FakeDecider( { "gap_material": Answer(0.5, 0.0), # the grey zone "wants_revision": Answer(0.96, 0.92), } ) engine, _ = engine_for(tmp_path, turns, decider) result = await engine.start("Ingest webhooks") for phase in ( "requirements_discovery", "solution_architecture", "solution_implementation", ): result = await engine.resume(result.execution_id, approve(phase)) [question] = result.prompt["questions"] assert ( "found gaps in the Target architecture: A timeout is set." in (question["question"]) ) result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."}) assert reopened(result)[0][0] == "target_architecture" # Reopening the graph withdrew the design Approval, so it is asked for again. assert result.prompt["approval"]["phase"] == "solution_architecture" # --- the user's message: act, queue or close -------------------------------------------- def to_design_approval(engine): async def run(): first = await engine.start("Ingest webhooks") return await engine.resume( first.execution_id, approve("requirements_discovery") ) return run() def changes(phase: str, feedback: str) -> dict: return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}} async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path): turns = once() turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 decider = FakeDecider( { "change_target": Answer("design_recommendations", 0.88), "change_clarity": Answer(1.8, 0.8), } ) engine, _ = engine_for(tmp_path, turns, decider) design = await to_design_approval(engine) result = await engine.resume( design.execution_id, changes("solution_architecture", "Use customer-managed keys on the topic."), ) assert reopened(result) == [ ("design_recommendations", "Use customer-managed keys on the topic.") ] # Product selection was not touched; the design is asked for again once redone. assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0 assert result.prompt["approval"]["phase"] == "solution_architecture" [(state, questions, _)] = decider.asked assert "The user's message: Use customer-managed keys on the topic." in state assert questions["change_target"].options == ( "scope", "requirements", "technical_decomposition", "product_selection", "design_recommendations", "target_architecture", ) assert "asks_change" not in questions # a structured request is known to be one async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves( tmp_path, ): turns = once() turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 decider = FakeDecider( { "change_target": Answer("product_selection", 0.31), # not relied on "change_clarity": Answer(1.5, 0.7), } ) engine, _ = engine_for(tmp_path, turns, decider) design = await to_design_approval(engine) asked = await engine.resume( design.execution_id, changes("solution_architecture", "Tighten it up.") ) assert reopened(asked) == [] [question] = asked.prompt["questions"] assert question["question"].startswith("Do you want product_selection changed?") result = await engine.resume( asked.execution_id, {"reply": "No, change the target architecture: add a Check on timeouts."}, ) [(kind, reason)] = reopened(result) assert kind == "target_architecture" assert "Tighten it up." in reason and "add a Check on timeouts" in reason async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path): turns = once() turns[specialist(K.REQUIREMENTS)] = [ [sample_submission(K.REQUIREMENTS)], [sample_submission(K.REQUIREMENTS)], ] turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2 decider = FakeDecider( { "asks_change": Answer(0.93, 0.86), "change_target": Answer("requirements", 0.9), "change_clarity": Answer(1.9, 0.8), "already_agreed": Answer(0.03, 0.94), } ) engine, _ = engine_for(tmp_path, turns, decider) first = await engine.start("Ingest webhooks") result = await engine.resume( first.execution_id, {"reply": "Retention must be 7 years, not 30 days."} ) assert reopened(result) == [ ("requirements", "Retention must be 7 years, not 30 days.") ] async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named( tmp_path, ): turns = once() turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 engine, _ = engine_for(tmp_path, turns) design = await to_design_approval(engine) unnamed = await engine.resume( design.execution_id, changes("solution_architecture", "Make it cheaper.") ) assert reopened(unnamed) == [] assert unnamed.prompt["questions"][0]["question"].startswith( "Which Deliverable should change" ) named = await engine.resume( unnamed.execution_id, {"reply": "The design recommendations."} ) [(kind, reason)] = reopened(named) assert kind == "design_recommendations" assert reason == ( "Make it cheaper.\nThe user then said: The design recommendations." ) # --- permission is the Ledger's, whatever was decided ------------------------------------ async def test_past_the_revision_limit_the_user_decides(tmp_path): turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]] engine, _ = engine_for(tmp_path, turns, max_revisions=1) result = await engine.start("Ingest webhooks") for phase in ( "requirements_discovery", "solution_architecture", "solution_implementation", "solution_implementation", ): result = await engine.resume(result.execution_id, approve(phase)) assert len(reopened(result)) == 1 [question] = result.prompt["questions"] assert ( "deployment_guidance has reached its revision limit (1)" in (question["question"]) ) # --- the two faults of evaluation case 04 (2026-10-08) ----------------------------------- async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on( tmp_path, ): """The user is told once. Their reply is not read as a new request for changes, the Deliverable is not reopened again, and nothing is asked a second time.""" turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]] # A decider that would call any reply a clear request to change the code. decider = FakeDecider( { "asks_change": Answer(0.95, 0.9), "change_target": Answer("deployment_guidance", 0.95), "change_clarity": Answer(1.9, 0.8), } ) engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1) result = await engine.start("Ingest webhooks") for phase in ( "requirements_discovery", "solution_architecture", "solution_implementation", "solution_implementation", ): result = await engine.resume(result.execution_id, approve(phase)) [question] = result.prompt["questions"] assert "I will carry on with it as it stands" in question["question"] result = await engine.resume( result.execution_id, {"reply": "Please change the code again."} ) # It moves on to the Approval of validation; the code was reopened once only. assert result.prompt["approval"]["phase"] == "solution_validation" assert len(reopened(result)) == 1 assert result.state["revision_limit_asked"] == ["deployment_guidance"] assert decider.asked == [] # the reply was never put to the decider async def test_only_the_users_own_words_are_read_as_their_message(tmp_path): """A reply that quotes our question back must not be judged on our words.""" question = "Please confirm the requirements draft. Shall I change anything?" turns = once() turns[specialist(K.REQUIREMENTS)] = [ [Call("needs_input", {"questions": [question]})], [sample_submission(K.REQUIREMENTS)], ] decider = FakeDecider( { "asks_change": Answer(0.05, 0.9), "change_target": Answer("requirements", 0.5), "change_clarity": Answer(0.2, 0.5), } ) engine, _ = engine_for(tmp_path, turns, decider) first = await engine.start("Ingest webhooks") result = await engine.resume( first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."} ) # Nothing has been submitted yet, so nothing is judged on that turn. Let the # next reply, to the decomposer's approval prompt, be a text one. result = await engine.resume(result.execution_id, {"reply": "Looks fine."}) assert result.state["last_reply"] is None state, _, _ = decider.asked[-1] assert state.endswith("The user's message: Looks fine.") def test_a_quoted_question_is_removed_from_what_the_user_said(): from execution_engine.nodes.execution_nodes import _own_words asked = ["Please confirm the draft. Shall I change the design?", "Which region?"] reply = ( "Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n" "Q: Which region?\nA: europe-west2" ) assert _own_words(reply, asked) == "Yes, approved.\neurope-west2" assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years." async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path): from execution_engine.nodes.router import TurnView ledger = scoped_ledger() ledger.start([K.REQUIREMENTS], actor="orchestrator") ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst") decider = FakeDecider( { "asks_change": Answer(0.05, 0.9), "change_target": Answer("requirements", 0.5), "change_clarity": Answer(0.2, 0.5), } ) router = Router(decider, None, Thresholds()) view = TurnView( execution_id="e1", pre_approved=False, ledger=ledger, deliverables={}, pending_questions=[], outcomes=[], last_reply={ "kind": "reply", "text": "Yes, approved.", "asked": ["Shall I change the design?"], }, ) from execution_engine.nodes.router import Readings await router._read_message(view, Readings()) state, _, _ = decider.asked[0] assert "replying to these questions from the agents:\n- Shall I change" in state assert state.endswith("The user's message: Yes, approved.") async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run( tmp_path, ): """Whatever causes it, a run that only asks is stopped, not left to the turn limit.""" from execution_engine.nodes.execution_nodes import MAX_IDLE_ASKS from execution_engine.nodes.router import TurnResult class OnlyAsks: """An Orchestrator that asks the user something on every turn.""" async def turn(self, view, tools): ask = next(tool for tool in tools if tool.name == "ask_user") reply = await ask.handler(questions=["What now?"]) return TurnResult(reply.outcome) engine = ExecutionEngine( workflow=WORKFLOW, runtime=FakeAgentRuntime({}), roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=OnlyAsks(), ) result = await engine.start("Ingest webhooks") asked = 1 while result.status == "awaiting_user" and asked < 20: result = await engine.resume(result.execution_id, {"reply": "Carry on."}) asked += 1 assert result.status == "completed_with_warnings" assert asked == MAX_IDLE_ASKS + 1 assert "asked" in result.state["warnings"][-1] assert "no work done in between" in result.state["warnings"][-1] async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change( tmp_path, ): """Evaluation case 04: a requirement said again in reply to a later question.""" from execution_engine.nodes.router import Readings, TurnView ledger = scoped_ledger() ledger.start([K.REQUIREMENTS], actor="orchestrator") ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst") view = TurnView( execution_id="e1", pre_approved=False, ledger=ledger, deliverables={ "requirements": { "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} } }, pending_questions=[], outcomes=[], last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []}, ) asks = { "asks_change": Answer(0.9, 0.8), "change_target": Answer("requirements", 0.95), "change_clarity": Answer(1.9, 0.8), } restated, changed = Readings(), Readings() decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)}) await Router(decider, None, Thresholds())._read_message(view, restated) await Router( FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds() )._read_message(view, changed) assert restated.change == "close" assert changed.change == "act" state, questions, _ = decider.asked[0] assert "The agreed requirements:" in state and "NFR-1" in state assert "already_agreed" in questions