"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook. No language model chooses an action here. Specialists are scripted; the Orchestrator is the Router, with a scripted decider standing in for Jev, or none at all. """ import json from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import ( SAMPLE_DATA, Call, FakeAgentRuntime, FakeDecider, sample_submission, ) from app.adk.runtime import Answer, NoDecider, RoleSpec from app.agent import ExecutionEngine from app.nodes.router import ( REPLY_WRITER, Router, RuntimeReplyWriter, Thresholds, ) from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind K = DeliverableKind ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40), REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def once() -> dict: return { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } def validation(**changes) -> Call: """A validation submission whose verdicts differ from the all-pass sample.""" call = sample_submission(K.VALIDATION_RESULTS) verdicts = json.loads(SAMPLE_DATA["verdicts.json"]) verdicts.update(changes) return Call( call.tool, {**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}}, ) def failing(check: str = "K-1") -> Call: checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"] for item in checks: if item["check"] == check: item.update(verdict="fail", evidence="min instances is 1") return validation(checks=checks) def addition(reason: str) -> Call: return validation( proposed_additions=[ {"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason} ] ) def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options): runtime = FakeAgentRuntime(turns) router = Router( decider or NoDecider(), RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None, Thresholds(), ) engine = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=router, **options, ) return engine, runtime def actions(result) -> list[str]: return [decision["action"] for decision in result.state["decisions"]] def reopened(result) -> list[tuple[str, str]]: return [ (event["subject"], event["detail"]) for event in result.state["ledger"]["events"] if event["type"] == "reopened" ] def approve(phase: str) -> dict: return {"approval": {"phase": phase, "decision": "approve"}} # --- the playbook on rules alone ------------------------------------------------------- async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path): engine, runtime = engine_for(tmp_path, once()) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert "orchestrator" not in {run.role for run in runtime.runs} assert actions(result) == [ "delegate", # requirements "delegate", # technical decomposition "request_approval", "delegate", # product selection "delegate", # design recommendations "delegate", # Target architecture "request_approval", "delegate", # description and deployment guidance, together "request_approval", "delegate", # validation "proceed", "request_approval", "delegate", # solution guide "request_approval", "finish", ] groups = {decision["group"] for decision in result.state["decisions"]} assert groups == { "work is ready", "a Phase is complete", "validation decided", "packaging is approved", } async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path): engine, _ = engine_for(tmp_path, once()) result = await engine.start("Ingest webhooks", pre_approved=True) first = result.state["decisions"][0] assert first == { "turn": 1, "group": "work is ready", "action": "delegate", "judged": {}, } assert [d["turn"] for d in result.state["decisions"]] == sorted( d["turn"] for d in result.state["decisions"] ) async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings( tmp_path, ): turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) # The Specialist's question is relayed, answered automatically, and it stops # again; the router never invents work. assert result.status == "completed_with_warnings" # --- Approvals: the summary is written from data, and only data approves ---------------- async def test_the_approval_summary_is_assembled_from_the_data(tmp_path): engine, _ = engine_for(tmp_path, once()) first = await engine.start("Ingest webhooks") design = await engine.resume(first.execution_id, approve("requirements_discovery")) assert first.prompt["approval"]["summary"] == ( "The requirements_discovery Phase is ready: requirements, " "technical_decomposition submitted. 2 requirements." ) summary = design.prompt["approval"]["summary"] assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary assert design.prompt["approval"]["target_architecture"]["checks"] == 2 async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path): turns = { **once(), REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]], } engine, runtime = engine_for(tmp_path, turns, writer=True) first = await engine.start("Ingest webhooks") assert first.prompt["approval"]["summary"] == "Two requirements, ready." writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER) assert writer_run.tools == ("write_reply",) assert writer_run.task.context["facts"]["deliverables"] == [ "requirements", "technical_decomposition", ] async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path): engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted first = await engine.start("Ingest webhooks") assert first.prompt["approval"]["summary"].startswith( "The requirements_discovery Phase is ready" ) async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path): decider = FakeDecider( { "asks_change": Answer(0.03, 0.94), "change_target": Answer("requirements", 0.5), "change_clarity": Answer(0.0, 0.9), } ) engine, _ = engine_for(tmp_path, once(), decider) first = await engine.start("Ingest webhooks") again = await engine.resume(first.execution_id, {"reply": "Looks good to me."}) assert again.prompt["approval"]["phase"] == "requirements_discovery" assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved" # --- validation: the code, the graph, or proceed ---------------------------------------- async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved( tmp_path, ): turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [failing("K-1")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, runtime = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [ ("deployment_guidance", "K-1 failed: min instances is 1") ] assert "revise_code" in actions(result) architect_runs = [ run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE) ] assert len(architect_runs) == 1 async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated( tmp_path, ): """The code's author found the Check names a setting the resource lacks. No validation is spent on it: the Target architecture is revised first.""" cannot = sample_submission(K.DEPLOYMENT_GUIDANCE) implementation = json.loads(SAMPLE_DATA["implementation.json"]) implementation[1] = {"check": "K-2", "unimplementable": "no such argument"} turns = once() turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2 turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [ [ Call( cannot.tool, { **cannot.args, "attachments": { **cannot.args["attachments"], "implementation.json": json.dumps(implementation), }, }, ) ], [sample_submission(K.DEPLOYMENT_GUIDANCE)], ] engine, runtime = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [ ( "target_architecture", ( "Check K-2 cannot be implemented as written (no such argument). " "Rewrite it so it can be met, or remove it with a reason." ), ) ] assert "the code cannot meet a Check" in { decision["group"] for decision in result.state["decisions"] } validations = [ run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS) ] assert len(validations) == 1 async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path): """The code was revised and the same Check failed again: the Check is suspect.""" turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3 turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [failing("K-1")], [failing("K-1")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" [first, second] = reopened(result) assert first[0] == "deployment_guidance" assert second[0] == "target_architecture" assert "K-1 failed in two validations in a row" in second[1] assert actions(result).count("revise_code") == 1 async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path): turns = once() for kind in ( K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE, ): turns[specialist(kind)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [addition("NFR-1 has no Check on timeouts")], [sample_submission(K.VALIDATION_RESULTS)], ] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" [(kind, reason)] = reopened(result) assert kind == "target_architecture" assert reason == ( "Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)." ) async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path): turns = once() turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]] engine, _ = engine_for(tmp_path, turns) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert reopened(result) == [] async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path): turns = once() turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]] immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)}) engine, _ = engine_for(tmp_path, turns, immaterial) result = await engine.start("Ingest webhooks", pre_approved=True) # The rule alone would have revised the graph; the reading says it is not needed. assert reopened(result) == [] judged = next(d for d in result.state["decisions"] if d["action"] == "proceed") assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}} [(state, questions, scope)] = immaterial.asked assert "A timeout is set." in state and scope == result.execution_id assert questions["gap_material"].kind == "noul" async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path): turns = once() for kind in ( K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE, ): turns[specialist(kind)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [ [addition("NFR-1")], [sample_submission(K.VALIDATION_RESULTS)], ] decider = FakeDecider( { "gap_material": Answer(0.5, 0.0), # the grey zone "wants_revision": Answer(0.96, 0.92), } ) engine, _ = engine_for(tmp_path, turns, decider) result = await engine.start("Ingest webhooks") for phase in ( "requirements_discovery", "solution_architecture", "solution_implementation", ): result = await engine.resume(result.execution_id, approve(phase)) [question] = result.prompt["questions"] assert ( "found gaps in the Target architecture: A timeout is set." in (question["question"]) ) result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."}) assert reopened(result)[0][0] == "target_architecture" # Reopening the graph withdrew the design Approval, so it is asked for again. assert result.prompt["approval"]["phase"] == "solution_architecture" # --- the user's message: act, queue or close -------------------------------------------- def to_design_approval(engine): async def run(): first = await engine.start("Ingest webhooks") return await engine.resume( first.execution_id, approve("requirements_discovery") ) return run() def changes(phase: str, feedback: str) -> dict: return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}} async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path): turns = once() turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 decider = FakeDecider( { "change_target": Answer("design_recommendations", 0.88), "change_clarity": Answer(1.8, 0.8), } ) engine, _ = engine_for(tmp_path, turns, decider) design = await to_design_approval(engine) result = await engine.resume( design.execution_id, changes("solution_architecture", "Use customer-managed keys on the topic."), ) assert reopened(result) == [ ("design_recommendations", "Use customer-managed keys on the topic.") ] # Product selection was not touched; the design is asked for again once redone. assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0 assert result.prompt["approval"]["phase"] == "solution_architecture" [(state, questions, _)] = decider.asked assert "The user's message: Use customer-managed keys on the topic." in state assert questions["change_target"].options == ( "requirements", "technical_decomposition", "product_selection", "design_recommendations", "target_architecture", ) assert "asks_change" not in questions # a structured request is known to be one async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves( tmp_path, ): turns = once() turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 decider = FakeDecider( { "change_target": Answer("product_selection", 0.31), # not relied on "change_clarity": Answer(1.5, 0.7), } ) engine, _ = engine_for(tmp_path, turns, decider) design = await to_design_approval(engine) asked = await engine.resume( design.execution_id, changes("solution_architecture", "Tighten it up.") ) assert reopened(asked) == [] [question] = asked.prompt["questions"] assert question["question"].startswith("Do you want product_selection changed?") result = await engine.resume( asked.execution_id, {"reply": "No, change the target architecture: add a Check on timeouts."}, ) [(kind, reason)] = reopened(result) assert kind == "target_architecture" assert "Tighten it up." in reason and "add a Check on timeouts" in reason async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path): turns = once() turns[specialist(K.REQUIREMENTS)] = [ [sample_submission(K.REQUIREMENTS)], [sample_submission(K.REQUIREMENTS)], ] turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2 decider = FakeDecider( { "asks_change": Answer(0.93, 0.86), "change_target": Answer("requirements", 0.9), "change_clarity": Answer(1.9, 0.8), } ) engine, _ = engine_for(tmp_path, turns, decider) first = await engine.start("Ingest webhooks") result = await engine.resume( first.execution_id, {"reply": "Retention must be 7 years, not 30 days."} ) assert reopened(result) == [ ("requirements", "Retention must be 7 years, not 30 days.") ] async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named( tmp_path, ): turns = once() turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2 turns[specialist(K.TARGET_ARCHITECTURE)] *= 2 engine, _ = engine_for(tmp_path, turns) design = await to_design_approval(engine) unnamed = await engine.resume( design.execution_id, changes("solution_architecture", "Make it cheaper.") ) assert reopened(unnamed) == [] assert unnamed.prompt["questions"][0]["question"].startswith( "Which Deliverable should change" ) named = await engine.resume( unnamed.execution_id, {"reply": "The design recommendations."} ) [(kind, reason)] = reopened(named) assert kind == "design_recommendations" assert reason == ( "Make it cheaper.\nThe user then said: The design recommendations." ) # --- permission is the Ledger's, whatever was decided ------------------------------------ async def test_past_the_revision_limit_the_user_decides(tmp_path): turns = once() turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2 turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]] engine, _ = engine_for(tmp_path, turns, max_revisions=1) result = await engine.start("Ingest webhooks") for phase in ( "requirements_discovery", "solution_architecture", "solution_implementation", "solution_implementation", ): result = await engine.resume(result.execution_id, approve(phase)) assert len(reopened(result)) == 1 [question] = result.prompt["questions"] assert ( "deployment_guidance has reached its revision limit (1)" in (question["question"]) )