Files
gcp_solution_architecture_a…/tests/execution/test_router.py
Jonathan Boniface 5bb22d984e
Some checks failed
validation / verify (push) Failing after 8s
fixed multile issues
2026-10-07 17:33:36 +01:00

599 lines
20 KiB
Python

"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
No language model chooses an action here. Specialists are scripted; the Orchestrator
is the Router, with a scripted decider standing in for Jev, or none at all.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import (
SAMPLE_DATA,
Call,
FakeAgentRuntime,
FakeDecider,
sample_submission,
)
from app.adk.runtime import Answer, NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import (
REPLY_WRITER,
Router,
RuntimeReplyWriter,
Thresholds,
)
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
K = DeliverableKind
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
def validation(**changes) -> Call:
"""A validation submission whose verdicts differ from the all-pass sample."""
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts.update(changes)
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def failing(check: str = "K-1") -> Call:
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
for item in checks:
if item["check"] == check:
item.update(verdict="fail", evidence="min instances is 1")
return validation(checks=checks)
def addition(reason: str) -> Call:
return validation(
proposed_additions=[
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
]
)
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
runtime = FakeAgentRuntime(turns)
router = Router(
decider or NoDecider(),
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
Thresholds(),
)
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=router,
**options,
)
return engine, runtime
def actions(result) -> list[str]:
return [decision["action"] for decision in result.state["decisions"]]
def reopened(result) -> list[tuple[str, str]]:
return [
(event["subject"], event["detail"])
for event in result.state["ledger"]["events"]
if event["type"] == "reopened"
]
def approve(phase: str) -> dict:
return {"approval": {"phase": phase, "decision": "approve"}}
# --- the playbook on rules alone -------------------------------------------------------
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
engine, runtime = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert "orchestrator" not in {run.role for run in runtime.runs}
assert actions(result) == [
"delegate", # requirements
"delegate", # technical decomposition
"request_approval",
"delegate", # product selection
"delegate", # design recommendations
"delegate", # Target architecture
"request_approval",
"delegate", # description and deployment guidance, together
"request_approval",
"delegate", # validation
"proceed",
"request_approval",
"delegate", # solution guide
"request_approval",
"finish",
]
groups = {decision["group"] for decision in result.state["decisions"]}
assert groups == {
"work is ready",
"a Phase is complete",
"validation decided",
"packaging is approved",
}
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
engine, _ = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
first = result.state["decisions"][0]
assert first == {
"turn": 1,
"group": "work is ready",
"action": "delegate",
"judged": {},
}
assert [d["turn"] for d in result.state["decisions"]] == sorted(
d["turn"] for d in result.state["decisions"]
)
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
tmp_path,
):
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The Specialist's question is relayed, answered automatically, and it stops
# again; the router never invents work.
assert result.status == "completed_with_warnings"
# --- Approvals: the summary is written from data, and only data approves ----------------
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
engine, _ = engine_for(tmp_path, once())
first = await engine.start("Ingest webhooks")
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
assert first.prompt["approval"]["summary"] == (
"The requirements_discovery Phase is ready: requirements, "
"technical_decomposition submitted. 2 requirements."
)
summary = design.prompt["approval"]["summary"]
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
turns = {
**once(),
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
}
engine, runtime = engine_for(tmp_path, turns, writer=True)
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
assert writer_run.tools == ("write_reply",)
assert writer_run.task.context["facts"]["deliverables"] == [
"requirements",
"technical_decomposition",
]
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"].startswith(
"The requirements_discovery Phase is ready"
)
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
decider = FakeDecider(
{
"asks_change": Answer(0.03, 0.94),
"change_target": Answer("requirements", 0.5),
"change_clarity": Answer(0.0, 0.9),
}
)
engine, _ = engine_for(tmp_path, once(), decider)
first = await engine.start("Ingest webhooks")
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
assert again.prompt["approval"]["phase"] == "requirements_discovery"
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
# --- validation: the code, the graph, or proceed ----------------------------------------
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
tmp_path,
):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
("deployment_guidance", "K-1 failed: min instances is 1")
]
assert "revise_code" in actions(result)
architect_runs = [
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
]
assert len(architect_runs) == 1
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
tmp_path,
):
"""The code's author found the Check names a setting the resource lacks. No
validation is spent on it: the Target architecture is revised first."""
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
implementation = json.loads(SAMPLE_DATA["implementation.json"])
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
[
Call(
cannot.tool,
{
**cannot.args,
"attachments": {
**cannot.args["attachments"],
"implementation.json": json.dumps(implementation),
},
},
)
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
(
"target_architecture",
(
"Check K-2 cannot be implemented as written (no such argument). "
"Rewrite it so it can be met, or remove it with a reason."
),
)
]
assert "the code cannot meet a Check" in {
decision["group"] for decision in result.state["decisions"]
}
validations = [
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
]
assert len(validations) == 1
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
"""The code was revised and the same Check failed again: the Check is suspect."""
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[first, second] = reopened(result)
assert first[0] == "deployment_guidance"
assert second[0] == "target_architecture"
assert "K-1 failed in two validations in a row" in second[1]
assert actions(result).count("revise_code") == 1
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1 has no Check on timeouts")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert reason == (
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
)
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == []
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
engine, _ = engine_for(tmp_path, turns, immaterial)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The rule alone would have revised the graph; the reading says it is not needed.
assert reopened(result) == []
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
[(state, questions, scope)] = immaterial.asked
assert "A timeout is set." in state and scope == result.execution_id
assert questions["gap_material"].kind == "noul"
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
decider = FakeDecider(
{
"gap_material": Answer(0.5, 0.0), # the grey zone
"wants_revision": Answer(0.96, 0.92),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
[question] = result.prompt["questions"]
assert (
"found gaps in the Target architecture: A timeout is set."
in (question["question"])
)
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
assert reopened(result)[0][0] == "target_architecture"
# Reopening the graph withdrew the design Approval, so it is asked for again.
assert result.prompt["approval"]["phase"] == "solution_architecture"
# --- the user's message: act, queue or close --------------------------------------------
def to_design_approval(engine):
async def run():
first = await engine.start("Ingest webhooks")
return await engine.resume(
first.execution_id, approve("requirements_discovery")
)
return run()
def changes(phase: str, feedback: str) -> dict:
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("design_recommendations", 0.88),
"change_clarity": Answer(1.8, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
result = await engine.resume(
design.execution_id,
changes("solution_architecture", "Use customer-managed keys on the topic."),
)
assert reopened(result) == [
("design_recommendations", "Use customer-managed keys on the topic.")
]
# Product selection was not touched; the design is asked for again once redone.
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
assert result.prompt["approval"]["phase"] == "solution_architecture"
[(state, questions, _)] = decider.asked
assert "The user's message: Use customer-managed keys on the topic." in state
assert questions["change_target"].options == (
"requirements",
"technical_decomposition",
"product_selection",
"design_recommendations",
"target_architecture",
)
assert "asks_change" not in questions # a structured request is known to be one
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
tmp_path,
):
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("product_selection", 0.31), # not relied on
"change_clarity": Answer(1.5, 0.7),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
asked = await engine.resume(
design.execution_id, changes("solution_architecture", "Tighten it up.")
)
assert reopened(asked) == []
[question] = asked.prompt["questions"]
assert question["question"].startswith("Do you want product_selection changed?")
result = await engine.resume(
asked.execution_id,
{"reply": "No, change the target architecture: add a Check on timeouts."},
)
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
turns = once()
turns[specialist(K.REQUIREMENTS)] = [
[sample_submission(K.REQUIREMENTS)],
[sample_submission(K.REQUIREMENTS)],
]
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
decider = FakeDecider(
{
"asks_change": Answer(0.93, 0.86),
"change_target": Answer("requirements", 0.9),
"change_clarity": Answer(1.9, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
first = await engine.start("Ingest webhooks")
result = await engine.resume(
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
)
assert reopened(result) == [
("requirements", "Retention must be 7 years, not 30 days.")
]
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
tmp_path,
):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
engine, _ = engine_for(tmp_path, turns)
design = await to_design_approval(engine)
unnamed = await engine.resume(
design.execution_id, changes("solution_architecture", "Make it cheaper.")
)
assert reopened(unnamed) == []
assert unnamed.prompt["questions"][0]["question"].startswith(
"Which Deliverable should change"
)
named = await engine.resume(
unnamed.execution_id, {"reply": "The design recommendations."}
)
[(kind, reason)] = reopened(named)
assert kind == "design_recommendations"
assert reason == (
"Make it cheaper.\nThe user then said: The design recommendations."
)
# --- permission is the Ledger's, whatever was decided ------------------------------------
async def test_past_the_revision_limit_the_user_decides(tmp_path):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
assert len(reopened(result)) == 1
[question] = result.prompt["questions"]
assert (
"deployment_guidance has reached its revision limit (1)"
in (question["question"])
)