Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
802 lines
27 KiB
Python
802 lines
27 KiB
Python
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
|
|
|
|
No language model chooses an action here. Specialists are scripted; the Orchestrator
|
|
is the Router, with a scripted decider standing in for Jev, or none at all.
|
|
"""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import (
|
|
SAMPLE_DATA,
|
|
Call,
|
|
FakeAgentRuntime,
|
|
FakeDecider,
|
|
sample_submission,
|
|
)
|
|
from app.adk.runtime import Answer, NoDecider, RoleSpec
|
|
from app.agent import ExecutionEngine
|
|
from app.nodes.router import (
|
|
REPLY_WRITER,
|
|
Router,
|
|
RuntimeReplyWriter,
|
|
Thresholds,
|
|
)
|
|
from app.recorder import InMemoryRecorder
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
K = DeliverableKind
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
|
|
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def once() -> dict:
|
|
return {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
|
|
|
|
def validation(**changes) -> Call:
|
|
"""A validation submission whose verdicts differ from the all-pass sample."""
|
|
call = sample_submission(K.VALIDATION_RESULTS)
|
|
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
verdicts.update(changes)
|
|
return Call(
|
|
call.tool,
|
|
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
|
)
|
|
|
|
|
|
def failing(check: str = "K-1") -> Call:
|
|
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
|
|
for item in checks:
|
|
if item["check"] == check:
|
|
item.update(verdict="fail", evidence="min instances is 1")
|
|
return validation(checks=checks)
|
|
|
|
|
|
def addition(reason: str) -> Call:
|
|
return validation(
|
|
proposed_additions=[
|
|
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
|
|
]
|
|
)
|
|
|
|
|
|
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
|
|
runtime = FakeAgentRuntime(turns)
|
|
router = Router(
|
|
decider or NoDecider(),
|
|
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
|
|
Thresholds(),
|
|
)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=router,
|
|
**options,
|
|
)
|
|
return engine, runtime
|
|
|
|
|
|
def actions(result) -> list[str]:
|
|
return [decision["action"] for decision in result.state["decisions"]]
|
|
|
|
|
|
def reopened(result) -> list[tuple[str, str]]:
|
|
return [
|
|
(event["subject"], event["detail"])
|
|
for event in result.state["ledger"]["events"]
|
|
if event["type"] == "reopened"
|
|
]
|
|
|
|
|
|
def approve(phase: str) -> dict:
|
|
return {"approval": {"phase": phase, "decision": "approve"}}
|
|
|
|
|
|
# --- the playbook on rules alone -------------------------------------------------------
|
|
|
|
|
|
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
|
|
engine, runtime = engine_for(tmp_path, once())
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert "orchestrator" not in {run.role for run in runtime.runs}
|
|
assert actions(result) == [
|
|
"delegate", # requirements
|
|
"delegate", # technical decomposition
|
|
"request_approval",
|
|
"delegate", # product selection
|
|
"delegate", # design recommendations
|
|
"delegate", # Target architecture
|
|
"request_approval",
|
|
"delegate", # description and deployment guidance, together
|
|
"request_approval",
|
|
"delegate", # validation
|
|
"proceed",
|
|
"request_approval",
|
|
"delegate", # solution guide
|
|
"request_approval",
|
|
"finish",
|
|
]
|
|
groups = {decision["group"] for decision in result.state["decisions"]}
|
|
assert groups == {
|
|
"work is ready",
|
|
"a Phase is complete",
|
|
"validation decided",
|
|
"packaging is approved",
|
|
}
|
|
|
|
|
|
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once())
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
first = result.state["decisions"][0]
|
|
assert first == {
|
|
"turn": 1,
|
|
"group": "work is ready",
|
|
"action": "delegate",
|
|
"judged": {},
|
|
}
|
|
assert [d["turn"] for d in result.state["decisions"]] == sorted(
|
|
d["turn"] for d in result.state["decisions"]
|
|
)
|
|
|
|
|
|
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
|
|
tmp_path,
|
|
):
|
|
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
|
|
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
# The Specialist's question is relayed, answered automatically, and it stops
|
|
# again; the router never invents work.
|
|
assert result.status == "completed_with_warnings"
|
|
|
|
|
|
# --- Approvals: the summary is written from data, and only data approves ----------------
|
|
|
|
|
|
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once())
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
|
|
|
|
assert first.prompt["approval"]["summary"] == (
|
|
"The requirements_discovery Phase is ready: requirements, "
|
|
"technical_decomposition submitted. 2 requirements."
|
|
)
|
|
summary = design.prompt["approval"]["summary"]
|
|
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
|
|
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
|
|
|
|
|
|
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
|
|
turns = {
|
|
**once(),
|
|
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
|
|
}
|
|
engine, runtime = engine_for(tmp_path, turns, writer=True)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
|
|
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
|
|
assert writer_run.tools == ("write_reply",)
|
|
assert writer_run.task.context["facts"]["deliverables"] == [
|
|
"requirements",
|
|
"technical_decomposition",
|
|
]
|
|
|
|
|
|
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
|
|
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
assert first.prompt["approval"]["summary"].startswith(
|
|
"The requirements_discovery Phase is ready"
|
|
)
|
|
|
|
|
|
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.03, 0.94),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.0, 0.9),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, once(), decider)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
|
|
|
|
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
|
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
|
|
|
|
|
|
# --- validation: the code, the graph, or proceed ----------------------------------------
|
|
|
|
|
|
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[failing("K-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, runtime = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == [
|
|
("deployment_guidance", "K-1 failed: min instances is 1")
|
|
]
|
|
assert "revise_code" in actions(result)
|
|
architect_runs = [
|
|
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
|
|
]
|
|
assert len(architect_runs) == 1
|
|
|
|
|
|
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
|
|
tmp_path,
|
|
):
|
|
"""The code's author found the Check names a setting the resource lacks. No
|
|
validation is spent on it: the Target architecture is revised first."""
|
|
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
|
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
|
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
|
|
turns = once()
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
|
[
|
|
Call(
|
|
cannot.tool,
|
|
{
|
|
**cannot.args,
|
|
"attachments": {
|
|
**cannot.args["attachments"],
|
|
"implementation.json": json.dumps(implementation),
|
|
},
|
|
},
|
|
)
|
|
],
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
]
|
|
engine, runtime = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == [
|
|
(
|
|
"target_architecture",
|
|
(
|
|
"Check K-2 cannot be implemented as written (no such argument). "
|
|
"Rewrite it so it can be met, or remove it with a reason."
|
|
),
|
|
)
|
|
]
|
|
assert "the code cannot meet a Check" in {
|
|
decision["group"] for decision in result.state["decisions"]
|
|
}
|
|
validations = [
|
|
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
|
|
]
|
|
assert len(validations) == 1
|
|
|
|
|
|
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
|
|
"""The code was revised and the same Check failed again: the Check is suspect."""
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
|
|
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[failing("K-1")],
|
|
[failing("K-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
[first, second] = reopened(result)
|
|
assert first[0] == "deployment_guidance"
|
|
assert second[0] == "target_architecture"
|
|
assert "K-1 failed in two validations in a row" in second[1]
|
|
assert actions(result).count("revise_code") == 1
|
|
|
|
|
|
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
|
|
turns = once()
|
|
for kind in (
|
|
K.TARGET_ARCHITECTURE,
|
|
K.ARCHITECTURE_DESCRIPTION,
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
):
|
|
turns[specialist(kind)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[addition("NFR-1 has no Check on timeouts")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
[(kind, reason)] = reopened(result)
|
|
assert kind == "target_architecture"
|
|
assert reason == (
|
|
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
|
|
)
|
|
|
|
|
|
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert reopened(result) == []
|
|
|
|
|
|
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
|
|
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
|
|
engine, _ = engine_for(tmp_path, turns, immaterial)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
# The rule alone would have revised the graph; the reading says it is not needed.
|
|
assert reopened(result) == []
|
|
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
|
|
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
|
|
[(state, questions, scope)] = immaterial.asked
|
|
assert "A timeout is set." in state and scope == result.execution_id
|
|
assert questions["gap_material"].kind == "noul"
|
|
|
|
|
|
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
|
|
turns = once()
|
|
for kind in (
|
|
K.TARGET_ARCHITECTURE,
|
|
K.ARCHITECTURE_DESCRIPTION,
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
):
|
|
turns[specialist(kind)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [
|
|
[addition("NFR-1")],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
decider = FakeDecider(
|
|
{
|
|
"gap_material": Answer(0.5, 0.0), # the grey zone
|
|
"wants_revision": Answer(0.96, 0.92),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
|
|
[question] = result.prompt["questions"]
|
|
assert (
|
|
"found gaps in the Target architecture: A timeout is set."
|
|
in (question["question"])
|
|
)
|
|
|
|
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
|
|
|
|
assert reopened(result)[0][0] == "target_architecture"
|
|
# Reopening the graph withdrew the design Approval, so it is asked for again.
|
|
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
|
|
|
|
|
# --- the user's message: act, queue or close --------------------------------------------
|
|
|
|
|
|
def to_design_approval(engine):
|
|
async def run():
|
|
first = await engine.start("Ingest webhooks")
|
|
return await engine.resume(
|
|
first.execution_id, approve("requirements_discovery")
|
|
)
|
|
|
|
return run()
|
|
|
|
|
|
def changes(phase: str, feedback: str) -> dict:
|
|
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
|
|
|
|
|
|
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"change_target": Answer("design_recommendations", 0.88),
|
|
"change_clarity": Answer(1.8, 0.8),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
design = await to_design_approval(engine)
|
|
result = await engine.resume(
|
|
design.execution_id,
|
|
changes("solution_architecture", "Use customer-managed keys on the topic."),
|
|
)
|
|
|
|
assert reopened(result) == [
|
|
("design_recommendations", "Use customer-managed keys on the topic.")
|
|
]
|
|
# Product selection was not touched; the design is asked for again once redone.
|
|
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
|
|
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
|
[(state, questions, _)] = decider.asked
|
|
assert "The user's message: Use customer-managed keys on the topic." in state
|
|
assert questions["change_target"].options == (
|
|
"requirements",
|
|
"technical_decomposition",
|
|
"product_selection",
|
|
"design_recommendations",
|
|
"target_architecture",
|
|
)
|
|
assert "asks_change" not in questions # a structured request is known to be one
|
|
|
|
|
|
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"change_target": Answer("product_selection", 0.31), # not relied on
|
|
"change_clarity": Answer(1.5, 0.7),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
design = await to_design_approval(engine)
|
|
asked = await engine.resume(
|
|
design.execution_id, changes("solution_architecture", "Tighten it up.")
|
|
)
|
|
|
|
assert reopened(asked) == []
|
|
[question] = asked.prompt["questions"]
|
|
assert question["question"].startswith("Do you want product_selection changed?")
|
|
|
|
result = await engine.resume(
|
|
asked.execution_id,
|
|
{"reply": "No, change the target architecture: add a Check on timeouts."},
|
|
)
|
|
|
|
[(kind, reason)] = reopened(result)
|
|
assert kind == "target_architecture"
|
|
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
|
|
|
|
|
|
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.REQUIREMENTS)] = [
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
]
|
|
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.93, 0.86),
|
|
"change_target": Answer("requirements", 0.9),
|
|
"change_clarity": Answer(1.9, 0.8),
|
|
"already_agreed": Answer(0.03, 0.94),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
result = await engine.resume(
|
|
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
|
|
)
|
|
|
|
assert reopened(result) == [
|
|
("requirements", "Retention must be 7 years, not 30 days.")
|
|
]
|
|
|
|
|
|
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
|
|
tmp_path,
|
|
):
|
|
turns = once()
|
|
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
|
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
|
engine, _ = engine_for(tmp_path, turns)
|
|
|
|
design = await to_design_approval(engine)
|
|
unnamed = await engine.resume(
|
|
design.execution_id, changes("solution_architecture", "Make it cheaper.")
|
|
)
|
|
|
|
assert reopened(unnamed) == []
|
|
assert unnamed.prompt["questions"][0]["question"].startswith(
|
|
"Which Deliverable should change"
|
|
)
|
|
|
|
named = await engine.resume(
|
|
unnamed.execution_id, {"reply": "The design recommendations."}
|
|
)
|
|
|
|
[(kind, reason)] = reopened(named)
|
|
assert kind == "design_recommendations"
|
|
assert reason == (
|
|
"Make it cheaper.\nThe user then said: The design recommendations."
|
|
)
|
|
|
|
|
|
# --- permission is the Ledger's, whatever was decided ------------------------------------
|
|
|
|
|
|
async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
|
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
|
|
assert len(reopened(result)) == 1
|
|
[question] = result.prompt["questions"]
|
|
assert (
|
|
"deployment_guidance has reached its revision limit (1)"
|
|
in (question["question"])
|
|
)
|
|
|
|
|
|
# --- the two faults of evaluation case 04 (2026-10-08) -----------------------------------
|
|
|
|
|
|
async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on(
|
|
tmp_path,
|
|
):
|
|
"""The user is told once. Their reply is not read as a new request for changes,
|
|
the Deliverable is not reopened again, and nothing is asked a second time."""
|
|
turns = once()
|
|
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
|
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
|
# A decider that would call any reply a clear request to change the code.
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.95, 0.9),
|
|
"change_target": Answer("deployment_guidance", 0.95),
|
|
"change_clarity": Answer(1.9, 0.8),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
for phase in (
|
|
"requirements_discovery",
|
|
"solution_architecture",
|
|
"solution_implementation",
|
|
"solution_implementation",
|
|
):
|
|
result = await engine.resume(result.execution_id, approve(phase))
|
|
[question] = result.prompt["questions"]
|
|
assert "I will carry on with it as it stands" in question["question"]
|
|
|
|
result = await engine.resume(
|
|
result.execution_id, {"reply": "Please change the code again."}
|
|
)
|
|
|
|
# It moves on to the Approval of validation; the code was reopened once only.
|
|
assert result.prompt["approval"]["phase"] == "solution_validation"
|
|
assert len(reopened(result)) == 1
|
|
assert result.state["revision_limit_asked"] == ["deployment_guidance"]
|
|
assert decider.asked == [] # the reply was never put to the decider
|
|
|
|
|
|
async def test_only_the_users_own_words_are_read_as_their_message(tmp_path):
|
|
"""A reply that quotes our question back must not be judged on our words."""
|
|
question = "Please confirm the requirements draft. Shall I change anything?"
|
|
turns = once()
|
|
turns[specialist(K.REQUIREMENTS)] = [
|
|
[Call("needs_input", {"questions": [question]})],
|
|
[sample_submission(K.REQUIREMENTS)],
|
|
]
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.05, 0.9),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.2, 0.5),
|
|
}
|
|
)
|
|
engine, _ = engine_for(tmp_path, turns, decider)
|
|
first = await engine.start("Ingest webhooks")
|
|
|
|
result = await engine.resume(
|
|
first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."}
|
|
)
|
|
# Nothing has been submitted yet, so nothing is judged on that turn. Let the
|
|
# next reply, to the decomposer's approval prompt, be a text one.
|
|
result = await engine.resume(result.execution_id, {"reply": "Looks fine."})
|
|
|
|
assert result.state["last_reply"] is None
|
|
state, _, _ = decider.asked[-1]
|
|
assert state.endswith("The user's message: Looks fine.")
|
|
|
|
|
|
def test_a_quoted_question_is_removed_from_what_the_user_said():
|
|
from app.nodes.execution_nodes import _own_words
|
|
|
|
asked = ["Please confirm the draft. Shall I change the design?", "Which region?"]
|
|
reply = (
|
|
"Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n"
|
|
"Q: Which region?\nA: europe-west2"
|
|
)
|
|
|
|
assert _own_words(reply, asked) == "Yes, approved.\neurope-west2"
|
|
assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years."
|
|
|
|
|
|
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
|
|
from app.nodes.router import TurnView
|
|
from app.states.ledger import Ledger
|
|
|
|
ledger = Ledger()
|
|
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
|
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
|
decider = FakeDecider(
|
|
{
|
|
"asks_change": Answer(0.05, 0.9),
|
|
"change_target": Answer("requirements", 0.5),
|
|
"change_clarity": Answer(0.2, 0.5),
|
|
}
|
|
)
|
|
router = Router(decider, None, Thresholds())
|
|
view = TurnView(
|
|
execution_id="e1",
|
|
pre_approved=False,
|
|
ledger=ledger,
|
|
deliverables={},
|
|
pending_questions=[],
|
|
outcomes=[],
|
|
last_reply={
|
|
"kind": "reply",
|
|
"text": "Yes, approved.",
|
|
"asked": ["Shall I change the design?"],
|
|
},
|
|
)
|
|
from app.nodes.router import Readings
|
|
|
|
await router._read_message(view, Readings())
|
|
|
|
state, _, _ = decider.asked[0]
|
|
assert "replying to these questions from the agents:\n- Shall I change" in state
|
|
assert state.endswith("The user's message: Yes, approved.")
|
|
|
|
|
|
async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
|
tmp_path,
|
|
):
|
|
"""Whatever causes it, a run that only asks is stopped, not left to the turn limit."""
|
|
from app.nodes.execution_nodes import MAX_IDLE_ASKS
|
|
from app.nodes.router import TurnResult
|
|
|
|
class OnlyAsks:
|
|
"""An Orchestrator that asks the user something on every turn."""
|
|
|
|
async def turn(self, view, tools):
|
|
ask = next(tool for tool in tools if tool.name == "ask_user")
|
|
reply = await ask.handler(questions=["What now?"])
|
|
return TurnResult(reply.outcome)
|
|
|
|
engine = ExecutionEngine(
|
|
runtime=FakeAgentRuntime({}),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=OnlyAsks(),
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks")
|
|
asked = 1
|
|
while result.status == "awaiting_user" and asked < 20:
|
|
result = await engine.resume(result.execution_id, {"reply": "Carry on."})
|
|
asked += 1
|
|
|
|
assert result.status == "completed_with_warnings"
|
|
assert asked == MAX_IDLE_ASKS + 1
|
|
assert "asked" in result.state["warnings"][-1]
|
|
assert "no work done in between" in result.state["warnings"][-1]
|
|
|
|
|
|
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
|
|
tmp_path,
|
|
):
|
|
"""Evaluation case 04: a requirement said again in reply to a later question."""
|
|
from app.nodes.router import Readings, TurnView
|
|
from app.states.ledger import Ledger
|
|
|
|
ledger = Ledger()
|
|
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
|
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
|
view = TurnView(
|
|
execution_id="e1",
|
|
pre_approved=False,
|
|
ledger=ledger,
|
|
deliverables={
|
|
"requirements": {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
}
|
|
},
|
|
pending_questions=[],
|
|
outcomes=[],
|
|
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
|
|
)
|
|
asks = {
|
|
"asks_change": Answer(0.9, 0.8),
|
|
"change_target": Answer("requirements", 0.95),
|
|
"change_clarity": Answer(1.9, 0.8),
|
|
}
|
|
|
|
restated, changed = Readings(), Readings()
|
|
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
|
|
await Router(decider, None, Thresholds())._read_message(view, restated)
|
|
await Router(
|
|
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
|
|
)._read_message(view, changed)
|
|
|
|
assert restated.change == "close"
|
|
assert changed.change == "act"
|
|
state, questions, _ = decider.asked[0]
|
|
assert "The agreed requirements:" in state and "NFR-1" in state
|
|
assert "already_agreed" in questions
|