feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

View File

@@ -8,11 +8,18 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from starlette.testclient import TestClient
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.fake_runtime import (
Call,
FakeAgentRuntime,
FakeDecider,
sample_submission,
)
from app.adk.roles import load_roles
from app.adk.runtime import Answer
from app.agent import Assembly, ExecutionEngine
from app.config import get_settings
from app.main import create_app
from app.nodes.intake import Intake
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
@@ -67,7 +74,9 @@ def complete_run_turns() -> dict:
}
def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntime]:
def client_for(
tmp_path: Path, turns: dict, decider=None
) -> tuple[TestClient, FakeAgentRuntime]:
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
runtime = FakeAgentRuntime(turns)
@@ -81,7 +90,13 @@ def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntim
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
yield Assembly(engine, bundle, roles, {"compute": "API not enabled"})
yield Assembly(
engine,
bundle,
roles,
{"compute": "API not enabled"},
intake=Intake(decider, tmp_path) if decider else None,
)
return TestClient(create_app(get_settings(), fake_assembly)), runtime
@@ -319,3 +334,56 @@ def test_generate_groups_the_execution_under_the_callers_customer_case(tmp_path)
assert "customer_case" in refused.json()["error"]
assert accepted.status_code == 200, accepted.json()
assert (tmp_path / "lessons" / "candidates" / "acme-webhooks").is_dir()
# --- a message that starts something is read first (intake) ----------------------------
def test_a_question_is_answered_and_starts_no_execution(tmp_path):
decider = FakeDecider({"intake": Answer("question", 0.97)})
client, runtime = client_for(tmp_path, {}, decider)
with client:
task = send(client, [{"text": "What can you do?"}])
text, data = status_parts(task)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert "Nothing was started" in text
assert data == {"status": "answered", "intake": "question"}
assert runtime.runs == []
assert not (tmp_path / "deliverables").exists()
def test_a_new_request_starts_an_execution_as_before(tmp_path):
decider = FakeDecider({"intake": Answer("new_request", 0.99)})
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
client, runtime = client_for(tmp_path, turns, decider)
with client:
task = send(client, [{"text": "Design an ingestion pipeline."}])
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
assert "Which region?" in status_parts(task)[0]
assert len(runtime.runs) == 1
def test_an_unsure_reading_starts_an_execution(tmp_path):
decider = FakeDecider({"intake": Answer("question", 0.5)})
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
client, runtime = client_for(tmp_path, turns, decider)
with client:
task = send(client, [{"text": "hello"}])
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
assert len(runtime.runs) == 1
def test_a_pre_approved_task_is_not_read_as_a_question(tmp_path):
decider = FakeDecider({"intake": Answer("question", 0.99)})
client, runtime = client_for(tmp_path, complete_run_turns(), decider)
with client:
task = send(
client, [{"text": "What can you do?"}], metadata={"pre_approved": True}
)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert decider.asked == []
assert runtime.runs

View File

@@ -199,9 +199,11 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
assert len(cases) >= 2
for case in cases:
assert case.required_products
for pattern, answer in case.scripted_answers:
for pattern, answer, *asker in case.scripted_answers:
re.compile(pattern)
assert answer
for who in asker:
re.compile(who)
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
@@ -528,3 +530,87 @@ async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_pass
"judge unavailable: Jev gave no answer"
)
assert not check(result, "recommendations_quality").passed
# --- the scripted user (evaluation case 04) --------------------------------------------
def test_a_scripted_fact_is_given_only_to_who_it_was_written_for():
from app.nodes.execution_nodes import automatic_reply
scripted = [
["region|location", "europe-west1.", "requirements"],
["approve|confirm", "Approved. Please proceed."],
]
def reply(asker: str, question: str) -> str:
prompt = {"questions": [{"from": asker, "question": question}]}
return automatic_reply(prompt, scripted)["reply"]
assert reply("requirements", "Which region?").endswith("A: europe-west1.")
# A later Specialist confirming a draft that mentions the region gets an approval.
assert reply(
"validation_results", "Region is europe-west1. Please confirm my draft."
).endswith("A: Approved. Please proceed.")
# With no answer for it, the reply says no user is available.
assert "No user is available" in reply("product_selection", "Which region?")
def test_case_04_answers_a_draft_confirmation_with_an_approval():
from app.nodes.execution_nodes import automatic_reply
from eval.scoring import load_cases
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
case = next(c for c in cases if c.id == "case-04-interactive-web-app")
def reply(asker: str, question: str) -> str:
prompt = {"questions": [{"from": asker, "question": question}]}
return automatic_reply(prompt, [list(a) for a in case.scripted_answers])[
"reply"
]
question = "I checked encryption, GDPR and password rotation. Please confirm."
assert reply("validation_results", question).endswith(
"A: Approved. Please proceed."
)
assert "europe-west1" in reply("requirements", "Which region should it run in?")
# --- calibrating the judges ------------------------------------------------------------
def test_calibration_reports_agreement_stability_and_between_judges():
from eval.calibrate_judge import report
items = [
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": 1},
{"id": "b", "document": "d.md", "kind": "graded", "text": "?", "label": 0.67},
{"id": "c", "document": "d.md", "kind": "yes_no", "text": "?", "label": None},
]
by_judge = {
"jev": {"a": [1.0, 1.0], "b": [0.33, 0.67], "c": [0.0, 0.0]},
"gemini": {"a": [1.0, 1.0], "b": [0.67, 0.67], "c": [1.0, 1.0]},
}
result = report(items, by_judge)
assert result["questions"] == 3 and result["labelled"] == 2
assert result["judges"]["jev"]["agreement_with_labels"] == 0.5
assert result["judges"]["gemini"]["agreement_with_labels"] == 1.0
assert result["judges"]["jev"]["stability"] == 0.67
assert result["judges"]["gemini"]["stability"] == 1.0
assert result["between_judges"] == 0.33
def test_calibration_without_labels_still_measures_stability():
from eval.calibrate_judge import report
items = [
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": None}
]
result = report(items, {"jev": {"a": [1.0, 1.0, 1.0]}})
assert result["labelled"] == 0
assert result["judges"]["jev"]["agreement_with_labels"] is None
assert result["judges"]["jev"]["stability"] == 1.0

View File

@@ -399,3 +399,44 @@ async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path)
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
"Make the flow clearer."
]
# --- requirements name no products ------------------------------------------------------
async def test_a_requirement_that_names_a_product_is_raised():
from app.tools.requirements_review import RequirementsReview
requirements = [
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
]
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
raised = await RequirementsReview(decider)(
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
)
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
_, questions, scope = decider.asked[0]
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
assert "Accept signed webhooks." in questions["FR-2"].instructions
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
from app.tools.requirements_review import RequirementsReview
decider = FakeDecider({})
review = RequirementsReview(decider)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
assert decider.asked == []
assert (
await RequirementsReview(NoDecider())(
K.REQUIREMENTS,
{"requirements.json": SAMPLE_DATA["requirements.json"]},
None,
)
== ()
)

View File

@@ -488,12 +488,21 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
assert result.status == "completed_with_warnings"
assert "revision limit" in result.state["warnings"][0]
# No Deliverable files are written, but the request stays on record.
# What was produced is kept, marked as unfinished, and the request is on record.
case = tmp_path / "deliverables" / "cases" / "unassigned"
assert not (case / "executions").exists()
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
assert intent["status"] == "completed_with_warnings"
assert intent["execution"] is None
assert intent["execution"] == f"executions/{result.execution_id}"
out_dir = case / intent["execution"]
manifest = json.loads((out_dir / "execution.json").read_text())
assert manifest["status"] == "completed_with_warnings"
assert "revision limit" in manifest["warnings"][0]
assert manifest["solution_guide"] is None
assert not (out_dir / "solution-architecture-guide.md").exists()
assert (out_dir / "deliverables" / "requirements.md").is_file()
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
assert manifest["phases"]["packaging"] != "approved"
# --- budgets -------------------------------------------------------------------------------------
@@ -880,7 +889,10 @@ async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
"terraform/main.tf",
"terraform/variables.tf",
],
"status": "submitted",
}
assert manifest["status"] == "completed" and manifest["warnings"] == []
assert set(manifest["phases"].values()) == {"approved"}
for entry in manifest["deliverables"].values():
assert (out / entry["document"]).is_file()
assert all((out / name).is_file() for name in entry["attachments"])
@@ -969,3 +981,92 @@ async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
assert entry["status"] == "in_progress"
assert entry["statement"] == "intent/001.md"
assert entry["execution_id"] == paused.execution_id
# --- a user sees a Deliverable before it is first submitted -----------------------------
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()],
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
# It tries to submit straight away, is refused, and asks.
[
sample_submission(K.REQUIREMENTS),
Call(
"needs_input",
{"questions": ["Please approve the draft."], "draft": "# r"},
),
],
submits(K.REQUIREMENTS),
],
}
)
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
first = await engine.start("Ingestion")
refusal = runtime.runs[1].results[0][1]
await engine.resume(first.execution_id, {"reply": "Approved."})
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
# Once they have replied, the submission is accepted.
assert runtime.runs[-2].results[0][1].startswith("Submitted")
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
"Ingestion", pre_approved=True
)
assert result.status == "completed"
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
the Orchestrator and then submitted. The user had answered a question about the
Deliverable but had never seen it."""
draft = Call(
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
)
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()], # relays "the requirements analyst stopped"
[delegate(K.REQUIREMENTS)],
[ask_user()], # relays the draft
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
[], # stops without ending its turn
[sample_submission(K.REQUIREMENTS), draft],
submits(K.REQUIREMENTS),
],
}
)
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
first = await engine.start("Ingestion")
assert "stopped" in first.prompt["questions"][0]["question"]
second = await engine.resume(first.execution_id, {"reply": "Try again."})
await engine.resume(first.execution_id, {"reply": "Approved."})
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
assert (
analyst[1]
.results[0][1]
.startswith("Not submitted. The user has not seen this Deliverable")
)
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
assert analyst[2].results[0][1].startswith("Submitted")

View File

@@ -0,0 +1,58 @@
"""Intake: what a message that starts something is asking for."""
import json
from app.adk.fake_runtime import FakeDecider
from app.adk.runtime import Answer, NoDecider
from app.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
folder = tmp_path / "deliverables" / "cases" / case / "intent"
folder.mkdir(parents=True)
intents = [
{"version": n, "workflow_request": f"Design thing {n}.", "status": status}
for n, status in enumerate(statuses, start=1)
]
(folder / "history.json").write_text(
json.dumps({"customer_case": case, "intents": intents})
)
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
assert reading.kind == NEW_REQUEST and not reading.judged
async def test_a_confident_question_is_read_as_one(tmp_path):
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
state, questions, scope = decider.asked[0]
assert reading.kind == QUESTION and reading.judged
assert scope == "t1"
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
assert state.endswith("The user's message: How long does it take?")
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
async def test_a_change_needs_something_delivered_to_change(tmp_path):
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
delivered(tmp_path, "acme", ["completed", "in_progress"])
intake = Intake(decider, tmp_path)
with_history = await intake.read("Raise availability to 99.99%.", "acme")
without = await intake.read("Raise availability to 99.99%.", "new-client")
no_case = await intake.read("Raise availability to 99.99%.")
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
assert "- Design thing 1." in decider.asked[0][0]
assert "Design thing 2." not in decider.asked[0][0] # still in progress
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST

View File

@@ -308,3 +308,103 @@ async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
assert "## reliability (from your earlier consultation)" in second.results[0][1]
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
assert set(stored) == set(PILLARS)
# --- each advisor reviews its pillar's checklist ----------------------------------------
def reviewed(task):
pillar = task.context["pillar"]
return [
Call(
"submit_advice",
{
"content": f"Use strong defaults for {pillar}.",
"checklist": [
{
"item": f"{pillar} item one | with a bar",
"status": "addressed",
"note": "Covered by the first recommendation.",
},
{
"item": f"{pillar} item two",
"status": "not_applicable",
"note": "An organisation's process, not a design decision.",
},
{"item": "malformed", "status": "maybe", "note": "dropped"},
],
},
)
]
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
from app.tools.pillars import CHECKLIST_KEY
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="b")
# The design advisor is given the advice, not the review: it does not copy it.
assert "item one" not in result.message
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
review = memory[CHECKLIST_KEY]["security"]
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
async def test_the_review_is_written_into_the_deliverable_by_code():
from app.adk.fake_runtime import sample_submission
from app.adk.runtime import Submitted
from app.tools.execution import specialist_tools
from app.tools.pillars import REVIEW_HEADING
from app.workflows.deliverables import DeliverableKind
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
(submit,) = (
tool
for tool in specialist_tools(kind, (), allow_cloud_read=False, working=memory)
if tool.name == "submit_deliverable"
)
call = sample_submission(kind)
first = await submit.handler(**call.args)
# A Revision submits content that already carries the section.
again = await submit.handler(
content=first.outcome.content, attachments=call.args["attachments"]
)
assert isinstance(first.outcome, Submitted)
content = first.outcome.content
assert f"## {REVIEW_HEADING}" in content and "### security" in content
assert (
"| security item one \\| with a bar | Addressed | Covered by the first "
"recommendation. |"
) in content
assert "| security item two | Does not apply |" in content
assert "malformed" not in content
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
async def test_with_no_review_nothing_is_added():
from app.adk.fake_runtime import sample_submission
from app.tools.execution import specialist_tools
from app.tools.pillars import REVIEW_HEADING
from app.workflows.deliverables import DeliverableKind
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
(submit,) = (
tool
for tool in specialist_tools(kind, (), allow_cloud_read=False, working={})
if tool.name == "submit_deliverable"
)
result = await submit.handler(**sample_submission(kind).args)
assert REVIEW_HEADING not in result.outcome.content

View File

@@ -689,13 +689,11 @@ def test_a_non_functional_requirement_must_list_its_conditions():
{"requirements.json": json.dumps(requirements)},
)
needs = (
"needs `conditions`: a list with one entry for each separate thing it asks for"
)
assert missing == (
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"requirements.json: NFR-1 {needs}",
f"requirements.json: NFR-2 {needs}",
)
]
def test_a_recommendation_must_list_its_conditions():
@@ -709,7 +707,61 @@ def test_a_recommendation_must_list_its_conditions():
{"recommendations.json": json.dumps(recommendations)},
)
needs = (
"needs `conditions`: a list with one entry for each separate thing it asks for"
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"recommendations.json: DR-1 {needs}"
]
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
"""An outcome measured in operation is recorded, and no Check is asked for it."""
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Pages load within one second at p95.",
"conditions": [],
"outside_code": ["Measured p95 page load is under one second."],
}
)
assert missing == (f"recommendations.json: DR-1 {needs}",)
args = sample_submission(K.REQUIREMENTS).args
accepted = check_submission(
K.REQUIREMENTS,
args["content"],
{"requirements.json": json.dumps(requirements)},
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert accepted == ()
assert gaps == ()
def test_a_constraint_with_conditions_still_needs_a_check():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Two zones, 99.9%.",
"conditions": ["The service runs in two zones."],
"outside_code": ["Measured availability is 99.9%."],
}
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert gaps == ("Requirement NFR-2 has no Check",)

View File

@@ -529,6 +529,7 @@ async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
"asks_change": Answer(0.93, 0.86),
"change_target": Answer("requirements", 0.9),
"change_clarity": Answer(1.9, 0.8),
"already_agreed": Answer(0.03, 0.94),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
@@ -755,3 +756,46 @@ async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
assert asked == MAX_IDLE_ASKS + 1
assert "asked" in result.state["warnings"][-1]
assert "no work done in between" in result.state["warnings"][-1]
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
tmp_path,
):
"""Evaluation case 04: a requirement said again in reply to a later question."""
from app.nodes.router import Readings, TurnView
from app.states.ledger import Ledger
ledger = Ledger()
ledger.start([K.REQUIREMENTS], actor="orchestrator")
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
view = TurnView(
execution_id="e1",
pre_approved=False,
ledger=ledger,
deliverables={
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
}
},
pending_questions=[],
outcomes=[],
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
)
asks = {
"asks_change": Answer(0.9, 0.8),
"change_target": Answer("requirements", 0.95),
"change_clarity": Answer(1.9, 0.8),
}
restated, changed = Readings(), Readings()
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
await Router(decider, None, Thresholds())._read_message(view, restated)
await Router(
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
)._read_message(view, changed)
assert restated.change == "close"
assert changed.change == "act"
state, questions, _ = decider.asked[0]
assert "The agreed requirements:" in state and "NFR-1" in state
assert "already_agreed" in questions