feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -399,3 +399,44 @@ async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path)
|
||||
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
||||
"Make the flow clearer."
|
||||
]
|
||||
|
||||
|
||||
# --- requirements name no products ------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_requirement_that_names_a_product_is_raised():
|
||||
from app.tools.requirements_review import RequirementsReview
|
||||
|
||||
requirements = [
|
||||
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
|
||||
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
|
||||
]
|
||||
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
|
||||
|
||||
raised = await RequirementsReview(decider)(
|
||||
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
|
||||
)
|
||||
|
||||
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
|
||||
_, questions, scope = decider.asked[0]
|
||||
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
|
||||
assert "Accept signed webhooks." in questions["FR-2"].instructions
|
||||
|
||||
|
||||
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
|
||||
from app.tools.requirements_review import RequirementsReview
|
||||
|
||||
decider = FakeDecider({})
|
||||
review = RequirementsReview(decider)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
|
||||
assert decider.asked == []
|
||||
assert (
|
||||
await RequirementsReview(NoDecider())(
|
||||
K.REQUIREMENTS,
|
||||
{"requirements.json": SAMPLE_DATA["requirements.json"]},
|
||||
None,
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
@@ -488,12 +488,21 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert "revision limit" in result.state["warnings"][0]
|
||||
# No Deliverable files are written, but the request stays on record.
|
||||
# What was produced is kept, marked as unfinished, and the request is on record.
|
||||
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
||||
assert not (case / "executions").exists()
|
||||
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
||||
assert intent["status"] == "completed_with_warnings"
|
||||
assert intent["execution"] is None
|
||||
assert intent["execution"] == f"executions/{result.execution_id}"
|
||||
out_dir = case / intent["execution"]
|
||||
manifest = json.loads((out_dir / "execution.json").read_text())
|
||||
assert manifest["status"] == "completed_with_warnings"
|
||||
assert "revision limit" in manifest["warnings"][0]
|
||||
assert manifest["solution_guide"] is None
|
||||
assert not (out_dir / "solution-architecture-guide.md").exists()
|
||||
assert (out_dir / "deliverables" / "requirements.md").is_file()
|
||||
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
|
||||
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
|
||||
assert manifest["phases"]["packaging"] != "approved"
|
||||
|
||||
|
||||
# --- budgets -------------------------------------------------------------------------------------
|
||||
@@ -880,7 +889,10 @@ async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
],
|
||||
"status": "submitted",
|
||||
}
|
||||
assert manifest["status"] == "completed" and manifest["warnings"] == []
|
||||
assert set(manifest["phases"].values()) == {"approved"}
|
||||
for entry in manifest["deliverables"].values():
|
||||
assert (out / entry["document"]).is_file()
|
||||
assert all((out / name).is_file() for name in entry["attachments"])
|
||||
@@ -969,3 +981,92 @@ async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
||||
assert entry["status"] == "in_progress"
|
||||
assert entry["statement"] == "intent/001.md"
|
||||
assert entry["execution_id"] == paused.execution_id
|
||||
|
||||
|
||||
# --- a user sees a Deliverable before it is first submitted -----------------------------
|
||||
|
||||
|
||||
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
|
||||
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()],
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
# It tries to submit straight away, is refused, and asks.
|
||||
[
|
||||
sample_submission(K.REQUIREMENTS),
|
||||
Call(
|
||||
"needs_input",
|
||||
{"questions": ["Please approve the draft."], "draft": "# r"},
|
||||
),
|
||||
],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
refusal = runtime.runs[1].results[0][1]
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
# Once they have replied, the submission is accepted.
|
||||
assert runtime.runs[-2].results[0][1].startswith("Submitted")
|
||||
|
||||
|
||||
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
|
||||
runtime = FakeAgentRuntime(happy_path_turns())
|
||||
|
||||
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
|
||||
"Ingestion", pre_approved=True
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
|
||||
|
||||
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
|
||||
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
|
||||
the Orchestrator and then submitted. The user had answered a question about the
|
||||
Deliverable but had never seen it."""
|
||||
draft = Call(
|
||||
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
|
||||
)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays "the requirements analyst stopped"
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays the draft
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[], # stops without ending its turn
|
||||
[sample_submission(K.REQUIREMENTS), draft],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
assert "stopped" in first.prompt["questions"][0]["question"]
|
||||
second = await engine.resume(first.execution_id, {"reply": "Try again."})
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
|
||||
assert (
|
||||
analyst[1]
|
||||
.results[0][1]
|
||||
.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
)
|
||||
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
assert analyst[2].results[0][1].startswith("Submitted")
|
||||
|
||||
58
tests/execution/test_intake.py
Normal file
58
tests/execution/test_intake.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Intake: what a message that starts something is asking for."""
|
||||
|
||||
import json
|
||||
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer, NoDecider
|
||||
from app.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
|
||||
|
||||
|
||||
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
|
||||
folder = tmp_path / "deliverables" / "cases" / case / "intent"
|
||||
folder.mkdir(parents=True)
|
||||
intents = [
|
||||
{"version": n, "workflow_request": f"Design thing {n}.", "status": status}
|
||||
for n, status in enumerate(statuses, start=1)
|
||||
]
|
||||
(folder / "history.json").write_text(
|
||||
json.dumps({"customer_case": case, "intents": intents})
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
|
||||
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
|
||||
|
||||
assert reading.kind == NEW_REQUEST and not reading.judged
|
||||
|
||||
|
||||
async def test_a_confident_question_is_read_as_one(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert reading.kind == QUESTION and reading.judged
|
||||
assert scope == "t1"
|
||||
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
|
||||
assert state.endswith("The user's message: How long does it take?")
|
||||
|
||||
|
||||
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
|
||||
|
||||
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
|
||||
|
||||
|
||||
async def test_a_change_needs_something_delivered_to_change(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
|
||||
delivered(tmp_path, "acme", ["completed", "in_progress"])
|
||||
intake = Intake(decider, tmp_path)
|
||||
|
||||
with_history = await intake.read("Raise availability to 99.99%.", "acme")
|
||||
without = await intake.read("Raise availability to 99.99%.", "new-client")
|
||||
no_case = await intake.read("Raise availability to 99.99%.")
|
||||
|
||||
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
|
||||
assert "- Design thing 1." in decider.asked[0][0]
|
||||
assert "Design thing 2." not in decider.asked[0][0] # still in progress
|
||||
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
|
||||
@@ -308,3 +308,103 @@ async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
|
||||
# --- each advisor reviews its pillar's checklist ----------------------------------------
|
||||
|
||||
|
||||
def reviewed(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [
|
||||
Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": f"Use strong defaults for {pillar}.",
|
||||
"checklist": [
|
||||
{
|
||||
"item": f"{pillar} item one | with a bar",
|
||||
"status": "addressed",
|
||||
"note": "Covered by the first recommendation.",
|
||||
},
|
||||
{
|
||||
"item": f"{pillar} item two",
|
||||
"status": "not_applicable",
|
||||
"note": "An organisation's process, not a design decision.",
|
||||
},
|
||||
{"item": "malformed", "status": "maybe", "note": "dropped"},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
|
||||
from app.tools.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
# The design advisor is given the advice, not the review: it does not copy it.
|
||||
assert "item one" not in result.message
|
||||
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
|
||||
review = memory[CHECKLIST_KEY]["security"]
|
||||
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
|
||||
|
||||
|
||||
async def test_the_review_is_written_into_the_deliverable_by_code():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.adk.runtime import Submitted
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working=memory)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
call = sample_submission(kind)
|
||||
|
||||
first = await submit.handler(**call.args)
|
||||
# A Revision submits content that already carries the section.
|
||||
again = await submit.handler(
|
||||
content=first.outcome.content, attachments=call.args["attachments"]
|
||||
)
|
||||
|
||||
assert isinstance(first.outcome, Submitted)
|
||||
content = first.outcome.content
|
||||
assert f"## {REVIEW_HEADING}" in content and "### security" in content
|
||||
assert (
|
||||
"| security item one \\| with a bar | Addressed | Covered by the first "
|
||||
"recommendation. |"
|
||||
) in content
|
||||
assert "| security item two | Does not apply |" in content
|
||||
assert "malformed" not in content
|
||||
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
|
||||
|
||||
|
||||
async def test_with_no_review_nothing_is_added():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working={})
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
|
||||
result = await submit.handler(**sample_submission(kind).args)
|
||||
|
||||
assert REVIEW_HEADING not in result.outcome.content
|
||||
|
||||
@@ -689,13 +689,11 @@ def test_a_non_functional_requirement_must_list_its_conditions():
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
)
|
||||
assert missing == (
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"requirements.json: NFR-1 {needs}",
|
||||
f"requirements.json: NFR-2 {needs}",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_recommendation_must_list_its_conditions():
|
||||
@@ -709,7 +707,61 @@ def test_a_recommendation_must_list_its_conditions():
|
||||
{"recommendations.json": json.dumps(recommendations)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"recommendations.json: DR-1 {needs}"
|
||||
]
|
||||
|
||||
|
||||
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
|
||||
"""An outcome measured in operation is recorded, and no Check is asked for it."""
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Pages load within one second at p95.",
|
||||
"conditions": [],
|
||||
"outside_code": ["Measured p95 page load is under one second."],
|
||||
}
|
||||
)
|
||||
assert missing == (f"recommendations.json: DR-1 {needs}",)
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
|
||||
accepted = check_submission(
|
||||
K.REQUIREMENTS,
|
||||
args["content"],
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert accepted == ()
|
||||
assert gaps == ()
|
||||
|
||||
|
||||
def test_a_constraint_with_conditions_still_needs_a_check():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Two zones, 99.9%.",
|
||||
"conditions": ["The service runs in two zones."],
|
||||
"outside_code": ["Measured availability is 99.9%."],
|
||||
}
|
||||
)
|
||||
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert gaps == ("Requirement NFR-2 has no Check",)
|
||||
|
||||
@@ -529,6 +529,7 @@ async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
||||
"asks_change": Answer(0.93, 0.86),
|
||||
"change_target": Answer("requirements", 0.9),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
"already_agreed": Answer(0.03, 0.94),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
@@ -755,3 +756,46 @@ async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
||||
assert asked == MAX_IDLE_ASKS + 1
|
||||
assert "asked" in result.state["warnings"][-1]
|
||||
assert "no work done in between" in result.state["warnings"][-1]
|
||||
|
||||
|
||||
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
|
||||
tmp_path,
|
||||
):
|
||||
"""Evaluation case 04: a requirement said again in reply to a later question."""
|
||||
from app.nodes.router import Readings, TurnView
|
||||
from app.states.ledger import Ledger
|
||||
|
||||
ledger = Ledger()
|
||||
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
view = TurnView(
|
||||
execution_id="e1",
|
||||
pre_approved=False,
|
||||
ledger=ledger,
|
||||
deliverables={
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
}
|
||||
},
|
||||
pending_questions=[],
|
||||
outcomes=[],
|
||||
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
|
||||
)
|
||||
asks = {
|
||||
"asks_change": Answer(0.9, 0.8),
|
||||
"change_target": Answer("requirements", 0.95),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
|
||||
restated, changed = Readings(), Readings()
|
||||
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
|
||||
await Router(decider, None, Thresholds())._read_message(view, restated)
|
||||
await Router(
|
||||
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
|
||||
)._read_message(view, changed)
|
||||
|
||||
assert restated.change == "close"
|
||||
assert changed.change == "act"
|
||||
state, questions, _ = decider.asked[0]
|
||||
assert "The agreed requirements:" in state and "NFR-1" in state
|
||||
assert "already_agreed" in questions
|
||||
|
||||
Reference in New Issue
Block a user