feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,11 +8,18 @@ import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.fake_runtime import (
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.roles import load_roles
|
||||
from app.adk.runtime import Answer
|
||||
from app.agent import Assembly, ExecutionEngine
|
||||
from app.config import get_settings
|
||||
from app.main import create_app
|
||||
from app.nodes.intake import Intake
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
@@ -67,7 +74,9 @@ def complete_run_turns() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
def client_for(
|
||||
tmp_path: Path, turns: dict, decider=None
|
||||
) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
@@ -81,7 +90,13 @@ def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntim
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
yield Assembly(engine, bundle, roles, {"compute": "API not enabled"})
|
||||
yield Assembly(
|
||||
engine,
|
||||
bundle,
|
||||
roles,
|
||||
{"compute": "API not enabled"},
|
||||
intake=Intake(decider, tmp_path) if decider else None,
|
||||
)
|
||||
|
||||
return TestClient(create_app(get_settings(), fake_assembly)), runtime
|
||||
|
||||
@@ -319,3 +334,56 @@ def test_generate_groups_the_execution_under_the_callers_customer_case(tmp_path)
|
||||
assert "customer_case" in refused.json()["error"]
|
||||
assert accepted.status_code == 200, accepted.json()
|
||||
assert (tmp_path / "lessons" / "candidates" / "acme-webhooks").is_dir()
|
||||
|
||||
|
||||
# --- a message that starts something is read first (intake) ----------------------------
|
||||
|
||||
|
||||
def test_a_question_is_answered_and_starts_no_execution(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.97)})
|
||||
client, runtime = client_for(tmp_path, {}, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "What can you do?"}])
|
||||
|
||||
text, data = status_parts(task)
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert "Nothing was started" in text
|
||||
assert data == {"status": "answered", "intake": "question"}
|
||||
assert runtime.runs == []
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
|
||||
|
||||
def test_a_new_request_starts_an_execution_as_before(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("new_request", 0.99)})
|
||||
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
||||
client, runtime = client_for(tmp_path, turns, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "Design an ingestion pipeline."}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
assert "Which region?" in status_parts(task)[0]
|
||||
assert len(runtime.runs) == 1
|
||||
|
||||
|
||||
def test_an_unsure_reading_starts_an_execution(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.5)})
|
||||
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
||||
client, runtime = client_for(tmp_path, turns, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "hello"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
assert len(runtime.runs) == 1
|
||||
|
||||
|
||||
def test_a_pre_approved_task_is_not_read_as_a_question(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.99)})
|
||||
client, runtime = client_for(tmp_path, complete_run_turns(), decider)
|
||||
with client:
|
||||
task = send(
|
||||
client, [{"text": "What can you do?"}], metadata={"pre_approved": True}
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert decider.asked == []
|
||||
assert runtime.runs
|
||||
|
||||
@@ -199,9 +199,11 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
|
||||
assert len(cases) >= 2
|
||||
for case in cases:
|
||||
assert case.required_products
|
||||
for pattern, answer in case.scripted_answers:
|
||||
for pattern, answer, *asker in case.scripted_answers:
|
||||
re.compile(pattern)
|
||||
assert answer
|
||||
for who in asker:
|
||||
re.compile(who)
|
||||
|
||||
|
||||
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
|
||||
@@ -528,3 +530,87 @@ async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_pass
|
||||
"judge unavailable: Jev gave no answer"
|
||||
)
|
||||
assert not check(result, "recommendations_quality").passed
|
||||
|
||||
|
||||
# --- the scripted user (evaluation case 04) --------------------------------------------
|
||||
|
||||
|
||||
def test_a_scripted_fact_is_given_only_to_who_it_was_written_for():
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
|
||||
scripted = [
|
||||
["region|location", "europe-west1.", "requirements"],
|
||||
["approve|confirm", "Approved. Please proceed."],
|
||||
]
|
||||
|
||||
def reply(asker: str, question: str) -> str:
|
||||
prompt = {"questions": [{"from": asker, "question": question}]}
|
||||
return automatic_reply(prompt, scripted)["reply"]
|
||||
|
||||
assert reply("requirements", "Which region?").endswith("A: europe-west1.")
|
||||
# A later Specialist confirming a draft that mentions the region gets an approval.
|
||||
assert reply(
|
||||
"validation_results", "Region is europe-west1. Please confirm my draft."
|
||||
).endswith("A: Approved. Please proceed.")
|
||||
# With no answer for it, the reply says no user is available.
|
||||
assert "No user is available" in reply("product_selection", "Which region?")
|
||||
|
||||
|
||||
def test_case_04_answers_a_draft_confirmation_with_an_approval():
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
from eval.scoring import load_cases
|
||||
|
||||
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
|
||||
case = next(c for c in cases if c.id == "case-04-interactive-web-app")
|
||||
|
||||
def reply(asker: str, question: str) -> str:
|
||||
prompt = {"questions": [{"from": asker, "question": question}]}
|
||||
return automatic_reply(prompt, [list(a) for a in case.scripted_answers])[
|
||||
"reply"
|
||||
]
|
||||
|
||||
question = "I checked encryption, GDPR and password rotation. Please confirm."
|
||||
assert reply("validation_results", question).endswith(
|
||||
"A: Approved. Please proceed."
|
||||
)
|
||||
assert "europe-west1" in reply("requirements", "Which region should it run in?")
|
||||
|
||||
|
||||
# --- calibrating the judges ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_calibration_reports_agreement_stability_and_between_judges():
|
||||
from eval.calibrate_judge import report
|
||||
|
||||
items = [
|
||||
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": 1},
|
||||
{"id": "b", "document": "d.md", "kind": "graded", "text": "?", "label": 0.67},
|
||||
{"id": "c", "document": "d.md", "kind": "yes_no", "text": "?", "label": None},
|
||||
]
|
||||
by_judge = {
|
||||
"jev": {"a": [1.0, 1.0], "b": [0.33, 0.67], "c": [0.0, 0.0]},
|
||||
"gemini": {"a": [1.0, 1.0], "b": [0.67, 0.67], "c": [1.0, 1.0]},
|
||||
}
|
||||
|
||||
result = report(items, by_judge)
|
||||
|
||||
assert result["questions"] == 3 and result["labelled"] == 2
|
||||
assert result["judges"]["jev"]["agreement_with_labels"] == 0.5
|
||||
assert result["judges"]["gemini"]["agreement_with_labels"] == 1.0
|
||||
assert result["judges"]["jev"]["stability"] == 0.67
|
||||
assert result["judges"]["gemini"]["stability"] == 1.0
|
||||
assert result["between_judges"] == 0.33
|
||||
|
||||
|
||||
def test_calibration_without_labels_still_measures_stability():
|
||||
from eval.calibrate_judge import report
|
||||
|
||||
items = [
|
||||
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": None}
|
||||
]
|
||||
|
||||
result = report(items, {"jev": {"a": [1.0, 1.0, 1.0]}})
|
||||
|
||||
assert result["labelled"] == 0
|
||||
assert result["judges"]["jev"]["agreement_with_labels"] is None
|
||||
assert result["judges"]["jev"]["stability"] == 1.0
|
||||
|
||||
@@ -399,3 +399,44 @@ async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path)
|
||||
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
||||
"Make the flow clearer."
|
||||
]
|
||||
|
||||
|
||||
# --- requirements name no products ------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_requirement_that_names_a_product_is_raised():
|
||||
from app.tools.requirements_review import RequirementsReview
|
||||
|
||||
requirements = [
|
||||
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
|
||||
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
|
||||
]
|
||||
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
|
||||
|
||||
raised = await RequirementsReview(decider)(
|
||||
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
|
||||
)
|
||||
|
||||
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
|
||||
_, questions, scope = decider.asked[0]
|
||||
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
|
||||
assert "Accept signed webhooks." in questions["FR-2"].instructions
|
||||
|
||||
|
||||
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
|
||||
from app.tools.requirements_review import RequirementsReview
|
||||
|
||||
decider = FakeDecider({})
|
||||
review = RequirementsReview(decider)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
|
||||
assert decider.asked == []
|
||||
assert (
|
||||
await RequirementsReview(NoDecider())(
|
||||
K.REQUIREMENTS,
|
||||
{"requirements.json": SAMPLE_DATA["requirements.json"]},
|
||||
None,
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
@@ -488,12 +488,21 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert "revision limit" in result.state["warnings"][0]
|
||||
# No Deliverable files are written, but the request stays on record.
|
||||
# What was produced is kept, marked as unfinished, and the request is on record.
|
||||
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
||||
assert not (case / "executions").exists()
|
||||
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
||||
assert intent["status"] == "completed_with_warnings"
|
||||
assert intent["execution"] is None
|
||||
assert intent["execution"] == f"executions/{result.execution_id}"
|
||||
out_dir = case / intent["execution"]
|
||||
manifest = json.loads((out_dir / "execution.json").read_text())
|
||||
assert manifest["status"] == "completed_with_warnings"
|
||||
assert "revision limit" in manifest["warnings"][0]
|
||||
assert manifest["solution_guide"] is None
|
||||
assert not (out_dir / "solution-architecture-guide.md").exists()
|
||||
assert (out_dir / "deliverables" / "requirements.md").is_file()
|
||||
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
|
||||
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
|
||||
assert manifest["phases"]["packaging"] != "approved"
|
||||
|
||||
|
||||
# --- budgets -------------------------------------------------------------------------------------
|
||||
@@ -880,7 +889,10 @@ async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
],
|
||||
"status": "submitted",
|
||||
}
|
||||
assert manifest["status"] == "completed" and manifest["warnings"] == []
|
||||
assert set(manifest["phases"].values()) == {"approved"}
|
||||
for entry in manifest["deliverables"].values():
|
||||
assert (out / entry["document"]).is_file()
|
||||
assert all((out / name).is_file() for name in entry["attachments"])
|
||||
@@ -969,3 +981,92 @@ async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
||||
assert entry["status"] == "in_progress"
|
||||
assert entry["statement"] == "intent/001.md"
|
||||
assert entry["execution_id"] == paused.execution_id
|
||||
|
||||
|
||||
# --- a user sees a Deliverable before it is first submitted -----------------------------
|
||||
|
||||
|
||||
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
|
||||
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()],
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
# It tries to submit straight away, is refused, and asks.
|
||||
[
|
||||
sample_submission(K.REQUIREMENTS),
|
||||
Call(
|
||||
"needs_input",
|
||||
{"questions": ["Please approve the draft."], "draft": "# r"},
|
||||
),
|
||||
],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
refusal = runtime.runs[1].results[0][1]
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
# Once they have replied, the submission is accepted.
|
||||
assert runtime.runs[-2].results[0][1].startswith("Submitted")
|
||||
|
||||
|
||||
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
|
||||
runtime = FakeAgentRuntime(happy_path_turns())
|
||||
|
||||
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
|
||||
"Ingestion", pre_approved=True
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
|
||||
|
||||
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
|
||||
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
|
||||
the Orchestrator and then submitted. The user had answered a question about the
|
||||
Deliverable but had never seen it."""
|
||||
draft = Call(
|
||||
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
|
||||
)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays "the requirements analyst stopped"
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays the draft
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[], # stops without ending its turn
|
||||
[sample_submission(K.REQUIREMENTS), draft],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
assert "stopped" in first.prompt["questions"][0]["question"]
|
||||
second = await engine.resume(first.execution_id, {"reply": "Try again."})
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
|
||||
assert (
|
||||
analyst[1]
|
||||
.results[0][1]
|
||||
.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
)
|
||||
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
assert analyst[2].results[0][1].startswith("Submitted")
|
||||
|
||||
58
tests/execution/test_intake.py
Normal file
58
tests/execution/test_intake.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Intake: what a message that starts something is asking for."""
|
||||
|
||||
import json
|
||||
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer, NoDecider
|
||||
from app.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
|
||||
|
||||
|
||||
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
|
||||
folder = tmp_path / "deliverables" / "cases" / case / "intent"
|
||||
folder.mkdir(parents=True)
|
||||
intents = [
|
||||
{"version": n, "workflow_request": f"Design thing {n}.", "status": status}
|
||||
for n, status in enumerate(statuses, start=1)
|
||||
]
|
||||
(folder / "history.json").write_text(
|
||||
json.dumps({"customer_case": case, "intents": intents})
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
|
||||
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
|
||||
|
||||
assert reading.kind == NEW_REQUEST and not reading.judged
|
||||
|
||||
|
||||
async def test_a_confident_question_is_read_as_one(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert reading.kind == QUESTION and reading.judged
|
||||
assert scope == "t1"
|
||||
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
|
||||
assert state.endswith("The user's message: How long does it take?")
|
||||
|
||||
|
||||
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
|
||||
|
||||
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
|
||||
|
||||
|
||||
async def test_a_change_needs_something_delivered_to_change(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
|
||||
delivered(tmp_path, "acme", ["completed", "in_progress"])
|
||||
intake = Intake(decider, tmp_path)
|
||||
|
||||
with_history = await intake.read("Raise availability to 99.99%.", "acme")
|
||||
without = await intake.read("Raise availability to 99.99%.", "new-client")
|
||||
no_case = await intake.read("Raise availability to 99.99%.")
|
||||
|
||||
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
|
||||
assert "- Design thing 1." in decider.asked[0][0]
|
||||
assert "Design thing 2." not in decider.asked[0][0] # still in progress
|
||||
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
|
||||
@@ -308,3 +308,103 @@ async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
|
||||
# --- each advisor reviews its pillar's checklist ----------------------------------------
|
||||
|
||||
|
||||
def reviewed(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [
|
||||
Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": f"Use strong defaults for {pillar}.",
|
||||
"checklist": [
|
||||
{
|
||||
"item": f"{pillar} item one | with a bar",
|
||||
"status": "addressed",
|
||||
"note": "Covered by the first recommendation.",
|
||||
},
|
||||
{
|
||||
"item": f"{pillar} item two",
|
||||
"status": "not_applicable",
|
||||
"note": "An organisation's process, not a design decision.",
|
||||
},
|
||||
{"item": "malformed", "status": "maybe", "note": "dropped"},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
|
||||
from app.tools.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
# The design advisor is given the advice, not the review: it does not copy it.
|
||||
assert "item one" not in result.message
|
||||
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
|
||||
review = memory[CHECKLIST_KEY]["security"]
|
||||
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
|
||||
|
||||
|
||||
async def test_the_review_is_written_into_the_deliverable_by_code():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.adk.runtime import Submitted
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working=memory)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
call = sample_submission(kind)
|
||||
|
||||
first = await submit.handler(**call.args)
|
||||
# A Revision submits content that already carries the section.
|
||||
again = await submit.handler(
|
||||
content=first.outcome.content, attachments=call.args["attachments"]
|
||||
)
|
||||
|
||||
assert isinstance(first.outcome, Submitted)
|
||||
content = first.outcome.content
|
||||
assert f"## {REVIEW_HEADING}" in content and "### security" in content
|
||||
assert (
|
||||
"| security item one \\| with a bar | Addressed | Covered by the first "
|
||||
"recommendation. |"
|
||||
) in content
|
||||
assert "| security item two | Does not apply |" in content
|
||||
assert "malformed" not in content
|
||||
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
|
||||
|
||||
|
||||
async def test_with_no_review_nothing_is_added():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working={})
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
|
||||
result = await submit.handler(**sample_submission(kind).args)
|
||||
|
||||
assert REVIEW_HEADING not in result.outcome.content
|
||||
|
||||
@@ -689,13 +689,11 @@ def test_a_non_functional_requirement_must_list_its_conditions():
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
)
|
||||
assert missing == (
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"requirements.json: NFR-1 {needs}",
|
||||
f"requirements.json: NFR-2 {needs}",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_recommendation_must_list_its_conditions():
|
||||
@@ -709,7 +707,61 @@ def test_a_recommendation_must_list_its_conditions():
|
||||
{"recommendations.json": json.dumps(recommendations)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"recommendations.json: DR-1 {needs}"
|
||||
]
|
||||
|
||||
|
||||
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
|
||||
"""An outcome measured in operation is recorded, and no Check is asked for it."""
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Pages load within one second at p95.",
|
||||
"conditions": [],
|
||||
"outside_code": ["Measured p95 page load is under one second."],
|
||||
}
|
||||
)
|
||||
assert missing == (f"recommendations.json: DR-1 {needs}",)
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
|
||||
accepted = check_submission(
|
||||
K.REQUIREMENTS,
|
||||
args["content"],
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert accepted == ()
|
||||
assert gaps == ()
|
||||
|
||||
|
||||
def test_a_constraint_with_conditions_still_needs_a_check():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Two zones, 99.9%.",
|
||||
"conditions": ["The service runs in two zones."],
|
||||
"outside_code": ["Measured availability is 99.9%."],
|
||||
}
|
||||
)
|
||||
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert gaps == ("Requirement NFR-2 has no Check",)
|
||||
|
||||
@@ -529,6 +529,7 @@ async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
||||
"asks_change": Answer(0.93, 0.86),
|
||||
"change_target": Answer("requirements", 0.9),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
"already_agreed": Answer(0.03, 0.94),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
@@ -755,3 +756,46 @@ async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
||||
assert asked == MAX_IDLE_ASKS + 1
|
||||
assert "asked" in result.state["warnings"][-1]
|
||||
assert "no work done in between" in result.state["warnings"][-1]
|
||||
|
||||
|
||||
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
|
||||
tmp_path,
|
||||
):
|
||||
"""Evaluation case 04: a requirement said again in reply to a later question."""
|
||||
from app.nodes.router import Readings, TurnView
|
||||
from app.states.ledger import Ledger
|
||||
|
||||
ledger = Ledger()
|
||||
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
view = TurnView(
|
||||
execution_id="e1",
|
||||
pre_approved=False,
|
||||
ledger=ledger,
|
||||
deliverables={
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
}
|
||||
},
|
||||
pending_questions=[],
|
||||
outcomes=[],
|
||||
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
|
||||
)
|
||||
asks = {
|
||||
"asks_change": Answer(0.9, 0.8),
|
||||
"change_target": Answer("requirements", 0.95),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
|
||||
restated, changed = Readings(), Readings()
|
||||
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
|
||||
await Router(decider, None, Thresholds())._read_message(view, restated)
|
||||
await Router(
|
||||
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
|
||||
)._read_message(view, changed)
|
||||
|
||||
assert restated.change == "close"
|
||||
assert changed.change == "act"
|
||||
state, questions, _ = decider.asked[0]
|
||||
assert "The agreed requirements:" in state and "NFR-1" in state
|
||||
assert "already_agreed" in questions
|
||||
|
||||
Reference in New Issue
Block a user