feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -488,12 +488,21 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert "revision limit" in result.state["warnings"][0]
|
||||
# No Deliverable files are written, but the request stays on record.
|
||||
# What was produced is kept, marked as unfinished, and the request is on record.
|
||||
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
||||
assert not (case / "executions").exists()
|
||||
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
||||
assert intent["status"] == "completed_with_warnings"
|
||||
assert intent["execution"] is None
|
||||
assert intent["execution"] == f"executions/{result.execution_id}"
|
||||
out_dir = case / intent["execution"]
|
||||
manifest = json.loads((out_dir / "execution.json").read_text())
|
||||
assert manifest["status"] == "completed_with_warnings"
|
||||
assert "revision limit" in manifest["warnings"][0]
|
||||
assert manifest["solution_guide"] is None
|
||||
assert not (out_dir / "solution-architecture-guide.md").exists()
|
||||
assert (out_dir / "deliverables" / "requirements.md").is_file()
|
||||
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
|
||||
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
|
||||
assert manifest["phases"]["packaging"] != "approved"
|
||||
|
||||
|
||||
# --- budgets -------------------------------------------------------------------------------------
|
||||
@@ -880,7 +889,10 @@ async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
],
|
||||
"status": "submitted",
|
||||
}
|
||||
assert manifest["status"] == "completed" and manifest["warnings"] == []
|
||||
assert set(manifest["phases"].values()) == {"approved"}
|
||||
for entry in manifest["deliverables"].values():
|
||||
assert (out / entry["document"]).is_file()
|
||||
assert all((out / name).is_file() for name in entry["attachments"])
|
||||
@@ -969,3 +981,92 @@ async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
||||
assert entry["status"] == "in_progress"
|
||||
assert entry["statement"] == "intent/001.md"
|
||||
assert entry["execution_id"] == paused.execution_id
|
||||
|
||||
|
||||
# --- a user sees a Deliverable before it is first submitted -----------------------------
|
||||
|
||||
|
||||
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
|
||||
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()],
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
# It tries to submit straight away, is refused, and asks.
|
||||
[
|
||||
sample_submission(K.REQUIREMENTS),
|
||||
Call(
|
||||
"needs_input",
|
||||
{"questions": ["Please approve the draft."], "draft": "# r"},
|
||||
),
|
||||
],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
refusal = runtime.runs[1].results[0][1]
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
# Once they have replied, the submission is accepted.
|
||||
assert runtime.runs[-2].results[0][1].startswith("Submitted")
|
||||
|
||||
|
||||
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
|
||||
runtime = FakeAgentRuntime(happy_path_turns())
|
||||
|
||||
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
|
||||
"Ingestion", pre_approved=True
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
|
||||
|
||||
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
|
||||
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
|
||||
the Orchestrator and then submitted. The user had answered a question about the
|
||||
Deliverable but had never seen it."""
|
||||
draft = Call(
|
||||
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
|
||||
)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays "the requirements analyst stopped"
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()], # relays the draft
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[], # stops without ending its turn
|
||||
[sample_submission(K.REQUIREMENTS), draft],
|
||||
submits(K.REQUIREMENTS),
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
|
||||
|
||||
first = await engine.start("Ingestion")
|
||||
assert "stopped" in first.prompt["questions"][0]["question"]
|
||||
second = await engine.resume(first.execution_id, {"reply": "Try again."})
|
||||
await engine.resume(first.execution_id, {"reply": "Approved."})
|
||||
|
||||
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
|
||||
assert (
|
||||
analyst[1]
|
||||
.results[0][1]
|
||||
.startswith("Not submitted. The user has not seen this Deliverable")
|
||||
)
|
||||
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
|
||||
assert analyst[2].results[0][1].startswith("Submitted")
|
||||
|
||||
Reference in New Issue
Block a user