feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

View File

@@ -488,12 +488,21 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
assert result.status == "completed_with_warnings"
assert "revision limit" in result.state["warnings"][0]
# No Deliverable files are written, but the request stays on record.
# What was produced is kept, marked as unfinished, and the request is on record.
case = tmp_path / "deliverables" / "cases" / "unassigned"
assert not (case / "executions").exists()
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
assert intent["status"] == "completed_with_warnings"
assert intent["execution"] is None
assert intent["execution"] == f"executions/{result.execution_id}"
out_dir = case / intent["execution"]
manifest = json.loads((out_dir / "execution.json").read_text())
assert manifest["status"] == "completed_with_warnings"
assert "revision limit" in manifest["warnings"][0]
assert manifest["solution_guide"] is None
assert not (out_dir / "solution-architecture-guide.md").exists()
assert (out_dir / "deliverables" / "requirements.md").is_file()
# Nothing unapproved reads as final: each Deliverable and Phase has its status.
assert manifest["deliverables"]["requirements"]["status"] == "submitted"
assert manifest["phases"]["packaging"] != "approved"
# --- budgets -------------------------------------------------------------------------------------
@@ -880,7 +889,10 @@ async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
"terraform/main.tf",
"terraform/variables.tf",
],
"status": "submitted",
}
assert manifest["status"] == "completed" and manifest["warnings"] == []
assert set(manifest["phases"].values()) == {"approved"}
for entry in manifest["deliverables"].values():
assert (out / entry["document"]).is_file()
assert all((out / name).is_file() for name in entry["attachments"])
@@ -969,3 +981,92 @@ async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
assert entry["status"] == "in_progress"
assert entry["statement"] == "intent/001.md"
assert entry["execution_id"] == paused.execution_id
# --- a user sees a Deliverable before it is first submitted -----------------------------
async def test_with_a_user_present_a_first_submission_waits_for_their_reply(tmp_path):
"""Evaluation case 04: a Specialist that ran out of budget submitted without asking."""
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()],
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
# It tries to submit straight away, is refused, and asks.
[
sample_submission(K.REQUIREMENTS),
Call(
"needs_input",
{"questions": ["Please approve the draft."], "draft": "# r"},
),
],
submits(K.REQUIREMENTS),
],
}
)
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
first = await engine.start("Ingestion")
refusal = runtime.runs[1].results[0][1]
await engine.resume(first.execution_id, {"reply": "Approved."})
assert refusal.startswith("Not submitted. The user has not seen this Deliverable")
assert first.prompt["questions"][0]["question"] == "Please approve the draft."
# Once they have replied, the submission is accepted.
assert runtime.runs[-2].results[0][1].startswith("Submitted")
async def test_a_pre_approved_execution_is_never_made_to_ask(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
result = await make_engine(tmp_path, runtime, confirm_drafts=True).start(
"Ingestion", pre_approved=True
)
assert result.status == "completed"
async def test_answering_another_question_is_not_being_shown_the_draft(tmp_path):
"""Evaluation case 04, 2026-10-08: a Specialist that stopped, was asked about by
the Orchestrator and then submitted. The user had answered a question about the
Deliverable but had never seen it."""
draft = Call(
"needs_input", {"questions": ["Please approve the draft."], "draft": "# r"}
)
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()], # relays "the requirements analyst stopped"
[delegate(K.REQUIREMENTS)],
[ask_user()], # relays the draft
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
[], # stops without ending its turn
[sample_submission(K.REQUIREMENTS), draft],
submits(K.REQUIREMENTS),
],
}
)
engine = make_engine(tmp_path, runtime, confirm_drafts=True)
first = await engine.start("Ingestion")
assert "stopped" in first.prompt["questions"][0]["question"]
second = await engine.resume(first.execution_id, {"reply": "Try again."})
await engine.resume(first.execution_id, {"reply": "Approved."})
analyst = [run for run in runtime.runs if run.role == specialist(K.REQUIREMENTS)]
assert (
analyst[1]
.results[0][1]
.startswith("Not submitted. The user has not seen this Deliverable")
)
assert second.prompt["questions"][0]["question"] == "Please approve the draft."
assert analyst[2].results[0][1].startswith("Submitted")