feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -308,3 +308,103 @@ async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
|
||||
# --- each advisor reviews its pillar's checklist ----------------------------------------
|
||||
|
||||
|
||||
def reviewed(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [
|
||||
Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": f"Use strong defaults for {pillar}.",
|
||||
"checklist": [
|
||||
{
|
||||
"item": f"{pillar} item one | with a bar",
|
||||
"status": "addressed",
|
||||
"note": "Covered by the first recommendation.",
|
||||
},
|
||||
{
|
||||
"item": f"{pillar} item two",
|
||||
"status": "not_applicable",
|
||||
"note": "An organisation's process, not a design decision.",
|
||||
},
|
||||
{"item": "malformed", "status": "maybe", "note": "dropped"},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
|
||||
from app.tools.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
# The design advisor is given the advice, not the review: it does not copy it.
|
||||
assert "item one" not in result.message
|
||||
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
|
||||
review = memory[CHECKLIST_KEY]["security"]
|
||||
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
|
||||
|
||||
|
||||
async def test_the_review_is_written_into_the_deliverable_by_code():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.adk.runtime import Submitted
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working=memory)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
call = sample_submission(kind)
|
||||
|
||||
first = await submit.handler(**call.args)
|
||||
# A Revision submits content that already carries the section.
|
||||
again = await submit.handler(
|
||||
content=first.outcome.content, attachments=call.args["attachments"]
|
||||
)
|
||||
|
||||
assert isinstance(first.outcome, Submitted)
|
||||
content = first.outcome.content
|
||||
assert f"## {REVIEW_HEADING}" in content and "### security" in content
|
||||
assert (
|
||||
"| security item one \\| with a bar | Addressed | Covered by the first "
|
||||
"recommendation. |"
|
||||
) in content
|
||||
assert "| security item two | Does not apply |" in content
|
||||
assert "malformed" not in content
|
||||
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
|
||||
|
||||
|
||||
async def test_with_no_review_nothing_is_added():
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.pillars import REVIEW_HEADING
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(kind, (), allow_cloud_read=False, working={})
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
|
||||
result = await submit.handler(**sample_submission(kind).args)
|
||||
|
||||
assert REVIEW_HEADING not in result.outcome.content
|
||||
|
||||
Reference in New Issue
Block a user