feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,11 +8,18 @@ import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.fake_runtime import (
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.roles import load_roles
|
||||
from app.adk.runtime import Answer
|
||||
from app.agent import Assembly, ExecutionEngine
|
||||
from app.config import get_settings
|
||||
from app.main import create_app
|
||||
from app.nodes.intake import Intake
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
@@ -67,7 +74,9 @@ def complete_run_turns() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
def client_for(
|
||||
tmp_path: Path, turns: dict, decider=None
|
||||
) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
@@ -81,7 +90,13 @@ def client_for(tmp_path: Path, turns: dict) -> tuple[TestClient, FakeAgentRuntim
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
yield Assembly(engine, bundle, roles, {"compute": "API not enabled"})
|
||||
yield Assembly(
|
||||
engine,
|
||||
bundle,
|
||||
roles,
|
||||
{"compute": "API not enabled"},
|
||||
intake=Intake(decider, tmp_path) if decider else None,
|
||||
)
|
||||
|
||||
return TestClient(create_app(get_settings(), fake_assembly)), runtime
|
||||
|
||||
@@ -319,3 +334,56 @@ def test_generate_groups_the_execution_under_the_callers_customer_case(tmp_path)
|
||||
assert "customer_case" in refused.json()["error"]
|
||||
assert accepted.status_code == 200, accepted.json()
|
||||
assert (tmp_path / "lessons" / "candidates" / "acme-webhooks").is_dir()
|
||||
|
||||
|
||||
# --- a message that starts something is read first (intake) ----------------------------
|
||||
|
||||
|
||||
def test_a_question_is_answered_and_starts_no_execution(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.97)})
|
||||
client, runtime = client_for(tmp_path, {}, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "What can you do?"}])
|
||||
|
||||
text, data = status_parts(task)
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert "Nothing was started" in text
|
||||
assert data == {"status": "answered", "intake": "question"}
|
||||
assert runtime.runs == []
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
|
||||
|
||||
def test_a_new_request_starts_an_execution_as_before(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("new_request", 0.99)})
|
||||
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
||||
client, runtime = client_for(tmp_path, turns, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "Design an ingestion pipeline."}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
assert "Which region?" in status_parts(task)[0]
|
||||
assert len(runtime.runs) == 1
|
||||
|
||||
|
||||
def test_an_unsure_reading_starts_an_execution(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.5)})
|
||||
turns = {"orchestrator": [[Call("ask_user", {"questions": ["Which region?"]})]]}
|
||||
client, runtime = client_for(tmp_path, turns, decider)
|
||||
with client:
|
||||
task = send(client, [{"text": "hello"}])
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_INPUT_REQUIRED"
|
||||
assert len(runtime.runs) == 1
|
||||
|
||||
|
||||
def test_a_pre_approved_task_is_not_read_as_a_question(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer("question", 0.99)})
|
||||
client, runtime = client_for(tmp_path, complete_run_turns(), decider)
|
||||
with client:
|
||||
task = send(
|
||||
client, [{"text": "What can you do?"}], metadata={"pre_approved": True}
|
||||
)
|
||||
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert decider.asked == []
|
||||
assert runtime.runs
|
||||
|
||||
Reference in New Issue
Block a user