feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -199,9 +199,11 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
|
||||
assert len(cases) >= 2
|
||||
for case in cases:
|
||||
assert case.required_products
|
||||
for pattern, answer in case.scripted_answers:
|
||||
for pattern, answer, *asker in case.scripted_answers:
|
||||
re.compile(pattern)
|
||||
assert answer
|
||||
for who in asker:
|
||||
re.compile(who)
|
||||
|
||||
|
||||
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
|
||||
@@ -528,3 +530,87 @@ async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_pass
|
||||
"judge unavailable: Jev gave no answer"
|
||||
)
|
||||
assert not check(result, "recommendations_quality").passed
|
||||
|
||||
|
||||
# --- the scripted user (evaluation case 04) --------------------------------------------
|
||||
|
||||
|
||||
def test_a_scripted_fact_is_given_only_to_who_it_was_written_for():
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
|
||||
scripted = [
|
||||
["region|location", "europe-west1.", "requirements"],
|
||||
["approve|confirm", "Approved. Please proceed."],
|
||||
]
|
||||
|
||||
def reply(asker: str, question: str) -> str:
|
||||
prompt = {"questions": [{"from": asker, "question": question}]}
|
||||
return automatic_reply(prompt, scripted)["reply"]
|
||||
|
||||
assert reply("requirements", "Which region?").endswith("A: europe-west1.")
|
||||
# A later Specialist confirming a draft that mentions the region gets an approval.
|
||||
assert reply(
|
||||
"validation_results", "Region is europe-west1. Please confirm my draft."
|
||||
).endswith("A: Approved. Please proceed.")
|
||||
# With no answer for it, the reply says no user is available.
|
||||
assert "No user is available" in reply("product_selection", "Which region?")
|
||||
|
||||
|
||||
def test_case_04_answers_a_draft_confirmation_with_an_approval():
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
from eval.scoring import load_cases
|
||||
|
||||
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
|
||||
case = next(c for c in cases if c.id == "case-04-interactive-web-app")
|
||||
|
||||
def reply(asker: str, question: str) -> str:
|
||||
prompt = {"questions": [{"from": asker, "question": question}]}
|
||||
return automatic_reply(prompt, [list(a) for a in case.scripted_answers])[
|
||||
"reply"
|
||||
]
|
||||
|
||||
question = "I checked encryption, GDPR and password rotation. Please confirm."
|
||||
assert reply("validation_results", question).endswith(
|
||||
"A: Approved. Please proceed."
|
||||
)
|
||||
assert "europe-west1" in reply("requirements", "Which region should it run in?")
|
||||
|
||||
|
||||
# --- calibrating the judges ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_calibration_reports_agreement_stability_and_between_judges():
|
||||
from eval.calibrate_judge import report
|
||||
|
||||
items = [
|
||||
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": 1},
|
||||
{"id": "b", "document": "d.md", "kind": "graded", "text": "?", "label": 0.67},
|
||||
{"id": "c", "document": "d.md", "kind": "yes_no", "text": "?", "label": None},
|
||||
]
|
||||
by_judge = {
|
||||
"jev": {"a": [1.0, 1.0], "b": [0.33, 0.67], "c": [0.0, 0.0]},
|
||||
"gemini": {"a": [1.0, 1.0], "b": [0.67, 0.67], "c": [1.0, 1.0]},
|
||||
}
|
||||
|
||||
result = report(items, by_judge)
|
||||
|
||||
assert result["questions"] == 3 and result["labelled"] == 2
|
||||
assert result["judges"]["jev"]["agreement_with_labels"] == 0.5
|
||||
assert result["judges"]["gemini"]["agreement_with_labels"] == 1.0
|
||||
assert result["judges"]["jev"]["stability"] == 0.67
|
||||
assert result["judges"]["gemini"]["stability"] == 1.0
|
||||
assert result["between_judges"] == 0.33
|
||||
|
||||
|
||||
def test_calibration_without_labels_still_measures_stability():
|
||||
from eval.calibrate_judge import report
|
||||
|
||||
items = [
|
||||
{"id": "a", "document": "d.md", "kind": "yes_no", "text": "?", "label": None}
|
||||
]
|
||||
|
||||
result = report(items, {"jev": {"a": [1.0, 1.0, 1.0]}})
|
||||
|
||||
assert result["labelled"] == 0
|
||||
assert result["judges"]["jev"]["agreement_with_labels"] is None
|
||||
assert result["judges"]["jev"]["stability"] == 1.0
|
||||
|
||||
Reference in New Issue
Block a user