feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -24,7 +24,7 @@ import os
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
@@ -58,8 +58,12 @@ RESULTS_DIR = ROOT / "results"
|
||||
class CountingRuntime:
|
||||
"""Counts agent runs and their outcomes per role, with a timeline of runs."""
|
||||
|
||||
def __init__(self, inner: AgentRuntime) -> None:
|
||||
def __init__(
|
||||
self, inner: AgentRuntime, on_run: Callable[[], None] | None = None
|
||||
) -> None:
|
||||
self._inner = inner
|
||||
# Called after every agent run, so a run cut short leaves a record behind.
|
||||
self._on_run = on_run
|
||||
self._started = time.monotonic()
|
||||
self.runs: Counter[str] = Counter()
|
||||
self.outcomes: Counter[str] = Counter()
|
||||
@@ -81,6 +85,8 @@ class CountingRuntime:
|
||||
"outcome": type(outcome).__name__,
|
||||
}
|
||||
)
|
||||
if self._on_run:
|
||||
self._on_run()
|
||||
return outcome
|
||||
|
||||
|
||||
@@ -145,9 +151,30 @@ async def run_case(
|
||||
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
|
||||
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
|
||||
)
|
||||
runtime = CountingRuntime(adk_runtime)
|
||||
recorder = InMemoryRecorder()
|
||||
started = time.monotonic()
|
||||
progress_path = run_dir / f"{case.id}-progress.json"
|
||||
|
||||
def write_progress() -> None:
|
||||
"""What the run has cost so far. Overwritten after every agent run and left
|
||||
in place, so an interrupted run still has its time, tokens and timeline."""
|
||||
progress_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"case_id": case.id,
|
||||
"finished": False,
|
||||
"seconds": round(time.monotonic() - started, 1),
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"tokens": token_report(adk_runtime.usage),
|
||||
"timeline": runtime.timeline,
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
runtime = CountingRuntime(adk_runtime, on_run=write_progress)
|
||||
recorder = InMemoryRecorder()
|
||||
async with open_assembly(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
@@ -166,6 +193,13 @@ async def run_case(
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
write_progress()
|
||||
progress_path.write_text(
|
||||
progress_path.read_text(encoding="utf-8").replace(
|
||||
'"finished": false', '"finished": true', 1
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
runner = TerraformRunner(
|
||||
binary=settings.TERRAFORM_BINARY,
|
||||
|
||||
Reference in New Issue
Block a user