feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

View File

@@ -35,6 +35,7 @@ from app.adk.runtime import (
from app.checkpointing import open_checkpointer
from app.config import Settings
from app.customer_case import record_intent
from app.nodes.intake import Intake
from app.nodes.router import (
REPLY_WRITER,
OrchestratorPolicy,
@@ -52,6 +53,7 @@ from app.tools.execution import ORCHESTRATOR
from app.tools.jev import JevDecider, JevLimits
from app.tools.local import TerraformRunner, local_tools
from app.tools.mcp import McpCatalog, auth_from_config
from app.tools.requirements_review import RequirementsReview
from app.tools.schema_review import Reviews, SchemaReview
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH
@@ -88,6 +90,7 @@ class ExecutionEngine:
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
) -> None:
required = {
ORCHESTRATOR,
@@ -109,6 +112,7 @@ class ExecutionEngine:
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
confirm_drafts=confirm_drafts,
).compile(checkpointer=checkpointer)
async def start(
@@ -120,9 +124,9 @@ class ExecutionEngine:
pre_approved: bool = False,
allow_cloud_read: bool | None = None,
execution_id: str | None = None,
scripted_answers: Sequence[tuple[str, str]] = (),
scripted_answers: Sequence[Sequence[str]] = (),
) -> Interaction:
"""Start an Execution. `scripted_answers` are (regex, answer) pairs a
"""Start an Execution. `scripted_answers` are (regex, answer[, asker]) entries a
pre-approved Execution uses to answer questions, for evaluation runs."""
customer_case = check_customer_case(customer_case)
execution_id = execution_id or str(uuid.uuid4())
@@ -156,6 +160,7 @@ class ExecutionEngine:
"awaiting_questions": [],
"answers": {},
"drafts": {},
"drafts_shown": [],
"working": {},
"pending_permission": None,
"prompt": None,
@@ -175,9 +180,7 @@ class ExecutionEngine:
"turns": 0,
"status": "running",
"warnings": [],
"scripted_answers": [
[pattern, answer] for pattern, answer in scripted_answers
],
"scripted_answers": [list(entry) for entry in scripted_answers],
}
await self._graph.ainvoke(initial, self._config(execution_id))
return await self.interaction(execution_id)
@@ -236,6 +239,7 @@ class Assembly:
roles: dict[str, RoleSpec]
mcp_gaps: dict[str, str]
decider: Decider = field(default_factory=NoDecider)
intake: Intake | None = None # reads a message that starts something
def open_decider(settings: Settings) -> Decider:
@@ -334,6 +338,11 @@ async def open_assembly(
if bundle.terraform_providers:
# In code: every setting a Check names exists on its resource type.
reviews.append(SchemaReview(runner))
if settings.JEV_REQUIREMENTS_REVIEW and isinstance(decider, JevDecider):
# Optional, and off by default: Jev says whether a requirement names a product.
reviews.append(
RequirementsReview(decider, act=settings.JEV_REQUIREMENTS_THRESHOLD)
)
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider):
# Optional, and off by default: Jev says which Check verifies each condition.
reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD))
@@ -354,9 +363,21 @@ async def open_assembly(
},
orchestrator=orchestrator,
submission_review=Reviews(reviews) if reviews else None,
confirm_drafts=settings.CONFIRM_DRAFTS,
)
try:
yield Assembly(engine, bundle, roles, dict(catalog.gaps), decider)
yield Assembly(
engine,
bundle,
roles,
dict(catalog.gaps),
decider,
Intake(
decider,
output_root or settings.BASE_DIR,
act=settings.JEV_ACT_THRESHOLD,
),
)
finally:
if isinstance(decider, JevDecider):
await decider.aclose()