feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,10 @@
|
||||
A new task starts an Execution; a message on an input-required task answers the
|
||||
Execution's prompt. Approvals and permissions count only as structured data parts,
|
||||
never as free text.
|
||||
|
||||
Before a new task starts anything its message is read (app/nodes/intake.py): a
|
||||
question is answered and starts nothing, and a change to delivered work is started
|
||||
as a new request with a note saying so, until Change requests are built.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -25,6 +29,7 @@ from google.protobuf import json_format
|
||||
|
||||
from app.agent import ExecutionEngine, Interaction
|
||||
from app.nodes.execution_nodes import GUIDE_FILENAME
|
||||
from app.nodes.intake import ABOUT, CHANGE_NOTE, CHANGE_REQUEST, QUESTION, Intake
|
||||
from app.presentation import describe_prompt
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
@@ -34,14 +39,17 @@ STRUCTURED_REPLY_KEYS = ("approval", "permission")
|
||||
|
||||
|
||||
class SolutionArchitectExecutor(AgentExecutor):
|
||||
def __init__(self, engine: ExecutionEngine) -> None:
|
||||
def __init__(self, engine: ExecutionEngine, intake: Intake | None = None) -> None:
|
||||
self._engine = engine
|
||||
self._intake = intake
|
||||
|
||||
async def execute(self, context: RequestContext, event_queue: EventQueue) -> None:
|
||||
task_id = context.task_id or ""
|
||||
updater = TaskUpdater(event_queue, task_id, context.context_id or "")
|
||||
new = context.current_task is None
|
||||
note: str | None = None
|
||||
try:
|
||||
if context.current_task is None:
|
||||
if new:
|
||||
history = [context.message] if context.message is not None else []
|
||||
await event_queue.enqueue_event(
|
||||
new_task(
|
||||
@@ -53,6 +61,27 @@ class SolutionArchitectExecutor(AgentExecutor):
|
||||
)
|
||||
await updater.start_work()
|
||||
options = _options(context)
|
||||
# A pre-approved caller is a batch job: it sends work, not questions.
|
||||
if self._intake and not options.get("pre_approved"):
|
||||
reading = await self._intake.read(
|
||||
context.get_user_input(),
|
||||
options.get("customer_case"),
|
||||
scope=task_id,
|
||||
)
|
||||
if reading.kind == QUESTION:
|
||||
await updater.complete(
|
||||
updater.new_agent_message(
|
||||
[
|
||||
new_text_part(ABOUT),
|
||||
new_data_part(
|
||||
{"status": "answered", "intake": QUESTION}
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
return
|
||||
if reading.kind == CHANGE_REQUEST:
|
||||
note = CHANGE_NOTE
|
||||
interaction = await self._engine.start(
|
||||
context.get_user_input(),
|
||||
session_id=context.context_id,
|
||||
@@ -76,7 +105,7 @@ class SolutionArchitectExecutor(AgentExecutor):
|
||||
)
|
||||
)
|
||||
return
|
||||
await self._publish(interaction, updater)
|
||||
await self._publish(interaction, updater, note if new else None)
|
||||
|
||||
async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None:
|
||||
# The checkpoint stays; cancelling only ends the A2A task.
|
||||
@@ -85,12 +114,15 @@ class SolutionArchitectExecutor(AgentExecutor):
|
||||
)
|
||||
await updater.cancel()
|
||||
|
||||
async def _publish(self, interaction: Interaction, updater: TaskUpdater) -> None:
|
||||
async def _publish(
|
||||
self, interaction: Interaction, updater: TaskUpdater, note: str | None = None
|
||||
) -> None:
|
||||
if interaction.status == "awaiting_user" and interaction.prompt is not None:
|
||||
prompt = dict(interaction.prompt)
|
||||
await updater.requires_input(
|
||||
updater.new_agent_message(
|
||||
[
|
||||
*([new_text_part(note)] if note else []),
|
||||
new_text_part(describe_prompt(prompt)),
|
||||
new_data_part({"prompt": prompt}),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user