Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
4.4 KiB
Python
101 lines
4.4 KiB
Python
"""The checkpointed state of one Execution. Plain, JSON-compatible data only."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import operator
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, TypedDict
|
|
|
|
# A Customer case id becomes a directory name, so it is a short lowercase slug.
|
|
CUSTOMER_CASE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
|
|
# Executions started without a Customer case are grouped here, so every Execution's
|
|
# files sit under a case.
|
|
UNASSIGNED_CASE = "unassigned"
|
|
|
|
|
|
def check_customer_case(customer_case: object) -> str:
|
|
"""The Customer case id, defaulting to UNASSIGNED_CASE; a ValueError when it is
|
|
not a safe slug."""
|
|
if customer_case is None:
|
|
return UNASSIGNED_CASE
|
|
if not isinstance(customer_case, str) or not CUSTOMER_CASE.match(customer_case):
|
|
raise ValueError(
|
|
"customer_case must be 1 to 64 lowercase letters, digits or hyphens, "
|
|
f"starting and ending with a letter or digit; got {customer_case!r}"
|
|
)
|
|
return customer_case
|
|
|
|
|
|
def case_dir(output_root: Path, customer_case: str) -> Path:
|
|
"""Everything generated for one Customer case."""
|
|
return output_root / "deliverables" / "cases" / customer_case
|
|
|
|
|
|
def execution_dir(output_root: Path, execution_id: str, customer_case: str) -> Path:
|
|
"""Where one Execution's files go, under its Customer case."""
|
|
return case_dir(output_root, customer_case) / "executions" / execution_id
|
|
|
|
|
|
def lessons_file(output_root: Path, execution_id: str, customer_case: str) -> Path:
|
|
return (
|
|
output_root / "lessons" / "candidates" / customer_case / f"{execution_id}.json"
|
|
)
|
|
|
|
|
|
def merge_by_key(
|
|
current: dict[str, dict[str, Any]] | None, update: dict[str, dict[str, Any]] | None
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Parallel Specialist tasks each write their own Deliverable's entry."""
|
|
return {**(current or {}), **(update or {})}
|
|
|
|
|
|
class ExecutionState(TypedDict, total=False):
|
|
execution_id: str
|
|
session_id: str
|
|
customer_case: str # the Customer case this Execution belongs to
|
|
intent_version: int # which statement of the case's intent started this Execution
|
|
workflow_request: str
|
|
pre_approved: bool
|
|
allow_cloud_read: bool | None
|
|
|
|
ledger: dict[str, Any] # Ledger.to_dict()
|
|
deliverables: dict[str, dict[str, Any]] # kind -> {content, attachments}
|
|
|
|
conversation: list[dict[str, str]] # {"from": "user" | "orchestrator", "text"}
|
|
notes: list[str] # facts for the Orchestrator's next turn
|
|
pending_questions: list[dict[str, Any]] # {kind, questions} not yet asked
|
|
awaiting_questions: list[dict[str, Any]] # {kind, questions} asked, not answered
|
|
answers: dict[str, list[dict[str, Any]]] # kind -> [{questions, reply}]
|
|
drafts_shown: list[str] # Deliverables a draft of which the user has been shown
|
|
drafts: dict[
|
|
str, dict[str, Any]
|
|
] # kind -> {content, attachments} awaiting the user
|
|
# kind -> working memory its tools keep across attempts (e.g. pillar advice)
|
|
working: Annotated[dict[str, dict[str, Any]], merge_by_key]
|
|
pending_permission: dict[str, Any] | None # {tier, requested_by}
|
|
prompt: dict[str, Any] | None # what the user is being asked
|
|
|
|
# Specialist outcomes, appended concurrently by parallel Specialist tasks and
|
|
# applied to the Ledger by the next Orchestrator turn.
|
|
outcomes: Annotated[list[dict[str, Any]], operator.add]
|
|
outcomes_applied: int
|
|
refusals: list[dict[str, Any]] # the Orchestrator's refused tool calls
|
|
events_recorded: int
|
|
|
|
# Routing (ADR-0007).
|
|
last_reply: dict[str, Any] | None # the user's message, until a turn routes it
|
|
pending_change: dict[str, Any] | None # a change the user was asked to confirm
|
|
awaiting_gap_decision: bool # the user was asked whether to close gaps
|
|
validation_settled: int # the validation attempt already decided on
|
|
awaiting_limit_decision: bool # the user was told a Revision limit was reached
|
|
revision_limit_asked: list[str] # Deliverables the user has been told that about
|
|
idle_asks: int # questions put to the user running, with nothing changing between
|
|
ask_fingerprint: str # the Ledger's state when the user was last asked
|
|
decisions: list[dict[str, Any]] # every routing decision, in order
|
|
|
|
turns: int
|
|
status: str # running | completed | completed_with_warnings
|
|
warnings: list[str]
|
|
scripted_answers: list[list[str]] # [regex, answer], pre-approved runs only
|