Case 04, the first interactive evaluation, stopped at 100 Orchestrator turns on two router faults: - A reply that quoted the agent's question back was judged on that question. A reply now reaches the router with quoted questions and Q:/A: labels removed, and the questions are given to the decider as background. - At a Revision limit the reply was read as another change request for the same Deliverable, and the refusal repeated. The user is now told once, the reply is taken as the answer, that Deliverable is not reopened again, and the Execution carries on with it as it stands. - Any run that asks the user four times running with no work done in between now ends with a warning. The validator is told to confirm with the user in an interactive Execution. Plans record the fixes and the rerun, which did not finish: both fixes held, and a third fault appeared that is not fixed here. A scripted answer restating a requirement, given to a draft-confirmation question, reopens requirements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""The checkpointed state of one Execution. Plain, JSON-compatible data only."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import operator
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, TypedDict
|
|
|
|
# A Customer case id becomes a directory name, so it is a short lowercase slug.
|
|
CUSTOMER_CASE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
|
|
# Executions started without a Customer case are grouped here, so every Execution's
|
|
# files sit under a case.
|
|
UNASSIGNED_CASE = "unassigned"
|
|
|
|
|
|
def check_customer_case(customer_case: object) -> str:
|
|
"""The Customer case id, defaulting to UNASSIGNED_CASE; a ValueError when it is
|
|
not a safe slug."""
|
|
if customer_case is None:
|
|
return UNASSIGNED_CASE
|
|
if not isinstance(customer_case, str) or not CUSTOMER_CASE.match(customer_case):
|
|
raise ValueError(
|
|
"customer_case must be 1 to 64 lowercase letters, digits or hyphens, "
|
|
f"starting and ending with a letter or digit; got {customer_case!r}"
|
|
)
|
|
return customer_case
|
|
|
|
|
|
def case_dir(output_root: Path, customer_case: str) -> Path:
|
|
"""Everything generated for one Customer case."""
|
|
return output_root / "deliverables" / "cases" / customer_case
|
|
|
|
|
|
def execution_dir(output_root: Path, execution_id: str, customer_case: str) -> Path:
|
|
"""Where one Execution's files go, under its Customer case."""
|
|
return case_dir(output_root, customer_case) / "executions" / execution_id
|
|
|
|
|
|
def lessons_file(output_root: Path, execution_id: str, customer_case: str) -> Path:
|
|
return (
|
|
output_root / "lessons" / "candidates" / customer_case / f"{execution_id}.json"
|
|
)
|
|
|
|
|
|
def merge_by_key(
|
|
current: dict[str, dict[str, Any]] | None, update: dict[str, dict[str, Any]] | None
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""Parallel Specialist tasks each write their own Deliverable's entry."""
|
|
return {**(current or {}), **(update or {})}
|
|
|
|
|
|
class ExecutionState(TypedDict, total=False):
|
|
execution_id: str
|
|
session_id: str
|
|
customer_case: str # the Customer case this Execution belongs to
|
|
intent_version: int # which statement of the case's intent started this Execution
|
|
workflow_request: str
|
|
pre_approved: bool
|
|
allow_cloud_read: bool | None
|
|
|
|
ledger: dict[str, Any] # Ledger.to_dict()
|
|
deliverables: dict[str, dict[str, Any]] # kind -> {content, attachments}
|
|
|
|
conversation: list[dict[str, str]] # {"from": "user" | "orchestrator", "text"}
|
|
notes: list[str] # facts for the Orchestrator's next turn
|
|
pending_questions: list[dict[str, Any]] # {kind, questions} not yet asked
|
|
awaiting_questions: list[dict[str, Any]] # {kind, questions} asked, not answered
|
|
answers: dict[str, list[dict[str, Any]]] # kind -> [{questions, reply}]
|
|
drafts: dict[
|
|
str, dict[str, Any]
|
|
] # kind -> {content, attachments} awaiting the user
|
|
# kind -> working memory its tools keep across attempts (e.g. pillar advice)
|
|
working: Annotated[dict[str, dict[str, Any]], merge_by_key]
|
|
pending_permission: dict[str, Any] | None # {tier, requested_by}
|
|
prompt: dict[str, Any] | None # what the user is being asked
|
|
|
|
# Specialist outcomes, appended concurrently by parallel Specialist tasks and
|
|
# applied to the Ledger by the next Orchestrator turn.
|
|
outcomes: Annotated[list[dict[str, Any]], operator.add]
|
|
outcomes_applied: int
|
|
refusals: list[dict[str, Any]] # the Orchestrator's refused tool calls
|
|
events_recorded: int
|
|
|
|
# Routing (ADR-0007).
|
|
last_reply: dict[str, Any] | None # the user's message, until a turn routes it
|
|
pending_change: dict[str, Any] | None # a change the user was asked to confirm
|
|
awaiting_gap_decision: bool # the user was asked whether to close gaps
|
|
validation_settled: int # the validation attempt already decided on
|
|
awaiting_limit_decision: bool # the user was told a Revision limit was reached
|
|
revision_limit_asked: list[str] # Deliverables the user has been told that about
|
|
idle_asks: int # questions put to the user running, with nothing changing between
|
|
ask_fingerprint: str # the Ledger's state when the user was last asked
|
|
decisions: list[dict[str, Any]] # every routing decision, in order
|
|
|
|
turns: int
|
|
status: str # running | completed | completed_with_warnings
|
|
warnings: list[str]
|
|
scripted_answers: list[list[str]] # [regex, answer], pre-approved runs only
|