feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

View File

@@ -105,6 +105,7 @@ def make_execution_nodes(
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
# How an Orchestrator turn is decided. The default drives the tools from the
@@ -180,6 +181,7 @@ def make_execution_nodes(
notes = list(state.get("notes", []))
pending_permission = state.get("pending_permission")
warnings = list(state.get("warnings", []))
shown = list(state.get("drafts_shown", []))
drafts = dict(state.get("drafts", {}))
outcomes = state.get("outcomes", [])
@@ -217,6 +219,9 @@ def make_execution_nodes(
"content": record["draft"],
"attachments": record.get("draft_attachments", {}),
}
# The user is being shown this Deliverable.
if kind.value not in shown:
shown.append(kind.value)
questions = record.get("questions") or [
(
f"The {record['role']} stopped: {record['summary']} "
@@ -229,6 +234,7 @@ def make_execution_nodes(
update: dict[str, Any] = {
"deliverables": deliverables,
"drafts": drafts,
"drafts_shown": shown,
"pending_questions": pending_questions,
"pending_permission": pending_permission,
"outcomes_applied": len(outcomes),
@@ -434,6 +440,11 @@ def make_execution_nodes(
working=working,
library=task_input.get("library"),
review=review if submission_review else None,
# With a user present, their first sight of a Deliverable comes
# before it is submitted, whatever the Specialist intends.
confirm_first=confirm_drafts
and not task_input["context"].get("pre_approved", True)
and not task_input.get("shown"),
),
refusals,
role.name,
@@ -559,7 +570,7 @@ def make_execution_nodes(
async def finalize(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
_write_outputs(output_root, state)
_write_outputs(output_root, state, "completed", ledger)
close(state, "completed", has_files=True)
await record_progress(state, ledger, "completed")
await record_lessons(state, ledger, "completed")
@@ -567,7 +578,10 @@ def make_execution_nodes(
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
close(state, "completed_with_warnings", has_files=False)
# What was produced is kept, marked as unfinished: a person deciding what to
# do next needs to read it, and it is the evidence of what went wrong.
wrote = _write_outputs(output_root, state, "completed_with_warnings", ledger)
close(state, "completed_with_warnings", has_files=wrote)
await record_progress(state, ledger, "completed_with_warnings")
await record_lessons(state, ledger, "completed_with_warnings")
return {"status": "completed_with_warnings"}
@@ -706,6 +720,9 @@ def _specialist_input(
"attempt": attempt,
"allow_cloud_read": state.get("allow_cloud_read"),
"working": dict(state.get("working", {}).get(kind.value, {})),
# Whether the user has been shown a draft of this Deliverable. Answering some
# other question about it, such as what to do after it stopped, is not that.
"shown": kind.value in state.get("drafts_shown", []),
"context": {
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
@@ -762,7 +779,14 @@ def automatic_reply(
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
) -> dict[str, Any]:
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
questions from scripted (pattern, answer) pairs where one matches."""
questions from scripted answers where one matches.
A scripted answer is (pattern, answer) or (pattern, answer, asker). It answers a
question its pattern matches; with an asker, only when that pattern also matches
who is asking (a Deliverable kind, or the orchestrator). The first that applies
wins. So facts written for requirements discovery are not given to a later
Specialist that only wants its draft confirmed.
"""
if "approval" in prompt:
return {
"approval": {"phase": prompt["approval"]["phase"], "decision": "approve"}
@@ -772,11 +796,13 @@ def automatic_reply(
lines = []
unanswered = False
for item in prompt.get("questions", []):
asker = str(item.get("from", ""))
answer = next(
(
scripted
for pattern, scripted in scripted_answers
if re.search(pattern, item["question"], re.IGNORECASE)
scripted[1]
for scripted in scripted_answers
if re.search(scripted[0], item["question"], re.IGNORECASE)
and (len(scripted) < 3 or re.search(scripted[2], asker, re.IGNORECASE))
),
None,
)
@@ -839,13 +865,21 @@ def _case(state: ExecutionState) -> str:
return state.get("customer_case") or UNASSIGNED_CASE
def _write_outputs(output_root: Path, state: ExecutionState) -> None:
"""Write a finished Execution's files under its Customer case, with a manifest
that ties the request to what was generated for it."""
def _write_outputs(
output_root: Path, state: ExecutionState, status: str, ledger: Ledger
) -> bool:
"""Write an Execution's files under its Customer case, with a manifest that ties
the request to what was generated for it. Returns whether anything was written.
An Execution that ended with warnings is written too. Its manifest carries the
warnings and each Deliverable's status, so nothing unapproved reads as final.
"""
customer_case = _case(state)
execution_id = state["execution_id"]
out_dir = execution_dir(output_root, execution_id, customer_case)
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
if not deliverables:
return False
files: dict[str, str] = {}
by_kind: dict[str, dict[str, Any]] = {}
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
@@ -856,7 +890,12 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
files[document] = deliverable["content"]
attachments = dict(deliverable.get("attachments", {}))
files.update(attachments)
by_kind[kind] = {"document": document, "attachments": sorted(attachments)}
by_kind[kind] = {
"document": document,
"attachments": sorted(attachments),
# submitted, or back in draft after something it depends on was reopened
"status": ledger.status(DeliverableKind(kind)).value,
}
for name, content in files.items():
path = out_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
@@ -868,7 +907,11 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
"session_id": state.get("session_id"),
"intent_version": state.get("intent_version"),
"workflow_request": state["workflow_request"],
"status": "completed",
"status": status,
"warnings": list(state.get("warnings", [])),
"phases": {
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
},
"solution_guide": GUIDE_FILENAME if guide else None,
"deliverables": by_kind,
}
@@ -876,3 +919,4 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
(out_dir / MANIFEST_FILENAME).write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
return True