feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -105,6 +105,7 @@ def make_execution_nodes(
|
||||
skill_versions: Mapping[str, str] | None = None,
|
||||
orchestrator: OrchestratorPolicy | None = None,
|
||||
submission_review: SubmissionReview | None = None,
|
||||
confirm_drafts: bool = False,
|
||||
) -> ExecutionNodes:
|
||||
tool_factory = tool_factory or _no_tools
|
||||
# How an Orchestrator turn is decided. The default drives the tools from the
|
||||
@@ -180,6 +181,7 @@ def make_execution_nodes(
|
||||
notes = list(state.get("notes", []))
|
||||
pending_permission = state.get("pending_permission")
|
||||
warnings = list(state.get("warnings", []))
|
||||
shown = list(state.get("drafts_shown", []))
|
||||
drafts = dict(state.get("drafts", {}))
|
||||
|
||||
outcomes = state.get("outcomes", [])
|
||||
@@ -217,6 +219,9 @@ def make_execution_nodes(
|
||||
"content": record["draft"],
|
||||
"attachments": record.get("draft_attachments", {}),
|
||||
}
|
||||
# The user is being shown this Deliverable.
|
||||
if kind.value not in shown:
|
||||
shown.append(kind.value)
|
||||
questions = record.get("questions") or [
|
||||
(
|
||||
f"The {record['role']} stopped: {record['summary']} "
|
||||
@@ -229,6 +234,7 @@ def make_execution_nodes(
|
||||
update: dict[str, Any] = {
|
||||
"deliverables": deliverables,
|
||||
"drafts": drafts,
|
||||
"drafts_shown": shown,
|
||||
"pending_questions": pending_questions,
|
||||
"pending_permission": pending_permission,
|
||||
"outcomes_applied": len(outcomes),
|
||||
@@ -434,6 +440,11 @@ def make_execution_nodes(
|
||||
working=working,
|
||||
library=task_input.get("library"),
|
||||
review=review if submission_review else None,
|
||||
# With a user present, their first sight of a Deliverable comes
|
||||
# before it is submitted, whatever the Specialist intends.
|
||||
confirm_first=confirm_drafts
|
||||
and not task_input["context"].get("pre_approved", True)
|
||||
and not task_input.get("shown"),
|
||||
),
|
||||
refusals,
|
||||
role.name,
|
||||
@@ -559,7 +570,7 @@ def make_execution_nodes(
|
||||
|
||||
async def finalize(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
_write_outputs(output_root, state)
|
||||
_write_outputs(output_root, state, "completed", ledger)
|
||||
close(state, "completed", has_files=True)
|
||||
await record_progress(state, ledger, "completed")
|
||||
await record_lessons(state, ledger, "completed")
|
||||
@@ -567,7 +578,10 @@ def make_execution_nodes(
|
||||
|
||||
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
close(state, "completed_with_warnings", has_files=False)
|
||||
# What was produced is kept, marked as unfinished: a person deciding what to
|
||||
# do next needs to read it, and it is the evidence of what went wrong.
|
||||
wrote = _write_outputs(output_root, state, "completed_with_warnings", ledger)
|
||||
close(state, "completed_with_warnings", has_files=wrote)
|
||||
await record_progress(state, ledger, "completed_with_warnings")
|
||||
await record_lessons(state, ledger, "completed_with_warnings")
|
||||
return {"status": "completed_with_warnings"}
|
||||
@@ -706,6 +720,9 @@ def _specialist_input(
|
||||
"attempt": attempt,
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
"working": dict(state.get("working", {}).get(kind.value, {})),
|
||||
# Whether the user has been shown a draft of this Deliverable. Answering some
|
||||
# other question about it, such as what to do after it stopped, is not that.
|
||||
"shown": kind.value in state.get("drafts_shown", []),
|
||||
"context": {
|
||||
"workflow_request": state["workflow_request"],
|
||||
"pre_approved": bool(state.get("pre_approved")),
|
||||
@@ -762,7 +779,14 @@ def automatic_reply(
|
||||
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
|
||||
) -> dict[str, Any]:
|
||||
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
|
||||
questions from scripted (pattern, answer) pairs where one matches."""
|
||||
questions from scripted answers where one matches.
|
||||
|
||||
A scripted answer is (pattern, answer) or (pattern, answer, asker). It answers a
|
||||
question its pattern matches; with an asker, only when that pattern also matches
|
||||
who is asking (a Deliverable kind, or the orchestrator). The first that applies
|
||||
wins. So facts written for requirements discovery are not given to a later
|
||||
Specialist that only wants its draft confirmed.
|
||||
"""
|
||||
if "approval" in prompt:
|
||||
return {
|
||||
"approval": {"phase": prompt["approval"]["phase"], "decision": "approve"}
|
||||
@@ -772,11 +796,13 @@ def automatic_reply(
|
||||
lines = []
|
||||
unanswered = False
|
||||
for item in prompt.get("questions", []):
|
||||
asker = str(item.get("from", ""))
|
||||
answer = next(
|
||||
(
|
||||
scripted
|
||||
for pattern, scripted in scripted_answers
|
||||
if re.search(pattern, item["question"], re.IGNORECASE)
|
||||
scripted[1]
|
||||
for scripted in scripted_answers
|
||||
if re.search(scripted[0], item["question"], re.IGNORECASE)
|
||||
and (len(scripted) < 3 or re.search(scripted[2], asker, re.IGNORECASE))
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -839,13 +865,21 @@ def _case(state: ExecutionState) -> str:
|
||||
return state.get("customer_case") or UNASSIGNED_CASE
|
||||
|
||||
|
||||
def _write_outputs(output_root: Path, state: ExecutionState) -> None:
|
||||
"""Write a finished Execution's files under its Customer case, with a manifest
|
||||
that ties the request to what was generated for it."""
|
||||
def _write_outputs(
|
||||
output_root: Path, state: ExecutionState, status: str, ledger: Ledger
|
||||
) -> bool:
|
||||
"""Write an Execution's files under its Customer case, with a manifest that ties
|
||||
the request to what was generated for it. Returns whether anything was written.
|
||||
|
||||
An Execution that ended with warnings is written too. Its manifest carries the
|
||||
warnings and each Deliverable's status, so nothing unapproved reads as final.
|
||||
"""
|
||||
customer_case = _case(state)
|
||||
execution_id = state["execution_id"]
|
||||
out_dir = execution_dir(output_root, execution_id, customer_case)
|
||||
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
|
||||
if not deliverables:
|
||||
return False
|
||||
files: dict[str, str] = {}
|
||||
by_kind: dict[str, dict[str, Any]] = {}
|
||||
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
|
||||
@@ -856,7 +890,12 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
|
||||
files[document] = deliverable["content"]
|
||||
attachments = dict(deliverable.get("attachments", {}))
|
||||
files.update(attachments)
|
||||
by_kind[kind] = {"document": document, "attachments": sorted(attachments)}
|
||||
by_kind[kind] = {
|
||||
"document": document,
|
||||
"attachments": sorted(attachments),
|
||||
# submitted, or back in draft after something it depends on was reopened
|
||||
"status": ledger.status(DeliverableKind(kind)).value,
|
||||
}
|
||||
for name, content in files.items():
|
||||
path = out_dir / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -868,7 +907,11 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
|
||||
"session_id": state.get("session_id"),
|
||||
"intent_version": state.get("intent_version"),
|
||||
"workflow_request": state["workflow_request"],
|
||||
"status": "completed",
|
||||
"status": status,
|
||||
"warnings": list(state.get("warnings", [])),
|
||||
"phases": {
|
||||
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
|
||||
},
|
||||
"solution_guide": GUIDE_FILENAME if guide else None,
|
||||
"deliverables": by_kind,
|
||||
}
|
||||
@@ -876,3 +919,4 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
|
||||
(out_dir / MANIFEST_FILENAME).write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user