feat: what a request does not say is a question for the customer
Some checks failed
validation / verify (push) Failing after 11s

A pillar's checklist item the request gives no information about was marked
"not addressed", which reads as a fault in the design. It is missing
information. The checklist review gains a fourth outcome, to confirm with the
customer, which must carry the question to ask. The questions are written
into the design recommendations and the guide, and to customer-questions.md;
the evaluation's judged check counts such an item as accounted for.

Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's
retention, training, context and rate limits (iteration 5 step 1), and the
owner's labels, which showed the judged question was wrong, not the judges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 19:30:12 +01:00
parent f9cc34817b
commit a48468d477
15 changed files with 375 additions and 17 deletions

View File

@@ -39,12 +39,18 @@ REVIEW_HEADING = "Architecture framework checklist review"
# What a Change request carries over from its Baseline's working memory.
KEPT_FOR_CHANGES = (ADVICE_KEY, CHECKLIST_KEY)
_CHANGE_LIMIT = 6_000
STATUSES = ("addressed", "not_applicable", "not_addressed")
NEEDS_CUSTOMER = "needs_customer_input"
STATUSES = ("addressed", "not_applicable", NEEDS_CUSTOMER, "not_addressed")
_STATUS_WORDS = {
"addressed": "Addressed",
"not_applicable": "Does not apply",
# The item applies, and the customer's request does not say what would settle
# it. That is missing information, not a fault in the design: it is reported
# with the question to put to the customer.
NEEDS_CUSTOMER: "To confirm with the customer",
"not_addressed": "Not addressed",
}
QUESTIONS_HEADING = "Questions for the customer"
AFFECTED = ("not at all", "only indirectly", "directly")
@@ -215,17 +221,29 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
if not content.strip():
return ToolResult("Not submitted: the advice is empty.")
review[:] = [
entries = [
{
"item": " ".join(str(entry.get("item", "")).split()),
"status": str(entry.get("status", "")),
"note": " ".join(str(entry.get("note", "")).split()),
"question": " ".join(str(entry.get("question") or "").split()),
}
for entry in checklist or ()
if isinstance(entry, Mapping)
and str(entry.get("item", "")).strip()
and entry.get("status") in STATUSES
]
unasked = [
entry["item"]
for entry in entries
if entry["status"] == NEEDS_CUSTOMER and not entry["question"]
]
if unasked:
return ToolResult(
"Not submitted: an item to confirm with the customer needs the "
"`question` to put to them. Missing for: " + "; ".join(unasked)
)
review[:] = entries
return ToolResult("Advice submitted.", Submitted(content))
text = {"type": "string"}
@@ -248,8 +266,15 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
"status": {"type": "string", "enum": list(STATUSES)},
"note": {
"type": "string",
"description": "How your advice addresses it, or "
"why it does not apply to this workload.",
"description": "How your advice addresses it, why "
"it does not apply to this workload, or what the "
"request leaves unsaid.",
},
"question": {
"type": "string",
"description": "For needs_customer_input only: "
"the question to put to the customer, answerable "
"by someone who knows their organisation.",
},
},
"required": ["item", "status", "note"],
@@ -274,7 +299,8 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
"",
(
"Each pillar advisor's review of its pillar's checklist for this "
"workload: what its advice addresses, and what does not apply and why."
"workload: what its advice addresses, what does not apply and why, and "
"what the request does not say enough to settle."
),
]
for pillar, entries in reviews.items():
@@ -301,4 +327,48 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
for entry in entries
if isinstance(entry, Mapping)
]
questions = customer_questions(working)
if questions:
parts += [
"",
f"### {QUESTIONS_HEADING}",
"",
(
"The request does not say enough to settle these. They are not "
"faults in the design: each is something to ask the customer, and "
"the answer may change a recommendation."
),
"",
"| Pillar | About | Question |",
"| --- | --- | --- |",
]
parts += [
"| "
+ " | ".join(
value.replace("|", "\\|")
for value in (item["pillar"], item["item"], item["question"])
)
+ " |"
for item in questions
]
return "\n".join(parts) + "\n"
def customer_questions(working: Mapping[str, Any]) -> list[dict[str, str]]:
"""What the advisors found the request does not say: each checklist item that
needs the customer's answer, with the question to put to them."""
reviews = working.get(CHECKLIST_KEY)
if not isinstance(reviews, Mapping):
return []
return [
{
"pillar": str(pillar),
"item": str(entry.get("item", "")),
"question": str(entry.get("question", "")),
}
for pillar, entries in reviews.items()
for entry in entries or ()
if isinstance(entry, Mapping)
and entry.get("status") == NEEDS_CUSTOMER
and entry.get("question")
]