feat: what a request does not say is a question for the customer
Some checks failed
validation / verify (push) Failing after 11s
Some checks failed
validation / verify (push) Failing after 11s
A pillar's checklist item the request gives no information about was marked "not addressed", which reads as a fault in the design. It is missing information. The checklist review gains a fourth outcome, to confirm with the customer, which must carry the question to ask. The questions are written into the design recommendations and the guide, and to customer-questions.md; the evaluation's judged check counts such an item as accounted for. Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's retention, training, context and rate limits (iteration 5 step 1), and the owner's labels, which showed the judged question was wrong, not the judges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -39,12 +39,18 @@ REVIEW_HEADING = "Architecture framework checklist review"
|
||||
# What a Change request carries over from its Baseline's working memory.
|
||||
KEPT_FOR_CHANGES = (ADVICE_KEY, CHECKLIST_KEY)
|
||||
_CHANGE_LIMIT = 6_000
|
||||
STATUSES = ("addressed", "not_applicable", "not_addressed")
|
||||
NEEDS_CUSTOMER = "needs_customer_input"
|
||||
STATUSES = ("addressed", "not_applicable", NEEDS_CUSTOMER, "not_addressed")
|
||||
_STATUS_WORDS = {
|
||||
"addressed": "Addressed",
|
||||
"not_applicable": "Does not apply",
|
||||
# The item applies, and the customer's request does not say what would settle
|
||||
# it. That is missing information, not a fault in the design: it is reported
|
||||
# with the question to put to the customer.
|
||||
NEEDS_CUSTOMER: "To confirm with the customer",
|
||||
"not_addressed": "Not addressed",
|
||||
}
|
||||
QUESTIONS_HEADING = "Questions for the customer"
|
||||
|
||||
|
||||
AFFECTED = ("not at all", "only indirectly", "directly")
|
||||
@@ -215,17 +221,29 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
|
||||
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
|
||||
if not content.strip():
|
||||
return ToolResult("Not submitted: the advice is empty.")
|
||||
review[:] = [
|
||||
entries = [
|
||||
{
|
||||
"item": " ".join(str(entry.get("item", "")).split()),
|
||||
"status": str(entry.get("status", "")),
|
||||
"note": " ".join(str(entry.get("note", "")).split()),
|
||||
"question": " ".join(str(entry.get("question") or "").split()),
|
||||
}
|
||||
for entry in checklist or ()
|
||||
if isinstance(entry, Mapping)
|
||||
and str(entry.get("item", "")).strip()
|
||||
and entry.get("status") in STATUSES
|
||||
]
|
||||
unasked = [
|
||||
entry["item"]
|
||||
for entry in entries
|
||||
if entry["status"] == NEEDS_CUSTOMER and not entry["question"]
|
||||
]
|
||||
if unasked:
|
||||
return ToolResult(
|
||||
"Not submitted: an item to confirm with the customer needs the "
|
||||
"`question` to put to them. Missing for: " + "; ".join(unasked)
|
||||
)
|
||||
review[:] = entries
|
||||
return ToolResult("Advice submitted.", Submitted(content))
|
||||
|
||||
text = {"type": "string"}
|
||||
@@ -248,8 +266,15 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
|
||||
"status": {"type": "string", "enum": list(STATUSES)},
|
||||
"note": {
|
||||
"type": "string",
|
||||
"description": "How your advice addresses it, or "
|
||||
"why it does not apply to this workload.",
|
||||
"description": "How your advice addresses it, why "
|
||||
"it does not apply to this workload, or what the "
|
||||
"request leaves unsaid.",
|
||||
},
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "For needs_customer_input only: "
|
||||
"the question to put to the customer, answerable "
|
||||
"by someone who knows their organisation.",
|
||||
},
|
||||
},
|
||||
"required": ["item", "status", "note"],
|
||||
@@ -274,7 +299,8 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
|
||||
"",
|
||||
(
|
||||
"Each pillar advisor's review of its pillar's checklist for this "
|
||||
"workload: what its advice addresses, and what does not apply and why."
|
||||
"workload: what its advice addresses, what does not apply and why, and "
|
||||
"what the request does not say enough to settle."
|
||||
),
|
||||
]
|
||||
for pillar, entries in reviews.items():
|
||||
@@ -301,4 +327,48 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
|
||||
for entry in entries
|
||||
if isinstance(entry, Mapping)
|
||||
]
|
||||
questions = customer_questions(working)
|
||||
if questions:
|
||||
parts += [
|
||||
"",
|
||||
f"### {QUESTIONS_HEADING}",
|
||||
"",
|
||||
(
|
||||
"The request does not say enough to settle these. They are not "
|
||||
"faults in the design: each is something to ask the customer, and "
|
||||
"the answer may change a recommendation."
|
||||
),
|
||||
"",
|
||||
"| Pillar | About | Question |",
|
||||
"| --- | --- | --- |",
|
||||
]
|
||||
parts += [
|
||||
"| "
|
||||
+ " | ".join(
|
||||
value.replace("|", "\\|")
|
||||
for value in (item["pillar"], item["item"], item["question"])
|
||||
)
|
||||
+ " |"
|
||||
for item in questions
|
||||
]
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def customer_questions(working: Mapping[str, Any]) -> list[dict[str, str]]:
|
||||
"""What the advisors found the request does not say: each checklist item that
|
||||
needs the customer's answer, with the question to put to them."""
|
||||
reviews = working.get(CHECKLIST_KEY)
|
||||
if not isinstance(reviews, Mapping):
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"pillar": str(pillar),
|
||||
"item": str(entry.get("item", "")),
|
||||
"question": str(entry.get("question", "")),
|
||||
}
|
||||
for pillar, entries in reviews.items()
|
||||
for entry in entries or ()
|
||||
if isinstance(entry, Mapping)
|
||||
and entry.get("status") == NEEDS_CUSTOMER
|
||||
and entry.get("question")
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user