feat: what a request does not say is a question for the customer
Some checks failed
validation / verify (push) Failing after 11s

A pillar's checklist item the request gives no information about was marked
"not addressed", which reads as a fault in the design. It is missing
information. The checklist review gains a fourth outcome, to confirm with the
customer, which must carry the question to ask. The questions are written
into the design recommendations and the guide, and to customer-questions.md;
the evaluation's judged check counts such an item as accounted for.

Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's
retention, training, context and rate limits (iteration 5 step 1), and the
owner's labels, which showed the judged question was wrong, not the judges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 19:30:12 +01:00
parent f9cc34817b
commit a48468d477
15 changed files with 375 additions and 17 deletions

View File

@@ -54,7 +54,13 @@ from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_to
from app.tools.guardrail import EXECUTION_SCOPE, instructs_agent
from app.tools.meaning import settle_meaning
from app.tools.modules import MODULE_TOOL, MODULES_OPTED_IN
from app.tools.pillars import ADVICE_KEY, KEPT_FOR_CHANGES, pillars_touched
from app.tools.pillars import (
ADVICE_KEY,
KEPT_FOR_CHANGES,
QUESTIONS_HEADING,
customer_questions,
pillars_touched,
)
from app.tools.skill_routing import read_first
from app.workflows.chain import (
IMPLEMENTATION_FILE,
@@ -83,6 +89,7 @@ NO_USER_REPLY = (
"'Assumptions', and proceed."
)
GUIDE_FILENAME = "solution-architecture-guide.md"
QUESTIONS_FILENAME = "customer-questions.md" # what to ask the customer
MANIFEST_FILENAME = "execution.json" # request, status and files of one Execution
@@ -1237,7 +1244,11 @@ def _written_before(out_dir: Path) -> set[str]:
return set()
if not isinstance(manifest, dict):
return set()
names = {manifest.get("solution_guide"), manifest.get("changes")}
names = {
manifest.get("solution_guide"),
manifest.get("changes"),
manifest.get("customer_questions"),
}
for entry in (manifest.get("deliverables") or {}).values():
if isinstance(entry, dict):
names.add(entry.get("document"))
@@ -1293,6 +1304,28 @@ def _write_outputs(
if kept:
# Working material a Change request starts from, such as each pillar's advice.
files[WORKING_FILENAME] = json.dumps(kept, indent=2, ensure_ascii=False)
questions = customer_questions(
state.get("working", {}).get(DeliverableKind.DESIGN_RECOMMENDATIONS.value, {})
)
if questions:
# What the request did not say enough to settle, as questions to put to
# the customer. Missing information, not a fault in the design.
files[QUESTIONS_FILENAME] = "\n".join(
[
f"# {QUESTIONS_HEADING}",
"",
(
"The request did not say enough to settle these. Each answer "
"may change a recommendation."
),
"",
*(
f"- **{item['pillar']}**, {item['item']} {item['question']}"
for item in questions
),
"",
]
)
baseline = state.get("baseline") or None
if baseline:
# What this Change request changed against its Baseline, worked out from
@@ -1334,6 +1367,8 @@ def _write_outputs(
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
},
"solution_guide": GUIDE_FILENAME if guide else None,
# Questions the request left open, for the customer; None when there are none.
"customer_questions": QUESTIONS_FILENAME if questions else None,
"changes": CHANGES_FILE if baseline else None,
"deliverables": by_kind,
}

View File

@@ -32,6 +32,21 @@ QUALITY_RUBRIC: Mapping[str, str] = {
_JUDGED = "design_recommendations"
def checklist_question(pillar: str, item: str) -> str:
"""Whether the recommendations account for one item of a pillar's checklist.
An item is accounted for in three ways. The third matters: a customer's request
seldom says enough to settle every item, and what it leaves unsaid is a
question for them, not a fault in the design (the owner, 2026-10-10).
"""
return (
f"Do the recommendations account for this {pillar} checklist item in any "
"of these ways: by recommending it, by explaining why it does not apply, "
"or by recording it as something to confirm with the customer? "
f"{item}"
)
async def assess(
decider: Decider,
deliverables: Mapping[str, Mapping[str, Any]],

View File

@@ -39,12 +39,18 @@ REVIEW_HEADING = "Architecture framework checklist review"
# What a Change request carries over from its Baseline's working memory.
KEPT_FOR_CHANGES = (ADVICE_KEY, CHECKLIST_KEY)
_CHANGE_LIMIT = 6_000
STATUSES = ("addressed", "not_applicable", "not_addressed")
NEEDS_CUSTOMER = "needs_customer_input"
STATUSES = ("addressed", "not_applicable", NEEDS_CUSTOMER, "not_addressed")
_STATUS_WORDS = {
"addressed": "Addressed",
"not_applicable": "Does not apply",
# The item applies, and the customer's request does not say what would settle
# it. That is missing information, not a fault in the design: it is reported
# with the question to put to the customer.
NEEDS_CUSTOMER: "To confirm with the customer",
"not_addressed": "Not addressed",
}
QUESTIONS_HEADING = "Questions for the customer"
AFFECTED = ("not at all", "only indirectly", "directly")
@@ -215,17 +221,29 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
if not content.strip():
return ToolResult("Not submitted: the advice is empty.")
review[:] = [
entries = [
{
"item": " ".join(str(entry.get("item", "")).split()),
"status": str(entry.get("status", "")),
"note": " ".join(str(entry.get("note", "")).split()),
"question": " ".join(str(entry.get("question") or "").split()),
}
for entry in checklist or ()
if isinstance(entry, Mapping)
and str(entry.get("item", "")).strip()
and entry.get("status") in STATUSES
]
unasked = [
entry["item"]
for entry in entries
if entry["status"] == NEEDS_CUSTOMER and not entry["question"]
]
if unasked:
return ToolResult(
"Not submitted: an item to confirm with the customer needs the "
"`question` to put to them. Missing for: " + "; ".join(unasked)
)
review[:] = entries
return ToolResult("Advice submitted.", Submitted(content))
text = {"type": "string"}
@@ -248,8 +266,15 @@ def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
"status": {"type": "string", "enum": list(STATUSES)},
"note": {
"type": "string",
"description": "How your advice addresses it, or "
"why it does not apply to this workload.",
"description": "How your advice addresses it, why "
"it does not apply to this workload, or what the "
"request leaves unsaid.",
},
"question": {
"type": "string",
"description": "For needs_customer_input only: "
"the question to put to the customer, answerable "
"by someone who knows their organisation.",
},
},
"required": ["item", "status", "note"],
@@ -274,7 +299,8 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
"",
(
"Each pillar advisor's review of its pillar's checklist for this "
"workload: what its advice addresses, and what does not apply and why."
"workload: what its advice addresses, what does not apply and why, and "
"what the request does not say enough to settle."
),
]
for pillar, entries in reviews.items():
@@ -301,4 +327,48 @@ def render_checklist_review(working: Mapping[str, Any]) -> str:
for entry in entries
if isinstance(entry, Mapping)
]
questions = customer_questions(working)
if questions:
parts += [
"",
f"### {QUESTIONS_HEADING}",
"",
(
"The request does not say enough to settle these. They are not "
"faults in the design: each is something to ask the customer, and "
"the answer may change a recommendation."
),
"",
"| Pillar | About | Question |",
"| --- | --- | --- |",
]
parts += [
"| "
+ " | ".join(
value.replace("|", "\\|")
for value in (item["pillar"], item["item"], item["question"])
)
+ " |"
for item in questions
]
return "\n".join(parts) + "\n"
def customer_questions(working: Mapping[str, Any]) -> list[dict[str, str]]:
"""What the advisors found the request does not say: each checklist item that
needs the customer's answer, with the question to put to them."""
reviews = working.get(CHECKLIST_KEY)
if not isinstance(reviews, Mapping):
return []
return [
{
"pillar": str(pillar),
"item": str(entry.get("item", "")),
"question": str(entry.get("question", "")),
}
for pillar, entries in reviews.items()
for entry in entries or ()
if isinstance(entry, Mapping)
and entry.get("status") == NEEDS_CUSTOMER
and entry.get("question")
]