feat: what a request does not say is a question for the customer
Some checks failed
validation / verify (push) Failing after 11s

A pillar's checklist item the request gives no information about was marked
"not addressed", which reads as a fault in the design. It is missing
information. The checklist review gains a fourth outcome, to confirm with the
customer, which must carry the question to ask. The questions are written
into the design recommendations and the guide, and to customer-questions.md;
the evaluation's judged check counts such an item as accounted for.

Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's
retention, training, context and rate limits (iteration 5 step 1), and the
owner's labels, which showed the judged question was wrong, not the judges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 19:30:12 +01:00
parent f9cc34817b
commit a48468d477
15 changed files with 375 additions and 17 deletions

View File

@@ -464,3 +464,163 @@ async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said(
assert len(runtime.runs) == 3 # nobody was consulted a second time
for pillar in PILLARS:
assert f"## {pillar} (from your earlier consultation)" in again.message
# --- what the request does not say is a question for the customer ------------------------
def unsure(question: str | None = "Do you have a central identity provider?"):
def turn(task):
entry = {
"item": "Identity is managed centrally",
"status": "needs_customer_input",
"note": "The request does not say how staff identities are managed.",
}
if question is not None:
entry["question"] = question
first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]})
# A second attempt, used only when the first is refused.
fixed = Call(
"submit_advice",
{
"content": "Advice.",
"checklist": [{**entry, "question": "Who manages staff identities?"}],
},
)
return [first, fixed]
return turn
async def test_an_item_the_request_cannot_settle_is_kept_with_its_question():
from app.tools.pillars import CHECKLIST_KEY, customer_questions
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
(entry,) = memory[CHECKLIST_KEY]["security"]
assert entry["status"] == "needs_customer_input"
assert customer_questions(memory)[0] == {
"pillar": "security",
"item": "Identity is managed centrally",
"question": "Do you have a central identity provider?",
}
assert len(customer_questions(memory)) == len(PILLARS)
async def test_an_item_for_the_customer_must_carry_the_question_to_ask():
from app.tools.pillars import CHECKLIST_KEY
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
refused, accepted = (reply for _, reply in runtime.runs[0].results)
assert refused.startswith("Not submitted: an item to confirm with the customer")
assert "Identity is managed centrally" in refused
(entry,) = memory[CHECKLIST_KEY]["security"]
assert entry["question"] == "Who manages staff identities?"
assert accepted == "Advice submitted."
def test_the_questions_are_reported_as_missing_information_not_as_faults():
from app.tools.pillars import (
CHECKLIST_KEY,
QUESTIONS_HEADING,
render_checklist_review,
)
working = {
CHECKLIST_KEY: {
"security": [
{
"item": "Scans run in the pipeline",
"status": "addressed",
"note": "DR-3",
},
{
"item": "Identity is managed centrally",
"status": "needs_customer_input",
"note": "Not said.",
"question": "Do you have a central identity provider?",
},
]
}
}
text = render_checklist_review(working)
assert "| Identity is managed centrally | To confirm with the customer |" in text
assert f"### {QUESTIONS_HEADING}" in text
assert "They are not faults in the design" in text
assert (
"| security | Identity is managed centrally | Do you have a central "
"identity provider? |"
) in text
# With nothing to ask there is no questions section.
only_addressed = {
CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]}
}
assert QUESTIONS_HEADING not in render_checklist_review(only_addressed)
async def test_an_execution_writes_its_questions_for_the_customer(tmp_path):
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import sample_submission
from app.adk.runtime import NoDecider
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.tools.pillars import CONSULT_TOOL
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
advisor = DEPENDENCY_GRAPH[kind].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
PILLAR_ADVISOR: ADVISOR,
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
turns = {
spec.specialist: [[sample_submission(k)]]
for k, spec in DEPENDENCY_GRAPH.items()
}
turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]]
turns[PILLAR_ADVISOR] = [unsure()] * 3
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
tool_factory=make_tool_factory(runtime, roles, pillar_skills=PILLARS),
)
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
assert result.status == "completed"
out = next(tmp_path.rglob("execution.json")).parent
manifest = json.loads((out / "execution.json").read_text())
assert manifest["customer_questions"] == "customer-questions.md"
questions = (out / "customer-questions.md").read_text()
assert questions.startswith("# Questions for the customer")
assert "Do you have a central identity provider?" in questions
assert (
"### Questions for the customer"
in (out / "deliverables" / "design_recommendations.md").read_text()
)