feat: what a request does not say is a question for the customer
Some checks failed
validation / verify (push) Failing after 11s
Some checks failed
validation / verify (push) Failing after 11s
A pillar's checklist item the request gives no information about was marked "not addressed", which reads as a fault in the design. It is missing information. The checklist review gains a fourth outcome, to confirm with the customer, which must carry the question to ask. The questions are written into the design recommendations and the guide, and to customer-questions.md; the evaluation's judged check counts such an item as accounted for. Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's retention, training, context and rate limits (iteration 5 step 1), and the owner's labels, which showed the judged question was wrong, not the judges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -464,3 +464,163 @@ async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said(
|
||||
assert len(runtime.runs) == 3 # nobody was consulted a second time
|
||||
for pillar in PILLARS:
|
||||
assert f"## {pillar} (from your earlier consultation)" in again.message
|
||||
|
||||
|
||||
# --- what the request does not say is a question for the customer ------------------------
|
||||
|
||||
|
||||
def unsure(question: str | None = "Do you have a central identity provider?"):
|
||||
def turn(task):
|
||||
entry = {
|
||||
"item": "Identity is managed centrally",
|
||||
"status": "needs_customer_input",
|
||||
"note": "The request does not say how staff identities are managed.",
|
||||
}
|
||||
if question is not None:
|
||||
entry["question"] = question
|
||||
first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]})
|
||||
# A second attempt, used only when the first is refused.
|
||||
fixed = Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": "Advice.",
|
||||
"checklist": [{**entry, "question": "Who manages staff identities?"}],
|
||||
},
|
||||
)
|
||||
return [first, fixed]
|
||||
|
||||
return turn
|
||||
|
||||
|
||||
async def test_an_item_the_request_cannot_settle_is_kept_with_its_question():
|
||||
from app.tools.pillars import CHECKLIST_KEY, customer_questions
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3})
|
||||
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
|
||||
(entry,) = memory[CHECKLIST_KEY]["security"]
|
||||
assert entry["status"] == "needs_customer_input"
|
||||
assert customer_questions(memory)[0] == {
|
||||
"pillar": "security",
|
||||
"item": "Identity is managed centrally",
|
||||
"question": "Do you have a central identity provider?",
|
||||
}
|
||||
assert len(customer_questions(memory)) == len(PILLARS)
|
||||
|
||||
|
||||
async def test_an_item_for_the_customer_must_carry_the_question_to_ask():
|
||||
from app.tools.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3})
|
||||
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
|
||||
refused, accepted = (reply for _, reply in runtime.runs[0].results)
|
||||
assert refused.startswith("Not submitted: an item to confirm with the customer")
|
||||
assert "Identity is managed centrally" in refused
|
||||
(entry,) = memory[CHECKLIST_KEY]["security"]
|
||||
assert entry["question"] == "Who manages staff identities?"
|
||||
assert accepted == "Advice submitted."
|
||||
|
||||
|
||||
def test_the_questions_are_reported_as_missing_information_not_as_faults():
|
||||
from app.tools.pillars import (
|
||||
CHECKLIST_KEY,
|
||||
QUESTIONS_HEADING,
|
||||
render_checklist_review,
|
||||
)
|
||||
|
||||
working = {
|
||||
CHECKLIST_KEY: {
|
||||
"security": [
|
||||
{
|
||||
"item": "Scans run in the pipeline",
|
||||
"status": "addressed",
|
||||
"note": "DR-3",
|
||||
},
|
||||
{
|
||||
"item": "Identity is managed centrally",
|
||||
"status": "needs_customer_input",
|
||||
"note": "Not said.",
|
||||
"question": "Do you have a central identity provider?",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
text = render_checklist_review(working)
|
||||
|
||||
assert "| Identity is managed centrally | To confirm with the customer |" in text
|
||||
assert f"### {QUESTIONS_HEADING}" in text
|
||||
assert "They are not faults in the design" in text
|
||||
assert (
|
||||
"| security | Identity is managed centrally | Do you have a central "
|
||||
"identity provider? |"
|
||||
) in text
|
||||
# With nothing to ask there is no questions section.
|
||||
only_addressed = {
|
||||
CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]}
|
||||
}
|
||||
assert QUESTIONS_HEADING not in render_checklist_review(only_addressed)
|
||||
|
||||
|
||||
async def test_an_execution_writes_its_questions_for_the_customer(tmp_path):
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.adk.runtime import NoDecider
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.pillars import CONSULT_TOOL
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
advisor = DEPENDENCY_GRAPH[kind].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
PILLAR_ADVISOR: ADVISOR,
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(k)]]
|
||||
for k, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]]
|
||||
turns[PILLAR_ADVISOR] = [unsure()] * 3
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
tool_factory=make_tool_factory(runtime, roles, pillar_skills=PILLARS),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
out = next(tmp_path.rglob("execution.json")).parent
|
||||
manifest = json.loads((out / "execution.json").read_text())
|
||||
assert manifest["customer_questions"] == "customer-questions.md"
|
||||
questions = (out / "customer-questions.md").read_text()
|
||||
assert questions.startswith("# Questions for the customer")
|
||||
assert "Do you have a central identity provider?" in questions
|
||||
assert (
|
||||
"### Questions for the customer"
|
||||
in (out / "deliverables" / "design_recommendations.md").read_text()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user