640 lines
23 KiB
Python
640 lines
23 KiB
Python
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
|
from execution_engine.tools.toolkit import make_tool_factory
|
|
from tests.reference_workflow.hooks import PILLAR_ADVISOR
|
|
from tests.reference_workflow.pillars import CONSULT_TOOL, pillar_consultation_tool
|
|
from tests.reference_workflow.workflow import WORKFLOW, with_pillars
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
|
|
|
PILLARS = {
|
|
"security": "waf-security",
|
|
"reliability": "waf-reliability",
|
|
"sustainability": "waf-sustainability",
|
|
}
|
|
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
|
|
|
|
|
|
def advice(pillar: str) -> list[Call]:
|
|
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
|
|
|
|
|
|
def turn_for_pillar(task):
|
|
return advice(task.context["pillar"])
|
|
|
|
|
|
async def consult(tool: ToolSpec, **args):
|
|
return await tool.handler(**args)
|
|
|
|
|
|
async def test_consults_every_pillar_with_only_its_skill():
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="Event ingestion on serverless compute.")
|
|
|
|
assert result.outcome is None
|
|
for pillar in PILLARS:
|
|
assert f"## {pillar}" in result.message
|
|
assert f"Use strong defaults for {pillar}." in result.message
|
|
runs = {run.task.context["pillar"]: run for run in runtime.runs}
|
|
assert set(runs) == set(PILLARS)
|
|
assert (
|
|
runs["security"].task.context["brief"]
|
|
== "Event ingestion on serverless compute."
|
|
)
|
|
assert runtime.runs[0].tools == ("submit_advice",)
|
|
|
|
|
|
async def test_advisors_are_scoped_to_their_pillar_skill():
|
|
seen: dict[str, tuple[str, ...]] = {}
|
|
|
|
class RecordingRuntime(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
seen[task.context["pillar"]] = role.skills
|
|
return await super().run(role, task, tools)
|
|
|
|
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
|
|
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
|
|
|
|
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
|
|
|
|
|
|
async def test_a_subset_of_pillars_can_be_consulted():
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="b", pillars=["reliability"])
|
|
|
|
assert "## reliability" in result.message
|
|
assert "## security" not in result.message
|
|
|
|
|
|
async def test_unknown_pillars_are_refused():
|
|
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
|
|
|
|
result = await consult(tool, brief="b", pillars=["speed"])
|
|
|
|
assert result.message.startswith("Refused")
|
|
assert "security" in result.message
|
|
|
|
|
|
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
|
|
def maybe_fail(task):
|
|
if task.context["pillar"] == "reliability":
|
|
return [Call("submit_advice", {"content": ""})]
|
|
return advice(task.context["pillar"])
|
|
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
|
|
|
|
result = await consult(
|
|
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
|
|
)
|
|
|
|
assert "Use strong defaults for security." in result.message
|
|
assert "## reliability\n\nNo guidance:" in result.message
|
|
|
|
|
|
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
|
|
async def noop(**_):
|
|
return ToolResult("ok")
|
|
|
|
def tool(name, tier="general"):
|
|
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
|
|
|
|
class Catalog:
|
|
def tools_for(self, server_ids):
|
|
available = {
|
|
"docs": [tool("docs__search")],
|
|
"compute": [tool("compute__list_instances", "cloud_read")],
|
|
}
|
|
return [t for sid in server_ids for t in available.get(sid, [])]
|
|
|
|
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
|
factory = make_tool_factory(
|
|
with_pillars({"security": "waf-security"}),
|
|
runtime,
|
|
{PILLAR_ADVISOR: advisor},
|
|
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
|
|
catalog=Catalog(),
|
|
)
|
|
|
|
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
|
|
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
|
|
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
|
|
"mermaid_check"
|
|
]
|
|
assert [t.name for t in factory(analyst, "requirements", {})] == [
|
|
"compute__list_instances"
|
|
]
|
|
|
|
[consult_tool] = factory(
|
|
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
|
)
|
|
await consult_tool.handler(brief="b")
|
|
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
|
|
|
|
|
|
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
|
roles = {PILLAR_ADVISOR: ADVISOR}
|
|
factory = make_tool_factory(with_pillars(PILLARS), FakeAgentRuntime({}), roles)
|
|
|
|
design = factory(
|
|
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
|
)
|
|
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
|
|
|
|
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
|
assert list(other) == []
|
|
|
|
|
|
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
|
|
|
|
|
|
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
|
|
memory: dict = {}
|
|
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
await consult(
|
|
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
|
|
)
|
|
|
|
def revised(task):
|
|
pillar = task.context["pillar"]
|
|
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
|
|
|
|
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
|
|
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
|
|
|
|
result = await consult(tool, brief="v2", pillars=["reliability"])
|
|
|
|
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
|
|
assert "## reliability\n\nRevised advice for reliability." in result.message
|
|
assert (
|
|
"## security (from your earlier consultation)\n\n"
|
|
"Use strong defaults for security." in result.message
|
|
)
|
|
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
|
|
assert set(memory["pillar_advice"]) == set(PILLARS)
|
|
|
|
|
|
async def test_omitting_pillars_consults_every_pillar_afresh():
|
|
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
|
|
|
result = await consult(tool, brief="v2")
|
|
|
|
assert len(runtime.runs) == 3
|
|
assert "earlier consultation" not in result.message
|
|
assert "old" not in memory["pillar_advice"].values()
|
|
|
|
|
|
async def test_failed_advice_is_not_remembered():
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
|
|
tool = pillar_consultation_tool(
|
|
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
|
|
)
|
|
|
|
result = await consult(tool, brief="b")
|
|
|
|
assert "## reliability\n\nNo guidance" in result.message
|
|
assert set(memory["pillar_advice"]) == {"security"}
|
|
|
|
|
|
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
|
|
import asyncio
|
|
|
|
from execution_engine.adk.runtime import RunLimiter
|
|
|
|
limiter = RunLimiter(1)
|
|
in_flight = 0
|
|
peak = 0
|
|
|
|
class SlowRuntime(FakeAgentRuntime):
|
|
async def run(self, role, task, tools):
|
|
nonlocal in_flight, peak
|
|
in_flight += 1
|
|
peak = max(peak, in_flight)
|
|
await asyncio.sleep(0.01)
|
|
try:
|
|
return await super().run(role, task, tools)
|
|
finally:
|
|
in_flight -= 1
|
|
|
|
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
|
|
|
|
async with limiter.slot(): # the design advisor's own run holds the only slot
|
|
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
|
|
|
|
assert peak == 1
|
|
assert result.message.count("Use strong defaults") == 3
|
|
|
|
|
|
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
sample_submission,
|
|
)
|
|
|
|
K, P = DeliverableKind, PhaseId
|
|
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
|
|
roles = {
|
|
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
|
PILLAR_ADVISOR: ADVISOR,
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
def delegate(*kinds):
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
def approve(phase):
|
|
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
|
|
|
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[
|
|
Call(
|
|
"reopen",
|
|
{"kind": "design_recommendations", "reason": "Tighten IAM."},
|
|
),
|
|
delegate(K.DESIGN_RECOMMENDATIONS),
|
|
],
|
|
[], # stops here: a pre-approved Execution ends with warnings
|
|
],
|
|
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
|
|
[sample_submission(K.REQUIREMENTS)]
|
|
],
|
|
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
|
|
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
|
|
],
|
|
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
|
|
[sample_submission(K.PRODUCT_SELECTION)]
|
|
],
|
|
design: [
|
|
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
|
|
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
|
|
],
|
|
PILLAR_ADVISOR: [turn_for_pillar] * 4,
|
|
}
|
|
)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
|
|
)
|
|
|
|
result = await engine.start("Ingestion", pre_approved=True)
|
|
|
|
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
|
|
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
|
|
second = [r for r in runtime.runs if r.role == design][1]
|
|
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
|
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
|
assert set(stored) == set(PILLARS)
|
|
|
|
|
|
# --- each advisor reviews its pillar's checklist ----------------------------------------
|
|
|
|
|
|
def reviewed(task):
|
|
pillar = task.context["pillar"]
|
|
return [
|
|
Call(
|
|
"submit_advice",
|
|
{
|
|
"content": f"Use strong defaults for {pillar}.",
|
|
"checklist": [
|
|
{
|
|
"item": f"{pillar} item one | with a bar",
|
|
"status": "addressed",
|
|
"note": "Covered by the first recommendation.",
|
|
},
|
|
{
|
|
"item": f"{pillar} item two",
|
|
"status": "not_applicable",
|
|
"note": "An organisation's process, not a design decision.",
|
|
},
|
|
{"item": "malformed", "status": "maybe", "note": "dropped"},
|
|
],
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
|
|
from tests.reference_workflow.pillars import CHECKLIST_KEY
|
|
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
|
|
|
result = await consult(tool, brief="b")
|
|
|
|
# The design advisor is given the advice, not the review: it does not copy it.
|
|
assert "item one" not in result.message
|
|
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
|
|
review = memory[CHECKLIST_KEY]["security"]
|
|
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
|
|
|
|
|
|
async def test_the_review_is_written_into_the_deliverable_by_code():
|
|
from execution_engine.adk.runtime import Submitted
|
|
from execution_engine.tools.execution import specialist_tools
|
|
from tests.reference_workflow.pillars import REVIEW_HEADING
|
|
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
|
|
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
|
await consult(
|
|
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
|
)
|
|
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
|
(submit,) = (
|
|
tool
|
|
for tool in specialist_tools(
|
|
WORKFLOW, kind, (), allow_cloud_read=False, working=memory
|
|
)
|
|
if tool.name == "submit_deliverable"
|
|
)
|
|
call = sample_submission(kind)
|
|
|
|
first = await submit.handler(**call.args)
|
|
# A Revision submits content that already carries the section.
|
|
again = await submit.handler(
|
|
content=first.outcome.content, attachments=call.args["attachments"]
|
|
)
|
|
|
|
assert isinstance(first.outcome, Submitted)
|
|
content = first.outcome.content
|
|
assert f"## {REVIEW_HEADING}" in content and "### security" in content
|
|
assert (
|
|
"| security item one \\| with a bar | Addressed | Covered by the first "
|
|
"recommendation. |"
|
|
) in content
|
|
assert "| security item two | Does not apply |" in content
|
|
assert "malformed" not in content
|
|
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
|
|
|
|
|
|
async def test_with_no_review_nothing_is_added():
|
|
from execution_engine.tools.execution import specialist_tools
|
|
from tests.reference_workflow.pillars import REVIEW_HEADING
|
|
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
|
|
|
|
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
|
(submit,) = (
|
|
tool
|
|
for tool in specialist_tools(
|
|
WORKFLOW, kind, (), allow_cloud_read=False, working={}
|
|
)
|
|
if tool.name == "submit_deliverable"
|
|
)
|
|
|
|
result = await submit.handler(**sample_submission(kind).args)
|
|
|
|
assert REVIEW_HEADING not in result.outcome.content
|
|
|
|
|
|
# --- only the pillars a change touches are consulted again -------------------------------
|
|
|
|
|
|
async def test_a_change_names_the_pillars_it_affects_directly():
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer
|
|
from tests.reference_workflow.pillars import pillars_touched
|
|
|
|
decider = FakeDecider(
|
|
{
|
|
"security": Answer(1.9, 0.9),
|
|
"reliability": Answer(1.0, 0.6),
|
|
"sustainability": Answer(0.2, 0.8),
|
|
}
|
|
)
|
|
|
|
chosen = await pillars_touched(
|
|
decider, list(PILLARS), "Use customer-managed keys.", scope="e1"
|
|
)
|
|
|
|
state, questions, scope = decider.asked[0]
|
|
assert chosen == ["security"]
|
|
assert scope == "e1" and state.endswith("Use customer-managed keys.")
|
|
assert questions["security"].kind == "score"
|
|
assert questions["security"].options == (
|
|
"not at all",
|
|
"only indirectly",
|
|
"directly",
|
|
)
|
|
|
|
|
|
async def test_a_change_that_touches_no_pillar_is_an_answer_and_no_decision_is_not():
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer, NoDecider
|
|
from tests.reference_workflow.pillars import pillars_touched
|
|
|
|
nothing = FakeDecider(dict.fromkeys(PILLARS, Answer(0.0, 0.99)))
|
|
|
|
assert await pillars_touched(nothing, list(PILLARS), "Rename a page.") == []
|
|
assert await pillars_touched(NoDecider(), list(PILLARS), "Rename a page.") is None
|
|
assert await pillars_touched(nothing, list(PILLARS), " ") is None
|
|
|
|
|
|
async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said():
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
|
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
|
await consult(tool, brief="b")
|
|
|
|
again = await consult(tool, brief="b", pillars=[])
|
|
|
|
assert len(runtime.runs) == 3 # nobody was consulted a second time
|
|
for pillar in PILLARS:
|
|
assert f"## {pillar} (from your earlier consultation)" in again.message
|
|
|
|
|
|
# --- what the request does not say is a question for the customer ------------------------
|
|
|
|
|
|
def unsure(question: str | None = "Do you have a central identity provider?"):
|
|
def turn(task):
|
|
entry = {
|
|
"item": "Identity is managed centrally",
|
|
"status": "needs_customer_input",
|
|
"note": "The request does not say how staff identities are managed.",
|
|
}
|
|
if question is not None:
|
|
entry["question"] = question
|
|
first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]})
|
|
# A second attempt, used only when the first is refused.
|
|
fixed = Call(
|
|
"submit_advice",
|
|
{
|
|
"content": "Advice.",
|
|
"checklist": [{**entry, "question": "Who manages staff identities?"}],
|
|
},
|
|
)
|
|
return [first, fixed]
|
|
|
|
return turn
|
|
|
|
|
|
async def test_an_item_the_request_cannot_settle_is_kept_with_its_question():
|
|
from tests.reference_workflow.pillars import CHECKLIST_KEY, customer_questions
|
|
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3})
|
|
|
|
await consult(
|
|
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
|
)
|
|
|
|
(entry,) = memory[CHECKLIST_KEY]["security"]
|
|
assert entry["status"] == "needs_customer_input"
|
|
assert customer_questions(memory)[0] == {
|
|
"pillar": "security",
|
|
"item": "Identity is managed centrally",
|
|
"question": "Do you have a central identity provider?",
|
|
}
|
|
assert len(customer_questions(memory)) == len(PILLARS)
|
|
|
|
|
|
async def test_an_item_for_the_customer_must_carry_the_question_to_ask():
|
|
from tests.reference_workflow.pillars import CHECKLIST_KEY
|
|
|
|
memory: dict = {}
|
|
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3})
|
|
|
|
await consult(
|
|
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
|
)
|
|
|
|
refused, accepted = (reply for _, reply in runtime.runs[0].results)
|
|
assert refused.startswith("Not submitted: an item to confirm with the customer")
|
|
assert "Identity is managed centrally" in refused
|
|
(entry,) = memory[CHECKLIST_KEY]["security"]
|
|
assert entry["question"] == "Who manages staff identities?"
|
|
assert accepted == "Advice submitted."
|
|
|
|
|
|
def test_the_questions_are_reported_as_missing_information_not_as_faults():
|
|
from tests.reference_workflow.pillars import (
|
|
CHECKLIST_KEY,
|
|
QUESTIONS_HEADING,
|
|
render_checklist_review,
|
|
)
|
|
|
|
working = {
|
|
CHECKLIST_KEY: {
|
|
"security": [
|
|
{
|
|
"item": "Scans run in the pipeline",
|
|
"status": "addressed",
|
|
"note": "DR-3",
|
|
},
|
|
{
|
|
"item": "Identity is managed centrally",
|
|
"status": "needs_customer_input",
|
|
"note": "Not said.",
|
|
"question": "Do you have a central identity provider?",
|
|
},
|
|
]
|
|
}
|
|
}
|
|
|
|
text = render_checklist_review(working)
|
|
|
|
assert "| Identity is managed centrally | To confirm with the customer |" in text
|
|
assert f"### {QUESTIONS_HEADING}" in text
|
|
assert "They are not faults in the design" in text
|
|
assert (
|
|
"| security | Identity is managed centrally | Do you have a central "
|
|
"identity provider? |"
|
|
) in text
|
|
# With nothing to ask there is no questions section.
|
|
only_addressed = {
|
|
CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]}
|
|
}
|
|
assert QUESTIONS_HEADING not in render_checklist_review(only_addressed)
|
|
|
|
|
|
async def test_an_execution_writes_its_questions_for_the_customer(tmp_path):
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.runtime import NoDecider
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.nodes.router import Router, Thresholds
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from tests.reference_workflow.pillars import CONSULT_TOOL
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
sample_submission,
|
|
)
|
|
|
|
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
|
advisor = DEPENDENCY_GRAPH[kind].specialist
|
|
roles = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
PILLAR_ADVISOR: ADVISOR,
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
turns = {
|
|
spec.specialist: [[sample_submission(k)]]
|
|
for k, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]]
|
|
turns[PILLAR_ADVISOR] = [unsure()] * 3
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
|
|
)
|
|
|
|
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
out = next(tmp_path.rglob("execution.json")).parent
|
|
manifest = json.loads((out / "execution.json").read_text())
|
|
assert manifest["customer_questions"] == "customer-questions.md"
|
|
questions = (out / "customer-questions.md").read_text()
|
|
assert questions.startswith("# Questions for the customer")
|
|
assert "Do you have a central identity provider?" in questions
|
|
assert (
|
|
"### Questions for the customer"
|
|
in (out / "deliverables" / "design_recommendations.md").read_text()
|
|
)
|