"""The design advisor's fan-out to one advisor per architecture-framework pillar.""" from app.adk.fake_runtime import Call, FakeAgentRuntime from app.adk.roles import PILLAR_ADVISOR from app.adk.runtime import RoleSpec, ToolResult, ToolSpec from app.tools.pillars import CONSULT_TOOL, pillar_consultation_tool from app.tools.toolkit import make_tool_factory PILLARS = { "security": "waf-security", "reliability": "waf-reliability", "sustainability": "waf-sustainability", } ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5) def advice(pillar: str) -> list[Call]: return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})] def turn_for_pillar(task): return advice(task.context["pillar"]) async def consult(tool: ToolSpec, **args): return await tool.handler(**args) async def test_consults_every_pillar_with_only_its_skill(): runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS) result = await consult(tool, brief="Event ingestion on serverless compute.") assert result.outcome is None for pillar in PILLARS: assert f"## {pillar}" in result.message assert f"Use strong defaults for {pillar}." in result.message runs = {run.task.context["pillar"]: run for run in runtime.runs} assert set(runs) == set(PILLARS) assert ( runs["security"].task.context["brief"] == "Event ingestion on serverless compute." ) assert runtime.runs[0].tools == ("submit_advice",) async def test_advisors_are_scoped_to_their_pillar_skill(): seen: dict[str, tuple[str, ...]] = {} class RecordingRuntime(FakeAgentRuntime): async def run(self, role, task, tools): seen[task.context["pillar"]] = role.skills return await super().run(role, task, tools) runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b") assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()} async def test_a_subset_of_pillars_can_be_consulted(): runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS) result = await consult(tool, brief="b", pillars=["reliability"]) assert "## reliability" in result.message assert "## security" not in result.message async def test_unknown_pillars_are_refused(): tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS) result = await consult(tool, brief="b", pillars=["speed"]) assert result.message.startswith("Refused") assert "security" in result.message async def test_an_advisor_that_fails_is_reported_without_losing_the_others(): def maybe_fail(task): if task.context["pillar"] == "reliability": return [Call("submit_advice", {"content": ""})] return advice(task.context["pillar"]) runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3}) result = await consult( pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b" ) assert "Use strong defaults for security." in result.message assert "## reliability\n\nNo guidance:" in result.message async def test_tool_factory_assembles_local_mcp_and_consultation_tools(): async def noop(**_): return ToolResult("ok") def tool(name, tier="general"): return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier) class Catalog: def tools_for(self, server_ids): available = { "docs": [tool("docs__search")], "compute": [tool("compute__list_instances", "cloud_read")], } return [t for sid in server_ids for t in available.get(sid, [])] advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute")) runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]}) factory = make_tool_factory( runtime, {PILLAR_ADVISOR: advisor}, {"security": "waf-security"}, local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")], catalog=Catalog(), ) diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",)) analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",)) assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [ "mermaid_check" ] assert [t.name for t in factory(analyst, "requirements", {})] == [ "compute__list_instances" ] [consult_tool] = factory( RoleSpec("design-advisor", "", budget=5), "design_recommendations", {} ) await consult_tool.handler(brief="b") assert runtime.runs[0].tools == ("submit_advice", "docs__search") async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor(): roles = {PILLAR_ADVISOR: ADVISOR} factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS) design = factory( RoleSpec("design-advisor", "", budget=5), "design_recommendations", {} ) other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {}) assert [tool.name for tool in design] == [CONSULT_TOOL] assert list(other) == [] # --- advice remembered across Revisions (iteration 4, step 3) -------------------------- async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory(): memory: dict = {} first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) await consult( pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1" ) def revised(task): pillar = task.context["pillar"] return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})] second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]}) tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory) result = await consult(tool, brief="v2", pillars=["reliability"]) assert [run.task.context["pillar"] for run in second.runs] == ["reliability"] assert "## reliability\n\nRevised advice for reliability." in result.message assert ( "## security (from your earlier consultation)\n\n" "Use strong defaults for security." in result.message ) assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability." assert set(memory["pillar_advice"]) == set(PILLARS) async def test_omitting_pillars_consults_every_pillar_afresh(): memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory) result = await consult(tool, brief="v2") assert len(runtime.runs) == 3 assert "earlier consultation" not in result.message assert "old" not in memory["pillar_advice"].values() async def test_failed_advice_is_not_remembered(): memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]}) tool = pillar_consultation_tool( runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory ) result = await consult(tool, brief="b") assert "## reliability\n\nNo guidance" in result.message assert set(memory["pillar_advice"]) == {"security"} async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock(): import asyncio from app.adk.runtime import RunLimiter limiter = RunLimiter(1) in_flight = 0 peak = 0 class SlowRuntime(FakeAgentRuntime): async def run(self, role, task, tools): nonlocal in_flight, peak in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0.01) try: return await super().run(role, task, tools) finally: in_flight -= 1 runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter) async with limiter.slot(): # the design advisor's own run holds the only slot result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5) assert peak == 1 assert result.message.count("Use strong defaults") == 3 async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path): from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import sample_submission from app.agent import ExecutionEngine from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId K, P = DeliverableKind, PhaseId design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist roles = { "orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40), PILLAR_ADVISOR: ADVISOR, **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def delegate(*kinds): return Call("delegate", {"kinds": [kind.value for kind in kinds]}) def approve(phase): return Call("request_approval", {"phase": phase.value, "summary": "done"}) submit = sample_submission(K.DESIGN_RECOMMENDATIONS) runtime = FakeAgentRuntime( { "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [ Call( "reopen", {"kind": "design_recommendations", "reason": "Tighten IAM."}, ), delegate(K.DESIGN_RECOMMENDATIONS), ], [], # stops here: a pre-approved Execution ends with warnings ], DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [ [sample_submission(K.REQUIREMENTS)] ], DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [ [sample_submission(K.TECHNICAL_DECOMPOSITION)] ], DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [ [sample_submission(K.PRODUCT_SELECTION)] ], design: [ [Call(CONSULT_TOOL, {"brief": "v1"}), submit], [Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit], ], PILLAR_ADVISOR: [turn_for_pillar] * 4, } ) engine = ExecutionEngine( runtime=runtime, roles=roles, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, tool_factory=make_tool_factory(runtime, roles, PILLARS), ) result = await engine.start("Ingestion", pre_approved=True) advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR] assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"] second = [r for r in runtime.runs if r.role == design][1] assert "## reliability (from your earlier consultation)" in second.results[0][1] stored = result.state["working"]["design_recommendations"]["pillar_advice"] assert set(stored) == set(PILLARS) # --- each advisor reviews its pillar's checklist ---------------------------------------- def reviewed(task): pillar = task.context["pillar"] return [ Call( "submit_advice", { "content": f"Use strong defaults for {pillar}.", "checklist": [ { "item": f"{pillar} item one | with a bar", "status": "addressed", "note": "Covered by the first recommendation.", }, { "item": f"{pillar} item two", "status": "not_applicable", "note": "An organisation's process, not a design decision.", }, {"item": "malformed", "status": "maybe", "note": "dropped"}, ], }, ) ] async def test_each_advisors_checklist_review_is_kept_beside_its_advice(): from app.tools.pillars import CHECKLIST_KEY memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory) result = await consult(tool, brief="b") # The design advisor is given the advice, not the review: it does not copy it. assert "item one" not in result.message assert set(memory[CHECKLIST_KEY]) == set(PILLARS) review = memory[CHECKLIST_KEY]["security"] assert [entry["status"] for entry in review] == ["addressed", "not_applicable"] async def test_the_review_is_written_into_the_deliverable_by_code(): from app.adk.fake_runtime import sample_submission from app.adk.runtime import Submitted from app.tools.execution import specialist_tools from app.tools.pillars import REVIEW_HEADING from app.workflows.deliverables import DeliverableKind memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3}) await consult( pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b" ) kind = DeliverableKind.DESIGN_RECOMMENDATIONS (submit,) = ( tool for tool in specialist_tools(kind, (), allow_cloud_read=False, working=memory) if tool.name == "submit_deliverable" ) call = sample_submission(kind) first = await submit.handler(**call.args) # A Revision submits content that already carries the section. again = await submit.handler( content=first.outcome.content, attachments=call.args["attachments"] ) assert isinstance(first.outcome, Submitted) content = first.outcome.content assert f"## {REVIEW_HEADING}" in content and "### security" in content assert ( "| security item one \\| with a bar | Addressed | Covered by the first " "recommendation. |" ) in content assert "| security item two | Does not apply |" in content assert "malformed" not in content assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1 async def test_with_no_review_nothing_is_added(): from app.adk.fake_runtime import sample_submission from app.tools.execution import specialist_tools from app.tools.pillars import REVIEW_HEADING from app.workflows.deliverables import DeliverableKind kind = DeliverableKind.DESIGN_RECOMMENDATIONS (submit,) = ( tool for tool in specialist_tools(kind, (), allow_cloud_read=False, working={}) if tool.name == "submit_deliverable" ) result = await submit.handler(**sample_submission(kind).args) assert REVIEW_HEADING not in result.outcome.content # --- only the pillars a change touches are consulted again ------------------------------- async def test_a_change_names_the_pillars_it_affects_directly(): from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer from app.tools.pillars import pillars_touched decider = FakeDecider( { "security": Answer(1.9, 0.9), "reliability": Answer(1.0, 0.6), "sustainability": Answer(0.2, 0.8), } ) chosen = await pillars_touched( decider, list(PILLARS), "Use customer-managed keys.", scope="e1" ) state, questions, scope = decider.asked[0] assert chosen == ["security"] assert scope == "e1" and state.endswith("Use customer-managed keys.") assert questions["security"].kind == "score" assert questions["security"].options == ( "not at all", "only indirectly", "directly", ) async def test_a_change_that_touches_no_pillar_is_an_answer_and_no_decision_is_not(): from app.adk.fake_runtime import FakeDecider from app.adk.runtime import Answer, NoDecider from app.tools.pillars import pillars_touched nothing = FakeDecider(dict.fromkeys(PILLARS, Answer(0.0, 0.99))) assert await pillars_touched(nothing, list(PILLARS), "Rename a page.") == [] assert await pillars_touched(NoDecider(), list(PILLARS), "Rename a page.") is None assert await pillars_touched(nothing, list(PILLARS), " ") is None async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said(): memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory) await consult(tool, brief="b") again = await consult(tool, brief="b", pillars=[]) assert len(runtime.runs) == 3 # nobody was consulted a second time for pillar in PILLARS: assert f"## {pillar} (from your earlier consultation)" in again.message # --- what the request does not say is a question for the customer ------------------------ def unsure(question: str | None = "Do you have a central identity provider?"): def turn(task): entry = { "item": "Identity is managed centrally", "status": "needs_customer_input", "note": "The request does not say how staff identities are managed.", } if question is not None: entry["question"] = question first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]}) # A second attempt, used only when the first is refused. fixed = Call( "submit_advice", { "content": "Advice.", "checklist": [{**entry, "question": "Who manages staff identities?"}], }, ) return [first, fixed] return turn async def test_an_item_the_request_cannot_settle_is_kept_with_its_question(): from app.tools.pillars import CHECKLIST_KEY, customer_questions memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3}) await consult( pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b" ) (entry,) = memory[CHECKLIST_KEY]["security"] assert entry["status"] == "needs_customer_input" assert customer_questions(memory)[0] == { "pillar": "security", "item": "Identity is managed centrally", "question": "Do you have a central identity provider?", } assert len(customer_questions(memory)) == len(PILLARS) async def test_an_item_for_the_customer_must_carry_the_question_to_ask(): from app.tools.pillars import CHECKLIST_KEY memory: dict = {} runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3}) await consult( pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b" ) refused, accepted = (reply for _, reply in runtime.runs[0].results) assert refused.startswith("Not submitted: an item to confirm with the customer") assert "Identity is managed centrally" in refused (entry,) = memory[CHECKLIST_KEY]["security"] assert entry["question"] == "Who manages staff identities?" assert accepted == "Advice submitted." def test_the_questions_are_reported_as_missing_information_not_as_faults(): from app.tools.pillars import ( CHECKLIST_KEY, QUESTIONS_HEADING, render_checklist_review, ) working = { CHECKLIST_KEY: { "security": [ { "item": "Scans run in the pipeline", "status": "addressed", "note": "DR-3", }, { "item": "Identity is managed centrally", "status": "needs_customer_input", "note": "Not said.", "question": "Do you have a central identity provider?", }, ] } } text = render_checklist_review(working) assert "| Identity is managed centrally | To confirm with the customer |" in text assert f"### {QUESTIONS_HEADING}" in text assert "They are not faults in the design" in text assert ( "| security | Identity is managed centrally | Do you have a central " "identity provider? |" ) in text # With nothing to ask there is no questions section. only_addressed = { CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]} } assert QUESTIONS_HEADING not in render_checklist_review(only_addressed) async def test_an_execution_writes_its_questions_for_the_customer(tmp_path): import json from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import sample_submission from app.adk.runtime import NoDecider from app.agent import ExecutionEngine from app.nodes.router import Router, Thresholds from app.recorder import InMemoryRecorder from app.tools.pillars import CONSULT_TOOL from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind kind = DeliverableKind.DESIGN_RECOMMENDATIONS advisor = DEPENDENCY_GRAPH[kind].specialist roles = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), PILLAR_ADVISOR: ADVISOR, **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } turns = { spec.specialist: [[sample_submission(k)]] for k, spec in DEPENDENCY_GRAPH.items() } turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]] turns[PILLAR_ADVISOR] = [unsure()] * 3 runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( runtime=runtime, roles=roles, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), tool_factory=make_tool_factory(runtime, roles, pillar_skills=PILLARS), ) result = await engine.start("Ingest", customer_case="acme", pre_approved=True) assert result.status == "completed" out = next(tmp_path.rglob("execution.json")).parent manifest = json.loads((out / "execution.json").read_text()) assert manifest["customer_questions"] == "customer-questions.md" questions = (out / "customer-questions.md").read_text() assert questions.startswith("# Questions for the customer") assert "Do you have a central identity provider?" in questions assert ( "### Questions for the customer" in (out / "deliverables" / "design_recommendations.md").read_text() )