feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -122,15 +122,15 @@ async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
|
||||
|
||||
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
|
||||
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
|
||||
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
|
||||
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
|
||||
"mermaid_check"
|
||||
]
|
||||
assert [t.name for t in factory(analyst, "requirements")] == [
|
||||
assert [t.name for t in factory(analyst, "requirements", {})] == [
|
||||
"compute__list_instances"
|
||||
]
|
||||
|
||||
[consult_tool] = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
await consult_tool.handler(brief="b")
|
||||
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
|
||||
@@ -140,8 +140,171 @@ async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
||||
roles = {PILLAR_ADVISOR: ADVISOR}
|
||||
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
|
||||
|
||||
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
|
||||
design = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
|
||||
|
||||
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
||||
assert list(other) == []
|
||||
|
||||
|
||||
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
|
||||
|
||||
|
||||
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
|
||||
memory: dict = {}
|
||||
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
|
||||
)
|
||||
|
||||
def revised(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
|
||||
|
||||
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
|
||||
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2", pillars=["reliability"])
|
||||
|
||||
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
|
||||
assert "## reliability\n\nRevised advice for reliability." in result.message
|
||||
assert (
|
||||
"## security (from your earlier consultation)\n\n"
|
||||
"Use strong defaults for security." in result.message
|
||||
)
|
||||
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
|
||||
assert set(memory["pillar_advice"]) == set(PILLARS)
|
||||
|
||||
|
||||
async def test_omitting_pillars_consults_every_pillar_afresh():
|
||||
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2")
|
||||
|
||||
assert len(runtime.runs) == 3
|
||||
assert "earlier consultation" not in result.message
|
||||
assert "old" not in memory["pillar_advice"].values()
|
||||
|
||||
|
||||
async def test_failed_advice_is_not_remembered():
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
|
||||
tool = pillar_consultation_tool(
|
||||
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
|
||||
)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
assert "## reliability\n\nNo guidance" in result.message
|
||||
assert set(memory["pillar_advice"]) == {"security"}
|
||||
|
||||
|
||||
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
|
||||
import asyncio
|
||||
|
||||
from app.adk.runtime import RunLimiter
|
||||
|
||||
limiter = RunLimiter(1)
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
|
||||
|
||||
async with limiter.slot(): # the design advisor's own run holds the only slot
|
||||
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
|
||||
|
||||
assert peak == 1
|
||||
assert result.message.count("Use strong defaults") == 3
|
||||
|
||||
|
||||
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
|
||||
K, P = DeliverableKind, PhaseId
|
||||
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
PILLAR_ADVISOR: ADVISOR,
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
def delegate(*kinds):
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
def approve(phase):
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "design_recommendations", "reason": "Tighten IAM."},
|
||||
),
|
||||
delegate(K.DESIGN_RECOMMENDATIONS),
|
||||
],
|
||||
[], # stops here: a pre-approved Execution ends with warnings
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
|
||||
[sample_submission(K.REQUIREMENTS)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
|
||||
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
|
||||
[sample_submission(K.PRODUCT_SELECTION)]
|
||||
],
|
||||
design: [
|
||||
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
|
||||
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
|
||||
],
|
||||
PILLAR_ADVISOR: [turn_for_pillar] * 4,
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
tool_factory=make_tool_factory(runtime, roles, PILLARS),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
|
||||
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
|
||||
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
|
||||
second = [r for r in runtime.runs if r.role == design][1]
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
Reference in New Issue
Block a user