feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -122,15 +122,15 @@ async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
"mermaid_check"
]
assert [t.name for t in factory(analyst, "requirements")] == [
assert [t.name for t in factory(analyst, "requirements", {})] == [
"compute__list_instances"
]
[consult_tool] = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
await consult_tool.handler(brief="b")
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
@@ -140,8 +140,171 @@ async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
design = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
assert [tool.name for tool in design] == [CONSULT_TOOL]
assert list(other) == []
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
memory: dict = {}
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
)
def revised(task):
pillar = task.context["pillar"]
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2", pillars=["reliability"])
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
assert "## reliability\n\nRevised advice for reliability." in result.message
assert (
"## security (from your earlier consultation)\n\n"
"Use strong defaults for security." in result.message
)
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
assert set(memory["pillar_advice"]) == set(PILLARS)
async def test_omitting_pillars_consults_every_pillar_afresh():
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2")
assert len(runtime.runs) == 3
assert "earlier consultation" not in result.message
assert "old" not in memory["pillar_advice"].values()
async def test_failed_advice_is_not_remembered():
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
tool = pillar_consultation_tool(
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
)
result = await consult(tool, brief="b")
assert "## reliability\n\nNo guidance" in result.message
assert set(memory["pillar_advice"]) == {"security"}
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
import asyncio
from app.adk.runtime import RunLimiter
limiter = RunLimiter(1)
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
async with limiter.slot(): # the design advisor's own run holds the only slot
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
assert peak == 1
assert result.message.count("Use strong defaults") == 3
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import sample_submission
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
K, P = DeliverableKind, PhaseId
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
PILLAR_ADVISOR: ADVISOR,
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds):
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase):
return Call("request_approval", {"phase": phase.value, "summary": "done"})
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[
Call(
"reopen",
{"kind": "design_recommendations", "reason": "Tighten IAM."},
),
delegate(K.DESIGN_RECOMMENDATIONS),
],
[], # stops here: a pre-approved Execution ends with warnings
],
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
[sample_submission(K.REQUIREMENTS)]
],
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
],
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
[sample_submission(K.PRODUCT_SELECTION)]
],
design: [
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
],
PILLAR_ADVISOR: [turn_for_pillar] * 4,
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
tool_factory=make_tool_factory(runtime, roles, PILLARS),
)
result = await engine.start("Ingestion", pre_approved=True)
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
second = [r for r in runtime.runs if r.role == design][1]
assert "## reliability (from your earlier consultation)" in second.results[0][1]
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
assert set(stored) == set(PILLARS)