feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -3,13 +3,14 @@
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
"""
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
@@ -50,12 +51,7 @@ def reopen(kind: DeliverableKind, reason: str) -> Call:
FINISH = Call("finish")
def submission(kind: DeliverableKind, note: str = "") -> Call:
spec = DEPENDENCY_GRAPH[kind]
body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections)
content = f"# {kind}\n{note or 'Summary.'}\n{body}\n"
attachments = {name: f"# {name}\n" for name in spec.attachments}
return Call("submit_deliverable", {"content": content, "attachments": attachments})
submission = sample_submission
def submits(kind: DeliverableKind) -> list[Call]:
@@ -84,9 +80,16 @@ def happy_path_turns(**extra_specialist_turns) -> dict:
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
request_approval(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[
request_approval(P.SOLUTION_IMPLEMENTATION),
delegate(K.VALIDATION_RESULTS),
],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
],
@@ -112,7 +115,14 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
assert set(ledger["phases"].values()) == {"approved"}
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
out = tmp_path / "deliverables" / "executions" / result.execution_id
out = (
tmp_path
/ "deliverables"
/ "cases"
/ "unassigned"
/ "executions"
/ result.execution_id
)
assert (out / "solution-architecture-guide.md").is_file()
for kind in K:
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
@@ -122,7 +132,7 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
recorded_types = [e.type for e in recorder.events[result.execution_id]]
assert recorded_types.count("approved") == 4
assert recorded_types.count("approved") == 5
async def test_independent_specialists_run_together_between_orchestrator_turns(
@@ -133,12 +143,12 @@ async def test_independent_specialists_run_together_between_orchestrator_turns(
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
roles = [run.role for run in runtime.runs]
fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3]
assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == {
specialist(K.ARCHITECTURE_DIAGRAM),
specialist(K.DESIGN_RECOMMENDATIONS),
sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5]
assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == {
specialist(K.ARCHITECTURE_DESCRIPTION),
specialist(K.DEPLOYMENT_GUIDANCE),
}
assert roles[fourth_turn + 3] == "orchestrator"
assert roles[sixth_turn + 3] == "orchestrator"
async def test_specialist_receives_its_dependencies_content(tmp_path):
@@ -154,9 +164,13 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
"requirements",
"technical_decomposition",
"product_selection",
"architecture_diagram",
"design_recommendations",
"target_architecture",
]
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
# The engine drew the diagram from the graph when the Target architecture was submitted.
assert dependencies["target_architecture"]["attachments"][
"architecture.mmd"
].startswith("flowchart")
assert describer_run.task.deliverable == "architecture_description"
@@ -334,16 +348,17 @@ async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path)
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
tmp_path,
):
def diagrammer_turn(task):
def describer_turn(task):
if task.context["answers"]:
return submits(K.ARCHITECTURE_DIAGRAM)
return [Call("needs_input", {"questions": ["Single region or multi-region?"]})]
return submits(K.ARCHITECTURE_DESCRIPTION)
return [Call("needs_input", {"questions": ["Who reads this description?"]})]
def advisor_turn(task):
def iac_turn(task):
if task.context["answers"]:
return submits(K.DESIGN_RECOMMENDATIONS)
return [Call("needs_input", {"questions": ["What is the RTO?"]})]
return submits(K.DEPLOYMENT_GUIDANCE)
return [Call("needs_input", {"questions": ["Which region?"]})]
both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
runtime = FakeAgentRuntime(
{
"orchestrator": [
@@ -351,47 +366,59 @@ async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[request_approval(P.SOLUTION_ARCHITECTURE)],
[both],
[ask_user()],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[both],
[ask_user("Continue?")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn],
specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn],
**{
specialist(kind): [submits(kind)]
for kind in (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
K.TARGET_ARCHITECTURE,
)
},
specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn],
specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn],
}
)
engine = make_engine(tmp_path, runtime)
approval = await engine.start("Ingestion")
first = await engine.resume(
design = await engine.resume(
approval.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
first = await engine.resume(
design.execution_id,
{"approval": {"phase": "solution_architecture", "decision": "approve"}},
)
assert first.prompt == {
"questions": [
{
"from": "architecture_diagram",
"question": "Single region or multi-region?",
"from": "architecture_description",
"question": "Who reads this description?",
},
{"from": "design_recommendations", "question": "What is the RTO?"},
{"from": "deployment_guidance", "question": "Which region?"},
]
}
second = await engine.resume(
first.execution_id, {"reply": "Single region, RTO 1 hour"}
)
second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"})
deliverables = second.state["ledger"]["deliverables"]
assert deliverables["architecture_diagram"]["status"] == "submitted"
assert deliverables["design_recommendations"]["status"] == "submitted"
advisor_retry = [
r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS)
assert deliverables["architecture_description"]["status"] == "submitted"
assert deliverables["deployment_guidance"]["status"] == "submitted"
iac_retry = [
r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert advisor_retry.task.context["answers"] == [
{"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"}
assert iac_retry.task.context["answers"] == [
{"questions": ["Which region?"], "reply": "Engineers, in London"}
]
@@ -404,14 +431,18 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
request_approval(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
]
@@ -430,9 +461,10 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
assert result.status == "completed"
events = result.state["ledger"]["events"]
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
assert withdrawn == ["solution_architecture"]
assert withdrawn == ["solution_implementation"]
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
assert requests.count("solution_architecture") == 2
assert requests.count("solution_implementation") == 2
assert requests.count("solution_architecture") == 1
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
assert iac_runs[1].task.context["feedback"] == [
"terraform plan: missing IAM binding"
@@ -456,7 +488,12 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
assert result.status == "completed_with_warnings"
assert "revision limit" in result.state["warnings"][0]
assert not (tmp_path / "deliverables").exists()
# No Deliverable files are written, but the request stays on record.
case = tmp_path / "deliverables" / "cases" / "unassigned"
assert not (case / "executions").exists()
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
assert intent["status"] == "completed_with_warnings"
assert intent["execution"] is None
# --- budgets -------------------------------------------------------------------------------------
@@ -514,7 +551,7 @@ def cloud_read_factory(calls: list[str]):
tier="cloud_read",
)
def factory(role: RoleSpec, deliverable: str | None):
def factory(role: RoleSpec, deliverable: str | None, working: dict):
return [tool] if role.name == "requirements-analyst" else []
return factory
@@ -686,3 +723,249 @@ async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
assert "No user is available" not in first_reply
second_reply = result.state["conversation"][4]["text"]
assert second_reply.startswith("No user is available")
# --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) -------
async def test_specialist_context_carries_its_contract(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS))
assert run.task.context["contract"] == {
"sections": ["Deployment dry-run", "Architecture and policy analysis"],
"attachments": ["verdicts.json"],
}
assert run.task.context["draft"] is None
async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path):
draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args
def confirm_then_submit_what_came_back(task):
returned = task.context["draft"]
return [Call("submit_deliverable", returned)]
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()],
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
[
Call(
"needs_input",
{
"questions": ["Do you approve this draft?"],
"draft": draft["content"],
"attachments": [
{"path": path, "content": content}
for path, content in draft["attachments"].items()
],
},
)
],
confirm_then_submit_what_came_back,
],
}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingestion")
assert paused.state["drafts"]["requirements"]["content"] == draft["content"]
after = await engine.resume(paused.execution_id, {"reply": "Approved."})
second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1]
assert second_run.task.context["draft"] == {
"content": draft["content"],
"attachments": draft["attachments"],
}
assert second_run.task.context["answers"][-1]["reply"] == "Approved."
assert after.state["deliverables"]["requirements"]["content"] == draft["content"]
assert after.state["drafts"] == {}
# --- cap on parallel Specialists (iteration 4, step 5) ---------------------------------
async def test_parallel_specialists_never_exceed_the_cap(tmp_path):
import asyncio
from app.adk.runtime import RunLimiter
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
if role.name == "orchestrator":
return await super().run(role, task, tools)
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime(happy_path_turns())
engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1))
result = await engine.start("Ingestion", pre_approved=True)
assert result.status == "completed"
assert peak == 1
async def test_without_a_cap_independent_specialists_overlap(tmp_path):
import asyncio
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
if role.name == "orchestrator":
return await super().run(role, task, tools)
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
assert peak == 2
# --- Customer cases group Executions ---------------------------------------------------
async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
recorder = InMemoryRecorder()
engine = make_engine(
tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder
)
result = await engine.start(
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
)
case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks"
out = case_dir / "executions" / result.execution_id
assert (out / "solution-architecture-guide.md").is_file()
assert result.state["customer_case"] == "acme-webhooks"
manifest = json.loads((out / "execution.json").read_text(encoding="utf-8"))
assert manifest["customer_case"] == "acme-webhooks"
assert manifest["workflow_request"] == "Ingestion"
assert manifest["intent_version"] == 1
assert manifest["solution_guide"] == "solution-architecture-guide.md"
assert manifest["deliverables"]["deployment_guidance"] == {
"document": "deliverables/deployment_guidance.md",
"attachments": [
"implementation.json",
"terraform/main.tf",
"terraform/variables.tf",
],
}
for entry in manifest["deliverables"].values():
assert (out / entry["document"]).is_file()
assert all((out / name).is_file() for name in entry["attachments"])
assert (
recorder.statuses[result.execution_id][-1]["details"]["customer_case"]
== "acme-webhooks"
)
lessons = (
tmp_path
/ "lessons"
/ "candidates"
/ "acme-webhooks"
/ f"{result.execution_id}.json"
)
assert lessons.is_file()
assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks"
async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path):
first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
)
second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion, revised", pre_approved=True, customer_case="acme-webhooks"
)
case = tmp_path / "deliverables" / "cases" / "acme-webhooks"
assert {path.name for path in (case / "executions").iterdir()} == {
first.execution_id,
second.execution_id,
}
history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8"))
assert history["customer_case"] == "acme-webhooks"
assert [
(i["version"], i["workflow_request"], i["status"], i["execution"])
for i in history["intents"]
] == [
(1, "Ingestion", "completed", f"executions/{first.execution_id}"),
(2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"),
]
assert "Ingestion, revised" in (case / "intent" / "002.md").read_text()
assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2
# A Customer case groups Executions; it never feeds one into the next.
assert second.state["conversation"][0]["text"] == "Ingestion, revised"
assert "deliverables" in second.state and first.state != second.state
@pytest.mark.parametrize(
"bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7]
)
async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad):
engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns()))
with pytest.raises(ValueError, match="customer_case"):
await engine.start("Ingestion", pre_approved=True, customer_case=bad)
assert not (tmp_path / "deliverables").exists()
async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path):
result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion", pre_approved=True
)
assert result.state["customer_case"] == "unassigned"
assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"]
assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [
"unassigned"
]
async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
runtime = FakeAgentRuntime(
{"orchestrator": [[ask_user("Which region should this run in?")]]}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks")
assert paused.status == "awaiting_user"
intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent"
statement = (intent / "001.md").read_text(encoding="utf-8")
assert "Ingest signed webhooks" in statement
assert paused.execution_id in statement
[entry] = json.loads((intent / "history.json").read_text())["intents"]
assert entry["status"] == "in_progress"
assert entry["statement"] == "intent/001.md"
assert entry["execution_id"] == paused.execution_id