feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -47,6 +47,7 @@ def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist(
|
||||
assert PHASE_ORDER == (
|
||||
PhaseId.REQUIREMENTS_DISCOVERY,
|
||||
PhaseId.SOLUTION_ARCHITECTURE,
|
||||
PhaseId.SOLUTION_IMPLEMENTATION,
|
||||
PhaseId.SOLUTION_VALIDATION,
|
||||
PhaseId.PACKAGING,
|
||||
)
|
||||
@@ -56,55 +57,68 @@ def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist(
|
||||
assert len(specialists) == len(set(specialists))
|
||||
|
||||
|
||||
def test_graph_matches_the_upstream_skill_dependencies():
|
||||
def test_the_target_architecture_sits_between_the_design_and_the_code():
|
||||
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
|
||||
K = DeliverableKind
|
||||
|
||||
assert depends[K.REQUIREMENTS] == set()
|
||||
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
|
||||
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
|
||||
assert depends[K.ARCHITECTURE_DIAGRAM] == {K.PRODUCT_SELECTION}
|
||||
assert depends[K.ARCHITECTURE_DESCRIPTION] == {
|
||||
K.ARCHITECTURE_DIAGRAM,
|
||||
K.PRODUCT_SELECTION,
|
||||
}
|
||||
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
|
||||
assert depends[K.DEPLOYMENT_GUIDANCE] == {
|
||||
assert depends[K.TARGET_ARCHITECTURE] == {
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
}
|
||||
# Everything after the graph is derived from it.
|
||||
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
|
||||
assert depends[K.SOLUTION_GUIDE] == set(
|
||||
deliverables_in(PhaseId.SOLUTION_VALIDATION)
|
||||
)
|
||||
|
||||
|
||||
def test_the_design_is_approved_before_code_is_written():
|
||||
design = PhaseId.SOLUTION_ARCHITECTURE
|
||||
implementation = PhaseId.SOLUTION_IMPLEMENTATION
|
||||
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
|
||||
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
|
||||
|
||||
|
||||
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
|
||||
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
|
||||
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == (
|
||||
design = (
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.DESIGN_RECOMMENDATIONS,
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
)
|
||||
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == (
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.ARCHITECTURE_DIAGRAM,
|
||||
)
|
||||
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
|
||||
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
|
||||
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
|
||||
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:4]
|
||||
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
|
||||
DeliverableKind.SOLUTION_GUIDE
|
||||
}
|
||||
|
||||
|
||||
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
|
||||
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
|
||||
|
||||
|
||||
def test_submission_meeting_the_contract_has_nothing_missing():
|
||||
content = (
|
||||
"# Deployment guidance\n"
|
||||
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
|
||||
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
|
||||
)
|
||||
attachments = {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
|
||||
attachments = {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "variable {}",
|
||||
"implementation.json": "[]",
|
||||
}
|
||||
|
||||
assert (
|
||||
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
|
||||
@@ -122,6 +136,7 @@ def test_submission_reports_missing_sections_and_attachments():
|
||||
assert missing == (
|
||||
"section: Step-by-step deployment instructions",
|
||||
"attachment: terraform/variables.tf",
|
||||
"attachment: implementation.json",
|
||||
)
|
||||
|
||||
|
||||
@@ -143,7 +158,7 @@ def test_section_matching_ignores_numbering_case_and_trailing_punctuation(headin
|
||||
"## Sustainability\n"
|
||||
)
|
||||
|
||||
assert check_submission(DeliverableKind.DESIGN_RECOMMENDATIONS, content, {}) == ()
|
||||
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
|
||||
|
||||
|
||||
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
|
||||
@@ -152,7 +167,7 @@ def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
|
||||
"## Non-functional requirements\n"
|
||||
)
|
||||
|
||||
assert check_submission(DeliverableKind.PRODUCT_SELECTION, content, {}) == ()
|
||||
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
|
||||
assert "section: Functional requirements" in check_submission(
|
||||
DeliverableKind.REQUIREMENTS, content, {}
|
||||
)
|
||||
@@ -173,3 +188,32 @@ def test_empty_content_is_refused():
|
||||
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
|
||||
"content: must not be empty",
|
||||
)
|
||||
|
||||
|
||||
# --- contracts beyond headings ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_architecture_description_needs_component_and_flow_sections():
|
||||
missing = check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
|
||||
)
|
||||
|
||||
assert missing == ("section: Components", "section: Flow")
|
||||
assert (
|
||||
check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION,
|
||||
"## Architecture components\nA.\n## Data flow\nB.\n",
|
||||
{},
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_a_required_attachment_must_have_content():
|
||||
missing = check_submission(
|
||||
DeliverableKind.VALIDATION_RESULTS,
|
||||
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
|
||||
{"verdicts.json": " \n"},
|
||||
)
|
||||
|
||||
assert missing == ("attachment: verdicts.json must not be empty",)
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
@@ -50,12 +51,7 @@ def reopen(kind: DeliverableKind, reason: str) -> Call:
|
||||
FINISH = Call("finish")
|
||||
|
||||
|
||||
def submission(kind: DeliverableKind, note: str = "") -> Call:
|
||||
spec = DEPENDENCY_GRAPH[kind]
|
||||
body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections)
|
||||
content = f"# {kind}\n{note or 'Summary.'}\n{body}\n"
|
||||
attachments = {name: f"# {name}\n" for name in spec.attachments}
|
||||
return Call("submit_deliverable", {"content": content, "attachments": attachments})
|
||||
submission = sample_submission
|
||||
|
||||
|
||||
def submits(kind: DeliverableKind) -> list[Call]:
|
||||
@@ -84,9 +80,16 @@ def happy_path_turns(**extra_specialist_turns) -> dict:
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
request_approval(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[
|
||||
request_approval(P.SOLUTION_IMPLEMENTATION),
|
||||
delegate(K.VALIDATION_RESULTS),
|
||||
],
|
||||
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[request_approval(P.PACKAGING), FINISH],
|
||||
],
|
||||
@@ -112,7 +115,14 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
|
||||
assert set(ledger["phases"].values()) == {"approved"}
|
||||
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
|
||||
|
||||
out = tmp_path / "deliverables" / "executions" / result.execution_id
|
||||
out = (
|
||||
tmp_path
|
||||
/ "deliverables"
|
||||
/ "cases"
|
||||
/ "unassigned"
|
||||
/ "executions"
|
||||
/ result.execution_id
|
||||
)
|
||||
assert (out / "solution-architecture-guide.md").is_file()
|
||||
for kind in K:
|
||||
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
|
||||
@@ -122,7 +132,7 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
|
||||
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
|
||||
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
|
||||
recorded_types = [e.type for e in recorder.events[result.execution_id]]
|
||||
assert recorded_types.count("approved") == 4
|
||||
assert recorded_types.count("approved") == 5
|
||||
|
||||
|
||||
async def test_independent_specialists_run_together_between_orchestrator_turns(
|
||||
@@ -133,12 +143,12 @@ async def test_independent_specialists_run_together_between_orchestrator_turns(
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
roles = [run.role for run in runtime.runs]
|
||||
fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3]
|
||||
assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == {
|
||||
specialist(K.ARCHITECTURE_DIAGRAM),
|
||||
specialist(K.DESIGN_RECOMMENDATIONS),
|
||||
sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5]
|
||||
assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == {
|
||||
specialist(K.ARCHITECTURE_DESCRIPTION),
|
||||
specialist(K.DEPLOYMENT_GUIDANCE),
|
||||
}
|
||||
assert roles[fourth_turn + 3] == "orchestrator"
|
||||
assert roles[sixth_turn + 3] == "orchestrator"
|
||||
|
||||
|
||||
async def test_specialist_receives_its_dependencies_content(tmp_path):
|
||||
@@ -154,9 +164,13 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"architecture_diagram",
|
||||
"design_recommendations",
|
||||
"target_architecture",
|
||||
]
|
||||
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
|
||||
# The engine drew the diagram from the graph when the Target architecture was submitted.
|
||||
assert dependencies["target_architecture"]["attachments"][
|
||||
"architecture.mmd"
|
||||
].startswith("flowchart")
|
||||
assert describer_run.task.deliverable == "architecture_description"
|
||||
|
||||
|
||||
@@ -334,16 +348,17 @@ async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path)
|
||||
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
|
||||
tmp_path,
|
||||
):
|
||||
def diagrammer_turn(task):
|
||||
def describer_turn(task):
|
||||
if task.context["answers"]:
|
||||
return submits(K.ARCHITECTURE_DIAGRAM)
|
||||
return [Call("needs_input", {"questions": ["Single region or multi-region?"]})]
|
||||
return submits(K.ARCHITECTURE_DESCRIPTION)
|
||||
return [Call("needs_input", {"questions": ["Who reads this description?"]})]
|
||||
|
||||
def advisor_turn(task):
|
||||
def iac_turn(task):
|
||||
if task.context["answers"]:
|
||||
return submits(K.DESIGN_RECOMMENDATIONS)
|
||||
return [Call("needs_input", {"questions": ["What is the RTO?"]})]
|
||||
return submits(K.DEPLOYMENT_GUIDANCE)
|
||||
return [Call("needs_input", {"questions": ["Which region?"]})]
|
||||
|
||||
both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
@@ -351,47 +366,59 @@ async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
||||
[delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE)],
|
||||
[both],
|
||||
[ask_user()],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[both],
|
||||
[ask_user("Continue?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
||||
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
|
||||
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
|
||||
specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn],
|
||||
specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn],
|
||||
**{
|
||||
specialist(kind): [submits(kind)]
|
||||
for kind in (
|
||||
K.REQUIREMENTS,
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
K.TARGET_ARCHITECTURE,
|
||||
)
|
||||
},
|
||||
specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
approval = await engine.start("Ingestion")
|
||||
first = await engine.resume(
|
||||
design = await engine.resume(
|
||||
approval.execution_id,
|
||||
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
||||
)
|
||||
first = await engine.resume(
|
||||
design.execution_id,
|
||||
{"approval": {"phase": "solution_architecture", "decision": "approve"}},
|
||||
)
|
||||
|
||||
assert first.prompt == {
|
||||
"questions": [
|
||||
{
|
||||
"from": "architecture_diagram",
|
||||
"question": "Single region or multi-region?",
|
||||
"from": "architecture_description",
|
||||
"question": "Who reads this description?",
|
||||
},
|
||||
{"from": "design_recommendations", "question": "What is the RTO?"},
|
||||
{"from": "deployment_guidance", "question": "Which region?"},
|
||||
]
|
||||
}
|
||||
|
||||
second = await engine.resume(
|
||||
first.execution_id, {"reply": "Single region, RTO 1 hour"}
|
||||
)
|
||||
second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"})
|
||||
|
||||
deliverables = second.state["ledger"]["deliverables"]
|
||||
assert deliverables["architecture_diagram"]["status"] == "submitted"
|
||||
assert deliverables["design_recommendations"]["status"] == "submitted"
|
||||
advisor_retry = [
|
||||
r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS)
|
||||
assert deliverables["architecture_description"]["status"] == "submitted"
|
||||
assert deliverables["deployment_guidance"]["status"] == "submitted"
|
||||
iac_retry = [
|
||||
r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
||||
][1]
|
||||
assert advisor_retry.task.context["answers"] == [
|
||||
{"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"}
|
||||
assert iac_retry.task.context["answers"] == [
|
||||
{"questions": ["Which region?"], "reply": "Engineers, in London"}
|
||||
]
|
||||
|
||||
|
||||
@@ -404,14 +431,18 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
request_approval(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[request_approval(P.PACKAGING), FINISH],
|
||||
]
|
||||
@@ -430,9 +461,10 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
|
||||
assert result.status == "completed"
|
||||
events = result.state["ledger"]["events"]
|
||||
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
|
||||
assert withdrawn == ["solution_architecture"]
|
||||
assert withdrawn == ["solution_implementation"]
|
||||
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
|
||||
assert requests.count("solution_architecture") == 2
|
||||
assert requests.count("solution_implementation") == 2
|
||||
assert requests.count("solution_architecture") == 1
|
||||
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
|
||||
assert iac_runs[1].task.context["feedback"] == [
|
||||
"terraform plan: missing IAM binding"
|
||||
@@ -456,7 +488,12 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert "revision limit" in result.state["warnings"][0]
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
# No Deliverable files are written, but the request stays on record.
|
||||
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
||||
assert not (case / "executions").exists()
|
||||
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
||||
assert intent["status"] == "completed_with_warnings"
|
||||
assert intent["execution"] is None
|
||||
|
||||
|
||||
# --- budgets -------------------------------------------------------------------------------------
|
||||
@@ -514,7 +551,7 @@ def cloud_read_factory(calls: list[str]):
|
||||
tier="cloud_read",
|
||||
)
|
||||
|
||||
def factory(role: RoleSpec, deliverable: str | None):
|
||||
def factory(role: RoleSpec, deliverable: str | None, working: dict):
|
||||
return [tool] if role.name == "requirements-analyst" else []
|
||||
|
||||
return factory
|
||||
@@ -686,3 +723,249 @@ async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
|
||||
assert "No user is available" not in first_reply
|
||||
second_reply = result.state["conversation"][4]["text"]
|
||||
assert second_reply.startswith("No user is available")
|
||||
|
||||
|
||||
# --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) -------
|
||||
|
||||
|
||||
async def test_specialist_context_carries_its_contract(tmp_path):
|
||||
runtime = FakeAgentRuntime(happy_path_turns())
|
||||
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS))
|
||||
assert run.task.context["contract"] == {
|
||||
"sections": ["Deployment dry-run", "Architecture and policy analysis"],
|
||||
"attachments": ["verdicts.json"],
|
||||
}
|
||||
assert run.task.context["draft"] is None
|
||||
|
||||
|
||||
async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path):
|
||||
draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args
|
||||
|
||||
def confirm_then_submit_what_came_back(task):
|
||||
returned = task.context["draft"]
|
||||
return [Call("submit_deliverable", returned)]
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()],
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[
|
||||
Call(
|
||||
"needs_input",
|
||||
{
|
||||
"questions": ["Do you approve this draft?"],
|
||||
"draft": draft["content"],
|
||||
"attachments": [
|
||||
{"path": path, "content": content}
|
||||
for path, content in draft["attachments"].items()
|
||||
],
|
||||
},
|
||||
)
|
||||
],
|
||||
confirm_then_submit_what_came_back,
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
|
||||
paused = await engine.start("Ingestion")
|
||||
assert paused.state["drafts"]["requirements"]["content"] == draft["content"]
|
||||
|
||||
after = await engine.resume(paused.execution_id, {"reply": "Approved."})
|
||||
|
||||
second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1]
|
||||
assert second_run.task.context["draft"] == {
|
||||
"content": draft["content"],
|
||||
"attachments": draft["attachments"],
|
||||
}
|
||||
assert second_run.task.context["answers"][-1]["reply"] == "Approved."
|
||||
assert after.state["deliverables"]["requirements"]["content"] == draft["content"]
|
||||
assert after.state["drafts"] == {}
|
||||
|
||||
|
||||
# --- cap on parallel Specialists (iteration 4, step 5) ---------------------------------
|
||||
|
||||
|
||||
async def test_parallel_specialists_never_exceed_the_cap(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from app.adk.runtime import RunLimiter
|
||||
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
if role.name == "orchestrator":
|
||||
return await super().run(role, task, tools)
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime(happy_path_turns())
|
||||
engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1))
|
||||
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert peak == 1
|
||||
|
||||
|
||||
async def test_without_a_cap_independent_specialists_overlap(tmp_path):
|
||||
import asyncio
|
||||
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
if role.name == "orchestrator":
|
||||
return await super().run(role, task, tools)
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime(happy_path_turns())
|
||||
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# --- Customer cases group Executions ---------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
||||
recorder = InMemoryRecorder()
|
||||
engine = make_engine(
|
||||
tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
|
||||
case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
||||
out = case_dir / "executions" / result.execution_id
|
||||
assert (out / "solution-architecture-guide.md").is_file()
|
||||
assert result.state["customer_case"] == "acme-webhooks"
|
||||
|
||||
manifest = json.loads((out / "execution.json").read_text(encoding="utf-8"))
|
||||
assert manifest["customer_case"] == "acme-webhooks"
|
||||
assert manifest["workflow_request"] == "Ingestion"
|
||||
assert manifest["intent_version"] == 1
|
||||
assert manifest["solution_guide"] == "solution-architecture-guide.md"
|
||||
assert manifest["deliverables"]["deployment_guidance"] == {
|
||||
"document": "deliverables/deployment_guidance.md",
|
||||
"attachments": [
|
||||
"implementation.json",
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
],
|
||||
}
|
||||
for entry in manifest["deliverables"].values():
|
||||
assert (out / entry["document"]).is_file()
|
||||
assert all((out / name).is_file() for name in entry["attachments"])
|
||||
assert (
|
||||
recorder.statuses[result.execution_id][-1]["details"]["customer_case"]
|
||||
== "acme-webhooks"
|
||||
)
|
||||
lessons = (
|
||||
tmp_path
|
||||
/ "lessons"
|
||||
/ "candidates"
|
||||
/ "acme-webhooks"
|
||||
/ f"{result.execution_id}.json"
|
||||
)
|
||||
assert lessons.is_file()
|
||||
assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks"
|
||||
|
||||
|
||||
async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path):
|
||||
first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion, revised", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
|
||||
case = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
||||
assert {path.name for path in (case / "executions").iterdir()} == {
|
||||
first.execution_id,
|
||||
second.execution_id,
|
||||
}
|
||||
history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8"))
|
||||
assert history["customer_case"] == "acme-webhooks"
|
||||
assert [
|
||||
(i["version"], i["workflow_request"], i["status"], i["execution"])
|
||||
for i in history["intents"]
|
||||
] == [
|
||||
(1, "Ingestion", "completed", f"executions/{first.execution_id}"),
|
||||
(2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"),
|
||||
]
|
||||
assert "Ingestion, revised" in (case / "intent" / "002.md").read_text()
|
||||
assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2
|
||||
# A Customer case groups Executions; it never feeds one into the next.
|
||||
assert second.state["conversation"][0]["text"] == "Ingestion, revised"
|
||||
assert "deliverables" in second.state and first.state != second.state
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7]
|
||||
)
|
||||
async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad):
|
||||
engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns()))
|
||||
|
||||
with pytest.raises(ValueError, match="customer_case"):
|
||||
await engine.start("Ingestion", pre_approved=True, customer_case=bad)
|
||||
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
|
||||
|
||||
async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path):
|
||||
result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion", pre_approved=True
|
||||
)
|
||||
|
||||
assert result.state["customer_case"] == "unassigned"
|
||||
assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"]
|
||||
assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [
|
||||
"unassigned"
|
||||
]
|
||||
|
||||
|
||||
async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
||||
runtime = FakeAgentRuntime(
|
||||
{"orchestrator": [[ask_user("Which region should this run in?")]]}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
|
||||
paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks")
|
||||
|
||||
assert paused.status == "awaiting_user"
|
||||
intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent"
|
||||
statement = (intent / "001.md").read_text(encoding="utf-8")
|
||||
assert "Ingest signed webhooks" in statement
|
||||
assert paused.execution_id in statement
|
||||
[entry] = json.loads((intent / "history.json").read_text())["intents"]
|
||||
assert entry["status"] == "in_progress"
|
||||
assert entry["statement"] == "intent/001.md"
|
||||
assert entry["execution_id"] == paused.execution_id
|
||||
|
||||
@@ -36,12 +36,18 @@ async def test_submission_takes_attachments_as_a_list():
|
||||
content=GUIDANCE,
|
||||
attachments=[
|
||||
{"path": "terraform/main.tf", "content": "resource {}"},
|
||||
{"path": "terraform/variables.tf", "content": ""},
|
||||
{"path": "terraform/variables.tf", "content": "variable {}"},
|
||||
{"path": "implementation.json", "content": "[]"},
|
||||
],
|
||||
)
|
||||
|
||||
assert result.outcome == Submitted(
|
||||
GUIDANCE, {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
|
||||
GUIDANCE,
|
||||
{
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "variable {}",
|
||||
"implementation.json": "[]",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -33,14 +33,11 @@ def through_phase_one(ledger: Ledger) -> None:
|
||||
def through_phase_two(ledger: Ledger) -> None:
|
||||
through_phase_one(ledger)
|
||||
produce(
|
||||
ledger,
|
||||
K.PRODUCT_SELECTION,
|
||||
K.ARCHITECTURE_DIAGRAM,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
|
||||
)
|
||||
approve(ledger, P.SOLUTION_ARCHITECTURE)
|
||||
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
|
||||
approve(ledger, P.SOLUTION_IMPLEMENTATION)
|
||||
|
||||
|
||||
def through_phase_three(ledger: Ledger) -> None:
|
||||
@@ -80,18 +77,18 @@ def test_cross_phase_dependency_needs_the_earlier_phase_approved():
|
||||
def test_independent_work_becomes_ready_together():
|
||||
ledger = Ledger()
|
||||
through_phase_one(ledger)
|
||||
produce(ledger, K.PRODUCT_SELECTION)
|
||||
produce(
|
||||
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
|
||||
)
|
||||
|
||||
assert set(ledger.ready_work()) == {
|
||||
K.ARCHITECTURE_DIAGRAM,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
}
|
||||
# Nothing in implementation is ready until the design is approved.
|
||||
assert ledger.ready_work() == ()
|
||||
|
||||
produce(ledger, K.ARCHITECTURE_DIAGRAM)
|
||||
approve(ledger, P.SOLUTION_ARCHITECTURE)
|
||||
|
||||
assert set(ledger.ready_work()) == {
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
}
|
||||
|
||||
|
||||
@@ -259,10 +256,12 @@ def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
|
||||
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
|
||||
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
|
||||
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.OPEN
|
||||
# Revising the code leaves the approved design standing.
|
||||
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
|
||||
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
|
||||
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
|
||||
assert set(result.withdrawn_approvals) == {
|
||||
P.SOLUTION_ARCHITECTURE,
|
||||
P.SOLUTION_IMPLEMENTATION,
|
||||
P.SOLUTION_VALIDATION,
|
||||
}
|
||||
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
|
||||
@@ -278,15 +277,15 @@ def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
|
||||
ledger = Ledger()
|
||||
through_phase_one(ledger)
|
||||
produce(ledger, K.PRODUCT_SELECTION)
|
||||
attempt = ledger.start([K.ARCHITECTURE_DIAGRAM], actor="orchestrator")[
|
||||
K.ARCHITECTURE_DIAGRAM
|
||||
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
|
||||
K.DESIGN_RECOMMENDATIONS
|
||||
]
|
||||
|
||||
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
|
||||
|
||||
assert ledger.status(K.ARCHITECTURE_DIAGRAM) is DeliverableStatus.PENDING
|
||||
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
|
||||
with pytest.raises(LedgerRefusal, match="attempt"):
|
||||
ledger.submit(K.ARCHITECTURE_DIAGRAM, attempt, actor="diagrammer")
|
||||
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
|
||||
|
||||
|
||||
def test_reopen_refuses_deliverables_that_were_never_submitted():
|
||||
@@ -356,3 +355,27 @@ def test_round_trips_through_plain_data():
|
||||
assert restored.ready_work() == ledger.ready_work()
|
||||
assert restored.max_revisions == 5
|
||||
assert restored.events == ledger.events
|
||||
|
||||
|
||||
def test_revising_the_target_architecture_returns_the_code_to_draft():
|
||||
ledger = Ledger()
|
||||
through_phase_three(ledger)
|
||||
|
||||
result = ledger.reopen(
|
||||
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
|
||||
)
|
||||
|
||||
assert set(result.reset) == {
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
K.VALIDATION_RESULTS,
|
||||
}
|
||||
assert set(result.withdrawn_approvals) == {
|
||||
P.SOLUTION_ARCHITECTURE,
|
||||
P.SOLUTION_IMPLEMENTATION,
|
||||
P.SOLUTION_VALIDATION,
|
||||
}
|
||||
# The code cannot be rewritten until the revised design is approved again.
|
||||
produce(ledger, K.TARGET_ARCHITECTURE)
|
||||
assert ledger.ready_work() == ()
|
||||
|
||||
239
tests/execution/test_lessons.py
Normal file
239
tests/execution/test_lessons.py
Normal file
@@ -0,0 +1,239 @@
|
||||
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.lessons import refusal
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec(
|
||||
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
|
||||
),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
|
||||
ADDITION = {
|
||||
"statement": "The service sets a request timeout.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1 has no Check on timeouts",
|
||||
}
|
||||
|
||||
|
||||
def delegate(*kinds: DeliverableKind) -> Call:
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
|
||||
def approve(phase: PhaseId) -> Call:
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def failing_validation() -> Call:
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
|
||||
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
|
||||
verdicts["proposed_additions"] = [ADDITION]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def turns_with_one_revision() -> dict:
|
||||
once = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
return {
|
||||
**once,
|
||||
"orchestrator": [
|
||||
# The first call is refused: requirements are not done yet.
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
approve(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
|
||||
),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[approve(P.PACKAGING), Call("finish")],
|
||||
],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [
|
||||
[
|
||||
Call("submit_deliverable", {"content": "No headings yet."}),
|
||||
sample_submission(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
],
|
||||
specialist(K.VALIDATION_RESULTS): [
|
||||
[failing_validation()],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def run(tmp_path, turns, **options):
|
||||
recorder = InMemoryRecorder()
|
||||
engine = ExecutionEngine(
|
||||
runtime=FakeAgentRuntime(turns),
|
||||
roles=ROLES,
|
||||
recorder=recorder,
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
skill_versions={"solution-architecture": "abc123"},
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
return result, recorder
|
||||
|
||||
|
||||
def of_type(record: dict, kind: str) -> list[dict]:
|
||||
return [c for c in record["candidates"] if c["type"] == kind]
|
||||
|
||||
|
||||
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
assert result.status == "completed"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert of_type(record, "revision") == [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "deployment_guidance",
|
||||
"reason": "Fix the ingress block.",
|
||||
"requested_by": "orchestrator",
|
||||
}
|
||||
]
|
||||
assert of_type(record, "failed_check") == [
|
||||
{"type": "failed_check", "attempt": 1, **FAILED}
|
||||
]
|
||||
assert of_type(record, "proposed_addition") == [
|
||||
{"type": "proposed_addition", "attempt": 1, **ADDITION}
|
||||
]
|
||||
refusals = of_type(record, "refusal")
|
||||
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
|
||||
("orchestrator", "delegate", None),
|
||||
(
|
||||
specialist(K.DEPLOYMENT_GUIDANCE),
|
||||
"submit_deliverable",
|
||||
"deployment_guidance",
|
||||
),
|
||||
}
|
||||
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
|
||||
|
||||
|
||||
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert record["execution_id"] == result.execution_id
|
||||
assert record["status"] == "completed"
|
||||
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
|
||||
assert record["skill_versions"] == {"solution-architecture": "abc123"}
|
||||
|
||||
|
||||
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
path = (
|
||||
tmp_path
|
||||
/ "lessons"
|
||||
/ "candidates"
|
||||
/ "unassigned"
|
||||
/ f"{result.execution_id}.json"
|
||||
)
|
||||
|
||||
assert (
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
== (recorder.lessons[result.execution_id])
|
||||
)
|
||||
|
||||
|
||||
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
|
||||
turns = {
|
||||
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
|
||||
specialist(K.REQUIREMENTS): [[]], # stops without submitting
|
||||
}
|
||||
|
||||
result, recorder = await run(tmp_path, turns)
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
assert of_type(record, "unfinished") == [
|
||||
{
|
||||
"type": "unfinished",
|
||||
"deliverable": "requirements",
|
||||
"role": specialist(K.REQUIREMENTS),
|
||||
"reason": "stopped",
|
||||
}
|
||||
]
|
||||
assert not (
|
||||
tmp_path / "deliverables" / "cases" / "unassigned" / "executions"
|
||||
).exists()
|
||||
|
||||
|
||||
async def test_a_cloud_read_refusal_keeps_only_its_leading_words(tmp_path):
|
||||
async def list_instances() -> ToolResult:
|
||||
return ToolResult("Discovery gap: project acme-prod-1234 API not enabled.")
|
||||
|
||||
tool = ToolSpec(
|
||||
"compute__list_instances",
|
||||
"List instances.",
|
||||
{"type": "object", "properties": {}},
|
||||
list_instances,
|
||||
tier="cloud_read",
|
||||
)
|
||||
|
||||
def factory(role, deliverable, working):
|
||||
return [tool] if deliverable == "requirements" else []
|
||||
|
||||
turns = {
|
||||
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[Call("compute__list_instances"), sample_submission(K.REQUIREMENTS)]
|
||||
],
|
||||
}
|
||||
|
||||
result, recorder = await run(tmp_path, turns, tool_factory=factory, max_revisions=3)
|
||||
await_record = recorder.lessons[result.execution_id]
|
||||
|
||||
# allow_cloud_read is False in a pre-approved Execution, so the gate refuses.
|
||||
[gap] = of_type(await_record, "refusal")
|
||||
assert gap["tool"] == "compute__list_instances"
|
||||
assert "acme-prod-1234" not in gap["message"]
|
||||
|
||||
|
||||
def test_refusal_messages_are_bounded():
|
||||
item = refusal(
|
||||
"validator", "validation_results", "terraform_validate", "general", "x" * 900
|
||||
)
|
||||
|
||||
assert len(item["message"]) == 500
|
||||
@@ -122,15 +122,15 @@ async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
|
||||
|
||||
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
|
||||
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
|
||||
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
|
||||
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
|
||||
"mermaid_check"
|
||||
]
|
||||
assert [t.name for t in factory(analyst, "requirements")] == [
|
||||
assert [t.name for t in factory(analyst, "requirements", {})] == [
|
||||
"compute__list_instances"
|
||||
]
|
||||
|
||||
[consult_tool] = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
await consult_tool.handler(brief="b")
|
||||
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
|
||||
@@ -140,8 +140,171 @@ async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
||||
roles = {PILLAR_ADVISOR: ADVISOR}
|
||||
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
|
||||
|
||||
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
|
||||
design = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
|
||||
|
||||
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
||||
assert list(other) == []
|
||||
|
||||
|
||||
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
|
||||
|
||||
|
||||
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
|
||||
memory: dict = {}
|
||||
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
|
||||
)
|
||||
|
||||
def revised(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
|
||||
|
||||
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
|
||||
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2", pillars=["reliability"])
|
||||
|
||||
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
|
||||
assert "## reliability\n\nRevised advice for reliability." in result.message
|
||||
assert (
|
||||
"## security (from your earlier consultation)\n\n"
|
||||
"Use strong defaults for security." in result.message
|
||||
)
|
||||
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
|
||||
assert set(memory["pillar_advice"]) == set(PILLARS)
|
||||
|
||||
|
||||
async def test_omitting_pillars_consults_every_pillar_afresh():
|
||||
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2")
|
||||
|
||||
assert len(runtime.runs) == 3
|
||||
assert "earlier consultation" not in result.message
|
||||
assert "old" not in memory["pillar_advice"].values()
|
||||
|
||||
|
||||
async def test_failed_advice_is_not_remembered():
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
|
||||
tool = pillar_consultation_tool(
|
||||
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
|
||||
)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
assert "## reliability\n\nNo guidance" in result.message
|
||||
assert set(memory["pillar_advice"]) == {"security"}
|
||||
|
||||
|
||||
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
|
||||
import asyncio
|
||||
|
||||
from app.adk.runtime import RunLimiter
|
||||
|
||||
limiter = RunLimiter(1)
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
|
||||
|
||||
async with limiter.slot(): # the design advisor's own run holds the only slot
|
||||
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
|
||||
|
||||
assert peak == 1
|
||||
assert result.message.count("Use strong defaults") == 3
|
||||
|
||||
|
||||
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
|
||||
K, P = DeliverableKind, PhaseId
|
||||
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
PILLAR_ADVISOR: ADVISOR,
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
def delegate(*kinds):
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
def approve(phase):
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "design_recommendations", "reason": "Tighten IAM."},
|
||||
),
|
||||
delegate(K.DESIGN_RECOMMENDATIONS),
|
||||
],
|
||||
[], # stops here: a pre-approved Execution ends with warnings
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
|
||||
[sample_submission(K.REQUIREMENTS)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
|
||||
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
|
||||
[sample_submission(K.PRODUCT_SELECTION)]
|
||||
],
|
||||
design: [
|
||||
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
|
||||
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
|
||||
],
|
||||
PILLAR_ADVISOR: [turn_for_pillar] * 4,
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
tool_factory=make_tool_factory(runtime, roles, PILLARS),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
|
||||
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
|
||||
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
|
||||
second = [r for r in runtime.runs if r.role == design][1]
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
@@ -12,7 +12,7 @@ def test_approval_prompt_explains_the_structured_reply():
|
||||
"approval": {
|
||||
"phase": "solution_architecture",
|
||||
"summary": "Cloud Run and Pub/Sub",
|
||||
"deliverables": ["product_selection", "architecture_diagram"],
|
||||
"deliverables": ["product_selection", "target_architecture"],
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -74,3 +74,27 @@ def test_generate_response_maps_deliverables_onto_legacy_artifacts():
|
||||
assert body["artifacts"]["terraform_code"] == "resource {}"
|
||||
assert body["artifacts"]["solution_guide"] is None
|
||||
assert body["warnings"] == ["revision limit"]
|
||||
|
||||
|
||||
def test_the_design_approval_says_what_the_user_is_committing_to():
|
||||
text = describe_prompt(
|
||||
{
|
||||
"approval": {
|
||||
"phase": "solution_architecture",
|
||||
"summary": "Cloud Run and Pub/Sub",
|
||||
"deliverables": ["target_architecture"],
|
||||
"target_architecture": {
|
||||
"elements": 9,
|
||||
"relationships": 11,
|
||||
"checks": 23,
|
||||
"not_applicable": [
|
||||
{"recommendation": "DR-9", "reason": "Single region."}
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert "9 Elements, 11 Relationships and 23 Checks" in text
|
||||
assert "No infrastructure code is written until you approve it" in text
|
||||
assert "Recommendation DR-9 is treated as not applicable: Single region." in text
|
||||
|
||||
616
tests/execution/test_requirements_chain.py
Normal file
616
tests/execution/test_requirements_chain.py
Normal file
@@ -0,0 +1,616 @@
|
||||
"""The Requirements chain: the Target architecture graph and the data files around it.
|
||||
|
||||
ADR-0005: the outcome is understood before code is written, and validation answers
|
||||
the graph's Checks and nothing else.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.chain import validation_outcome
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
check_submission,
|
||||
)
|
||||
from app.workflows.target_architecture import (
|
||||
GraphError,
|
||||
chain_gaps,
|
||||
parse_graph,
|
||||
render_mermaid,
|
||||
)
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
|
||||
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
|
||||
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
|
||||
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
|
||||
|
||||
|
||||
def graph_data() -> dict:
|
||||
return json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
|
||||
def problems(data: dict) -> tuple[str, ...]:
|
||||
with pytest.raises(GraphError) as raised:
|
||||
parse_graph(json.dumps(data))
|
||||
return raised.value.problems
|
||||
|
||||
|
||||
def gaps(data: dict, **overrides) -> tuple[str, ...]:
|
||||
links = {
|
||||
"requirements": REQUIREMENTS,
|
||||
"components": COMPONENTS,
|
||||
"products": PRODUCTS,
|
||||
"recommendations": RECOMMENDATIONS,
|
||||
**overrides,
|
||||
}
|
||||
return chain_gaps(parse_graph(json.dumps(data)), **links)
|
||||
|
||||
|
||||
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
|
||||
"""The Deliverables a submission builds on, from the coherent sample."""
|
||||
built = {}
|
||||
for kind in kinds:
|
||||
args = sample_submission(kind).args
|
||||
attachments = {
|
||||
name: replaced.get(name, text) for name, text in args["attachments"].items()
|
||||
}
|
||||
built[kind.value] = {"content": args["content"], "attachments": attachments}
|
||||
return built
|
||||
|
||||
|
||||
DESIGN = (
|
||||
K.REQUIREMENTS,
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
)
|
||||
|
||||
|
||||
# --- the graph is well-formed ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
|
||||
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
|
||||
assert graph.elements[1].component is None # a supporting Element
|
||||
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
|
||||
assert graph.check_ids == ("K-1", "K-2")
|
||||
assert graph.summary() == {
|
||||
"elements": 2,
|
||||
"relationships": 1,
|
||||
"checks": 2,
|
||||
"not_applicable": [],
|
||||
}
|
||||
|
||||
|
||||
def test_a_relationship_must_join_two_elements():
|
||||
data = graph_data()
|
||||
data["relationships"][0]["to"] = "E-9"
|
||||
|
||||
assert problems(data) == (
|
||||
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_must_apply_to_something_in_the_graph():
|
||||
data = graph_data()
|
||||
data["checks"][0]["applies_to"] = "E-9"
|
||||
|
||||
assert problems(data) == (
|
||||
(
|
||||
"target-architecture.json: Check K-1 applies to E-9, "
|
||||
"which is not an Element or Relationship"
|
||||
),
|
||||
"target-architecture.json: Element E-1 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_every_element_needs_a_check_and_every_id_is_used_once():
|
||||
data = graph_data()
|
||||
data["checks"] = [data["checks"][0]]
|
||||
data["relationships"][0]["id"] = "E-1"
|
||||
|
||||
assert problems(data) == (
|
||||
"target-architecture.json: id E-1 is used more than once",
|
||||
"target-architecture.json: Element E-2 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_without_a_statement_is_named():
|
||||
data = graph_data()
|
||||
data["checks"][1]["statement"] = " "
|
||||
|
||||
assert (
|
||||
"target-architecture.json.checks[1]: needs a non-empty statement"
|
||||
in problems(data)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
|
||||
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
|
||||
with pytest.raises(GraphError):
|
||||
parse_graph(text)
|
||||
|
||||
|
||||
# --- the Requirements chain has no gap -------------------------------------------------
|
||||
|
||||
|
||||
def test_the_sample_chain_is_complete():
|
||||
assert gaps(graph_data()) == ()
|
||||
|
||||
|
||||
def test_a_non_functional_requirement_with_no_check_is_a_gap():
|
||||
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
||||
|
||||
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
|
||||
"Requirement NFR-2 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_functional_requirement_is_carried_by_its_components_element():
|
||||
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
|
||||
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
|
||||
|
||||
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
|
||||
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
|
||||
|
||||
|
||||
def test_a_component_with_no_element_is_a_gap():
|
||||
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
|
||||
|
||||
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
|
||||
"Component C-2 has no Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_selected_product_that_no_element_uses_is_a_gap():
|
||||
armour = [{"component": "C-1", "product": "Example Armor"}]
|
||||
|
||||
assert gaps(graph_data(), products=armour) == (
|
||||
"Product Example Armor selected for C-1 appears on no Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
|
||||
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
||||
recommendations = [*RECOMMENDATIONS, cmek]
|
||||
|
||||
assert gaps(graph_data(), recommendations=recommendations) == (
|
||||
"Recommendation DR-2 has no Check",
|
||||
)
|
||||
|
||||
excused = graph_data()
|
||||
excused["not_applicable"] = [
|
||||
{"recommendation": "DR-2", "reason": "No regulated data."}
|
||||
]
|
||||
assert gaps(excused, recommendations=recommendations) == ()
|
||||
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
|
||||
{"recommendation": "DR-2", "reason": "No regulated data."}
|
||||
]
|
||||
|
||||
|
||||
def test_a_check_must_come_from_a_requirement_or_recommendation():
|
||||
data = graph_data()
|
||||
data["checks"][0]["source"] = "my own idea"
|
||||
|
||||
assert gaps(data) == (
|
||||
(
|
||||
"Check K-1 has source my own idea, which is not a requirement or design "
|
||||
"recommendation"
|
||||
),
|
||||
"Requirement NFR-1 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
|
||||
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
||||
built_on = upstream(
|
||||
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
|
||||
)
|
||||
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
||||
|
||||
assert check_submission(
|
||||
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
|
||||
) == ("Requirement NFR-2 has no Check",)
|
||||
assert (
|
||||
check_submission(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
args["content"],
|
||||
args["attachments"],
|
||||
upstream(*DESIGN),
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
# --- the links before the graph --------------------------------------------------------
|
||||
|
||||
|
||||
def test_requirements_need_ids_and_a_kind():
|
||||
def missing(items) -> tuple[str, ...]:
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
return check_submission(
|
||||
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
|
||||
)
|
||||
|
||||
assert missing(REQUIREMENTS) == ()
|
||||
assert missing([]) == ("requirements.json: lists no requirements",)
|
||||
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
|
||||
"requirements.json: id FR-1 is used more than once",
|
||||
"requirements.json: FR-1 kind must be one of functional, non_functional",
|
||||
)
|
||||
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
|
||||
"requirements.json[0]: needs text",
|
||||
)
|
||||
|
||||
|
||||
def test_every_functional_requirement_is_served_by_a_component():
|
||||
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
|
||||
|
||||
def missing(components) -> tuple[str, ...]:
|
||||
return check_submission(
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
args["content"],
|
||||
{"components.json": json.dumps(components)},
|
||||
upstream(K.REQUIREMENTS),
|
||||
)
|
||||
|
||||
assert missing(COMPONENTS) == ()
|
||||
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
|
||||
"components.json: C-1 serves NFR-9, which is not a requirement",
|
||||
"components.json: functional requirement FR-1 is served by no component",
|
||||
)
|
||||
|
||||
|
||||
def test_every_component_gets_a_product_and_recommendations_name_real_components():
|
||||
selection = sample_submission(K.PRODUCT_SELECTION).args
|
||||
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
||||
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
assert check_submission(
|
||||
K.PRODUCT_SELECTION,
|
||||
selection["content"],
|
||||
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
|
||||
built_on,
|
||||
) == (
|
||||
"products.json: C-7 is not a component",
|
||||
"products.json: component C-1 has no product selected",
|
||||
)
|
||||
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
|
||||
assert check_submission(
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
advice["content"],
|
||||
{"recommendations.json": json.dumps(wandering)},
|
||||
built_on,
|
||||
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
|
||||
|
||||
|
||||
# --- code and validation answer the graph ----------------------------------------------
|
||||
|
||||
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
|
||||
|
||||
|
||||
def test_deployment_guidance_must_account_for_every_check():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
partial = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": partial},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"implementation.json: K-7 is not a Check in the Target architecture",
|
||||
"implementation.json: Check K-2 is not accounted for",
|
||||
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
|
||||
)
|
||||
|
||||
|
||||
def verdicts(**changes) -> str:
|
||||
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
data.update(changes)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def validation_missing(text: str) -> tuple[str, ...]:
|
||||
args = sample_submission(K.VALIDATION_RESULTS).args
|
||||
return check_submission(
|
||||
K.VALIDATION_RESULTS,
|
||||
args["content"],
|
||||
{"verdicts.json": text},
|
||||
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
|
||||
)
|
||||
|
||||
|
||||
def test_validation_gives_exactly_one_verdict_per_check():
|
||||
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
|
||||
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
|
||||
|
||||
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
|
||||
assert validation_missing(verdicts(checks=[one, one, invented])) == (
|
||||
"verdicts.json.checks: check K-1 is used more than once",
|
||||
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
|
||||
"verdicts.json.checks: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
|
||||
"""What it notices beyond the Checks is a proposed addition, never a failure."""
|
||||
addition = {
|
||||
"statement": "Cloud Armor rate limiting is configured.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1",
|
||||
}
|
||||
text = verdicts(proposed_additions=[addition])
|
||||
|
||||
assert validation_missing(text) == ()
|
||||
outcome = validation_outcome(text)
|
||||
assert outcome["failed"] == []
|
||||
assert outcome["proposed_additions"] == [addition]
|
||||
|
||||
|
||||
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
|
||||
vague = [
|
||||
{"check": "K-1", "verdict": "probably", "evidence": "x"},
|
||||
{"check": "K-2", "verdict": "pass", "evidence": " "},
|
||||
]
|
||||
|
||||
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
|
||||
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
|
||||
"verdicts.json.checks[1]: needs evidence",
|
||||
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
|
||||
"verdicts.json.checks: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
|
||||
checks = [
|
||||
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
|
||||
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
|
||||
]
|
||||
|
||||
outcome = validation_outcome(verdicts(checks=checks))
|
||||
|
||||
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
|
||||
assert outcome["not_verifiable"] == [
|
||||
{"check": "K-2", "evidence": "No such argument."}
|
||||
]
|
||||
assert outcome["passed"] == []
|
||||
assert outcome["dry_run"]["verdict"] == "pass"
|
||||
|
||||
|
||||
# --- the diagram is a view of the graph ------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_diagram_shows_every_element_and_relationship_once():
|
||||
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
diagram = render_mermaid(graph)
|
||||
|
||||
assert diagram == render_mermaid(graph) # deterministic
|
||||
assert diagram.count("Ingress service") == 1
|
||||
assert diagram.count("Ingress identity") == 1
|
||||
assert diagram.count("-->") == 1
|
||||
assert "runs as" in diagram
|
||||
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
||||
|
||||
|
||||
async def test_names_with_characters_mermaid_reserves_still_render():
|
||||
data = graph_data()
|
||||
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
|
||||
data["elements"][0]["id"] = "svc/ingest.v2"
|
||||
data["relationships"][0]["from"] = "svc/ingest.v2"
|
||||
data["checks"][0]["applies_to"] = "svc/ingest.v2"
|
||||
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
|
||||
|
||||
diagram = render_mermaid(parse_graph(json.dumps(data)))
|
||||
|
||||
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
||||
assert "svc/ingest.v2" not in diagram
|
||||
|
||||
|
||||
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
|
||||
submit = specialist_tools(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
extra=[],
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream(*DESIGN),
|
||||
)[0]
|
||||
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
||||
|
||||
result = await submit.handler(**args)
|
||||
|
||||
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
)
|
||||
|
||||
|
||||
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
|
||||
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
||||
built_on = upstream(
|
||||
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
|
||||
)
|
||||
submit = specialist_tools(
|
||||
K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=built_on
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
|
||||
|
||||
assert result.outcome is None
|
||||
assert result.message == (
|
||||
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
|
||||
)
|
||||
|
||||
|
||||
# --- through the engine ----------------------------------------------------------------
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def delegate(*kinds: DeliverableKind) -> Call:
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
|
||||
def approve(phase: PhaseId) -> Call:
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def to_design_approval() -> list[list[Call]]:
|
||||
return [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
]
|
||||
|
||||
|
||||
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
|
||||
once = {
|
||||
specialist(kind): [[sample_submission(kind)]]
|
||||
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
|
||||
}
|
||||
engine, runtime = engine_for(
|
||||
tmp_path,
|
||||
{
|
||||
**once,
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY)],
|
||||
[delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
# Tries to start the code before asking: refused, then asks.
|
||||
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
paused = await engine.resume(
|
||||
first.execution_id,
|
||||
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
||||
)
|
||||
|
||||
assert paused.prompt["approval"]["phase"] == "solution_architecture"
|
||||
assert paused.prompt["approval"]["target_architecture"] == {
|
||||
"elements": 2,
|
||||
"relationships": 1,
|
||||
"checks": 2,
|
||||
"not_applicable": [],
|
||||
}
|
||||
refused = runtime.runs[-1].results[0]
|
||||
assert refused[0] == "delegate" and refused[1].startswith("Refused")
|
||||
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
|
||||
|
||||
|
||||
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
|
||||
failing = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
failing["checks"][0] = {
|
||||
"check": "K-1",
|
||||
"verdict": "fail",
|
||||
"evidence": "min instances is 1",
|
||||
}
|
||||
failed_validation = sample_submission(K.VALIDATION_RESULTS)
|
||||
failed_validation = Call(
|
||||
failed_validation.tool,
|
||||
{
|
||||
**failed_validation.args,
|
||||
"attachments": {"verdicts.json": json.dumps(failing)},
|
||||
},
|
||||
)
|
||||
once = {
|
||||
specialist(kind): [[sample_submission(kind)]]
|
||||
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
|
||||
}
|
||||
engine, runtime = engine_for(
|
||||
tmp_path,
|
||||
{
|
||||
**once,
|
||||
"orchestrator": [
|
||||
*to_design_approval(),
|
||||
[
|
||||
approve(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{
|
||||
"kind": "deployment_guidance",
|
||||
"reason": "K-1 failed: min instances is 1",
|
||||
},
|
||||
),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[],
|
||||
],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
],
|
||||
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
|
||||
},
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
turns = [run for run in runtime.runs if run.role == "orchestrator"]
|
||||
before, after = turns[6].task.context, turns[7].task.context
|
||||
assert before["validation"] is None
|
||||
assert after["validation"]["failed"] == [
|
||||
{"check": "K-1", "evidence": "min instances is 1"}
|
||||
]
|
||||
assert after["validation"]["proposed_additions"] == []
|
||||
# Revising the code leaves the approved design standing.
|
||||
phases = result.state["ledger"]["phases"]
|
||||
assert phases["solution_architecture"] == "approved"
|
||||
revised = [
|
||||
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
||||
][1]
|
||||
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
|
||||
Reference in New Issue
Block a user