Files
gcp_solution_architecture_a…/tests/execution/test_requirements_chain.py
Jonathan Boniface 4c4d44ef87 fix: show the Orchestrator a validation only while it stands
After deployment guidance was reopened, the Orchestrator's context still
carried the previous validation's verdicts, so it reopened the code for the
same Check three times without a validation in between (evaluation run of
2026-10-07, case 01). Record that run in the iteration 6 plan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 15:08:09 +01:00

620 lines
21 KiB
Python

"""The Requirements chain: the Target architecture graph and the data files around it.
ADR-0005: the outcome is understood before code is written, and validation answers
the graph's Checks and nothing else.
"""
import json
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.tools.execution import specialist_tools
from app.tools.local import mermaid_check
from app.workflows.chain import validation_outcome
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
DeliverableKind,
PhaseId,
check_submission,
)
from app.workflows.target_architecture import (
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
K = DeliverableKind
P = PhaseId
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
def graph_data() -> dict:
return json.loads(SAMPLE_DATA["target-architecture.json"])
def problems(data: dict) -> tuple[str, ...]:
with pytest.raises(GraphError) as raised:
parse_graph(json.dumps(data))
return raised.value.problems
def gaps(data: dict, **overrides) -> tuple[str, ...]:
links = {
"requirements": REQUIREMENTS,
"components": COMPONENTS,
"products": PRODUCTS,
"recommendations": RECOMMENDATIONS,
**overrides,
}
return chain_gaps(parse_graph(json.dumps(data)), **links)
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
"""The Deliverables a submission builds on, from the coherent sample."""
built = {}
for kind in kinds:
args = sample_submission(kind).args
attachments = {
name: replaced.get(name, text) for name, text in args["attachments"].items()
}
built[kind.value] = {"content": args["content"], "attachments": attachments}
return built
DESIGN = (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
)
# --- the graph is well-formed ----------------------------------------------------------
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
assert graph.elements[1].component is None # a supporting Element
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
assert graph.check_ids == ("K-1", "K-2")
assert graph.summary() == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
def test_a_relationship_must_join_two_elements():
data = graph_data()
data["relationships"][0]["to"] = "E-9"
assert problems(data) == (
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
)
def test_a_check_must_apply_to_something_in_the_graph():
data = graph_data()
data["checks"][0]["applies_to"] = "E-9"
assert problems(data) == (
(
"target-architecture.json: Check K-1 applies to E-9, "
"which is not an Element or Relationship"
),
"target-architecture.json: Element E-1 has no Check",
)
def test_every_element_needs_a_check_and_every_id_is_used_once():
data = graph_data()
data["checks"] = [data["checks"][0]]
data["relationships"][0]["id"] = "E-1"
assert problems(data) == (
"target-architecture.json: id E-1 is used more than once",
"target-architecture.json: Element E-2 has no Check",
)
def test_a_check_without_a_statement_is_named():
data = graph_data()
data["checks"][1]["statement"] = " "
assert (
"target-architecture.json.checks[1]: needs a non-empty statement"
in problems(data)
)
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
with pytest.raises(GraphError):
parse_graph(text)
# --- the Requirements chain has no gap -------------------------------------------------
def test_the_sample_chain_is_complete():
assert gaps(graph_data()) == ()
def test_a_non_functional_requirement_with_no_check_is_a_gap():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
"Requirement NFR-2 has no Check",
)
def test_a_functional_requirement_is_carried_by_its_components_element():
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
def test_a_component_with_no_element_is_a_gap():
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
"Component C-2 has no Element",
)
def test_a_selected_product_that_no_element_uses_is_a_gap():
armour = [{"component": "C-1", "product": "Example Armor"}]
assert gaps(graph_data(), products=armour) == (
"Product Example Armor selected for C-1 appears on no Element",
)
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
recommendations = [*RECOMMENDATIONS, cmek]
assert gaps(graph_data(), recommendations=recommendations) == (
"Recommendation DR-2 has no Check",
)
excused = graph_data()
excused["not_applicable"] = [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
assert gaps(excused, recommendations=recommendations) == ()
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
def test_a_check_must_come_from_a_requirement_or_recommendation():
data = graph_data()
data["checks"][0]["source"] = "my own idea"
assert gaps(data) == (
(
"Check K-1 has source my own idea, which is not a requirement or design "
"recommendation"
),
"Requirement NFR-1 has no Check",
)
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
built_on = upstream(
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
)
args = sample_submission(K.TARGET_ARCHITECTURE).args
assert check_submission(
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
) == ("Requirement NFR-2 has no Check",)
assert (
check_submission(
K.TARGET_ARCHITECTURE,
args["content"],
args["attachments"],
upstream(*DESIGN),
)
== ()
)
# --- the links before the graph --------------------------------------------------------
def test_requirements_need_ids_and_a_kind():
def missing(items) -> tuple[str, ...]:
args = sample_submission(K.REQUIREMENTS).args
return check_submission(
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
)
assert missing(REQUIREMENTS) == ()
assert missing([]) == ("requirements.json: lists no requirements",)
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
"requirements.json: id FR-1 is used more than once",
"requirements.json: FR-1 kind must be one of functional, non_functional",
)
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
"requirements.json[0]: needs text",
)
def test_every_functional_requirement_is_served_by_a_component():
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
def missing(components) -> tuple[str, ...]:
return check_submission(
K.TECHNICAL_DECOMPOSITION,
args["content"],
{"components.json": json.dumps(components)},
upstream(K.REQUIREMENTS),
)
assert missing(COMPONENTS) == ()
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
"components.json: C-1 serves NFR-9, which is not a requirement",
"components.json: functional requirement FR-1 is served by no component",
)
def test_every_component_gets_a_product_and_recommendations_name_real_components():
selection = sample_submission(K.PRODUCT_SELECTION).args
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert check_submission(
K.PRODUCT_SELECTION,
selection["content"],
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
built_on,
) == (
"products.json: C-7 is not a component",
"products.json: component C-1 has no product selected",
)
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
assert check_submission(
K.DESIGN_RECOMMENDATIONS,
advice["content"],
{"recommendations.json": json.dumps(wandering)},
built_on,
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
# --- code and validation answer the graph ----------------------------------------------
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
def test_deployment_guidance_must_account_for_every_check():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
partial = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": partial},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json: K-7 is not a Check in the Target architecture",
"implementation.json: Check K-2 is not accounted for",
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
)
def verdicts(**changes) -> str:
data = json.loads(SAMPLE_DATA["verdicts.json"])
data.update(changes)
return json.dumps(data)
def validation_missing(text: str) -> tuple[str, ...]:
args = sample_submission(K.VALIDATION_RESULTS).args
return check_submission(
K.VALIDATION_RESULTS,
args["content"],
{"verdicts.json": text},
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
)
def test_validation_gives_exactly_one_verdict_per_check():
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
assert validation_missing(verdicts(checks=[one, one, invented])) == (
"verdicts.json.checks: check K-1 is used more than once",
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
"""What it notices beyond the Checks is a proposed addition, never a failure."""
addition = {
"statement": "Cloud Armor rate limiting is configured.",
"applies_to": "E-1",
"reason": "NFR-1",
}
text = verdicts(proposed_additions=[addition])
assert validation_missing(text) == ()
outcome = validation_outcome(text)
assert outcome["failed"] == []
assert outcome["proposed_additions"] == [addition]
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
vague = [
{"check": "K-1", "verdict": "probably", "evidence": "x"},
{"check": "K-2", "verdict": "pass", "evidence": " "},
]
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
"verdicts.json.checks[1]: needs evidence",
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
checks = [
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
]
outcome = validation_outcome(verdicts(checks=checks))
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
assert outcome["not_verifiable"] == [
{"check": "K-2", "evidence": "No such argument."}
]
assert outcome["passed"] == []
assert outcome["dry_run"]["verdict"] == "pass"
# --- the diagram is a view of the graph ------------------------------------------------
async def test_the_diagram_shows_every_element_and_relationship_once():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
diagram = render_mermaid(graph)
assert diagram == render_mermaid(graph) # deterministic
assert diagram.count("Ingress service") == 1
assert diagram.count("Ingress identity") == 1
assert diagram.count("-->") == 1
assert "runs as" in diagram
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
async def test_names_with_characters_mermaid_reserves_still_render():
data = graph_data()
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
data["elements"][0]["id"] = "svc/ingest.v2"
data["relationships"][0]["from"] = "svc/ingest.v2"
data["checks"][0]["applies_to"] = "svc/ingest.v2"
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
diagram = render_mermaid(parse_graph(json.dumps(data)))
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
assert "svc/ingest.v2" not in diagram
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
submit = specialist_tools(
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=upstream(*DESIGN),
)[0]
args = sample_submission(K.TARGET_ARCHITECTURE).args
result = await submit.handler(**args)
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
parse_graph(SAMPLE_DATA["target-architecture.json"])
)
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
built_on = upstream(
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
)
submit = specialist_tools(
K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=built_on
)[0]
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
assert result.outcome is None
assert result.message == (
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
)
# --- through the engine ----------------------------------------------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def to_design_approval() -> list[list[Call]]:
return [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
]
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
return engine, runtime
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
# Tries to start the code before asking: refused, then asks.
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
],
},
)
first = await engine.start("Ingest webhooks")
paused = await engine.resume(
first.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert paused.prompt["approval"]["phase"] == "solution_architecture"
assert paused.prompt["approval"]["target_architecture"] == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
refused = runtime.runs[-1].results[0]
assert refused[0] == "delegate" and refused[1].startswith("Refused")
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
failing = json.loads(SAMPLE_DATA["verdicts.json"])
failing["checks"][0] = {
"check": "K-1",
"verdict": "fail",
"evidence": "min instances is 1",
}
failed_validation = sample_submission(K.VALIDATION_RESULTS)
failed_validation = Call(
failed_validation.tool,
{
**failed_validation.args,
"attachments": {"verdicts.json": json.dumps(failing)},
},
)
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
*to_design_approval(),
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{
"kind": "deployment_guidance",
"reason": "K-1 failed: min instances is 1",
},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
},
)
result = await engine.start("Ingest webhooks", pre_approved=True)
turns = [run for run in runtime.runs if run.role == "orchestrator"]
before, after = turns[6].task.context, turns[7].task.context
assert before["validation"] is None
assert after["validation"]["failed"] == [
{"check": "K-1", "evidence": "min instances is 1"}
]
assert after["validation"]["proposed_additions"] == []
# Once the code is reopened those verdicts are about work that no longer exists,
# so the next turn must not see them and reopen the same thing again.
assert turns[8].task.context["validation"] is None
# Revising the code leaves the approved design standing.
phases = result.state["ledger"]["phases"]
assert phases["solution_architecture"] == "approved"
revised = [
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]