Files
orchestration_engine/tests/execution/test_requirements_chain.py

889 lines
29 KiB
Python

"""The Requirements chain: the Target architecture graph and the data files around it.
ADR-0005: the outcome is understood before code is written, and validation answers
the graph's Checks and nothing else.
"""
import json
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.chain import unimplementable_checks, validation_outcome
from tests.reference_workflow.local import mermaid_check
from tests.reference_workflow.target_architecture import (
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
PhaseId,
check_submission,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
def graph_data() -> dict:
return json.loads(SAMPLE_DATA["target-architecture.json"])
def problems(data: dict) -> tuple[str, ...]:
with pytest.raises(GraphError) as raised:
parse_graph(json.dumps(data))
return raised.value.problems
def gaps(data: dict, **overrides) -> tuple[str, ...]:
links = {
"requirements": REQUIREMENTS,
"components": COMPONENTS,
"products": PRODUCTS,
"recommendations": RECOMMENDATIONS,
**overrides,
}
return chain_gaps(parse_graph(json.dumps(data)), **links)
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
"""The Deliverables a submission builds on, from the coherent sample."""
built = {}
for kind in kinds:
args = sample_submission(kind).args
attachments = {
name: replaced.get(name, text) for name, text in args["attachments"].items()
}
built[kind.value] = {"content": args["content"], "attachments": attachments}
return built
DESIGN = (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
)
# --- the graph is well-formed ----------------------------------------------------------
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
assert graph.elements[1].component is None # a supporting Element
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
assert graph.check_ids == ("K-1", "K-2")
assert graph.summary() == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
def test_a_relationship_must_join_two_elements():
data = graph_data()
data["relationships"][0]["to"] = "E-9"
assert problems(data) == (
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
)
def test_a_check_must_apply_to_something_in_the_graph():
data = graph_data()
data["checks"][0]["applies_to"] = "E-9"
assert problems(data) == (
(
"target-architecture.json: Check K-1 applies to E-9, "
"which is not an Element or Relationship"
),
"target-architecture.json: Element E-1 has no Check",
)
def test_every_element_needs_a_check_and_every_id_is_used_once():
data = graph_data()
data["checks"] = [data["checks"][0]]
data["relationships"][0]["id"] = "E-1"
assert problems(data) == (
"target-architecture.json: id E-1 is used more than once",
"target-architecture.json: Element E-2 has no Check",
)
def test_a_check_without_a_statement_is_named():
data = graph_data()
data["checks"][1]["statement"] = " "
assert (
"target-architecture.json.checks[1]: needs a non-empty statement"
in problems(data)
)
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
with pytest.raises(GraphError):
parse_graph(text)
# --- the Requirements chain has no gap -------------------------------------------------
def test_the_sample_chain_is_complete():
assert gaps(graph_data()) == ()
def test_a_non_functional_requirement_with_no_check_is_a_gap():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
"Requirement NFR-2 has no Check",
)
def test_a_functional_requirement_is_carried_by_its_components_element():
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
def test_a_component_with_no_element_is_a_gap():
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
"Component C-2 has no Element",
)
def test_a_selected_product_that_no_element_uses_is_a_gap():
armour = [{"component": "C-1", "product": "Example Armor"}]
assert gaps(graph_data(), products=armour) == (
"Product Example Armor selected for C-1 appears on no Element",
)
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
recommendations = [*RECOMMENDATIONS, cmek]
assert gaps(graph_data(), recommendations=recommendations) == (
"Recommendation DR-2 has no Check",
)
excused = graph_data()
excused["not_applicable"] = [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
assert gaps(excused, recommendations=recommendations) == ()
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
def test_a_check_must_come_from_a_requirement_or_recommendation():
data = graph_data()
data["checks"][0]["source"] = "my own idea"
assert gaps(data) == (
(
"Check K-1 has source my own idea, which is not a requirement or design "
"recommendation"
),
"Requirement NFR-1 has no Check",
)
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
built_on = upstream(
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
)
args = sample_submission(K.TARGET_ARCHITECTURE).args
assert check_submission(
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
) == ("Requirement NFR-2 has no Check",)
assert (
check_submission(
K.TARGET_ARCHITECTURE,
args["content"],
args["attachments"],
upstream(*DESIGN),
)
== ()
)
# --- the links before the graph --------------------------------------------------------
def test_requirements_need_ids_and_a_kind():
def missing(items) -> tuple[str, ...]:
args = sample_submission(K.REQUIREMENTS).args
return check_submission(
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
)
assert missing(REQUIREMENTS) == ()
assert missing([]) == ("requirements.json: lists no requirements",)
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
"requirements.json: id FR-1 is used more than once",
"requirements.json: FR-1 kind must be one of functional, non_functional",
)
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
"requirements.json[0]: needs text",
)
def test_every_functional_requirement_is_served_by_a_component():
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
def missing(components) -> tuple[str, ...]:
return check_submission(
K.TECHNICAL_DECOMPOSITION,
args["content"],
{"components.json": json.dumps(components)},
upstream(K.REQUIREMENTS),
)
assert missing(COMPONENTS) == ()
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
"components.json: C-1 serves NFR-9, which is not a requirement",
"components.json: functional requirement FR-1 is served by no component",
)
def test_every_component_gets_a_product_and_recommendations_name_real_components():
selection = sample_submission(K.PRODUCT_SELECTION).args
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert check_submission(
K.PRODUCT_SELECTION,
selection["content"],
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
built_on,
) == (
"products.json: C-7 is not a component",
"products.json: component C-1 has no product selected",
)
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
assert check_submission(
K.DESIGN_RECOMMENDATIONS,
advice["content"],
{"recommendations.json": json.dumps(wandering)},
built_on,
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
# --- code and validation answer the graph ----------------------------------------------
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
def test_deployment_guidance_must_account_for_every_check():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
partial = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": partial},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json: K-7 is not a Check in the Target architecture",
"implementation.json: Check K-2 is not accounted for",
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
)
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{
"check": "K-2",
"unimplementable": "example_service_account has no such argument",
},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == ()
assert unimplementable_checks(implementation) == [
{"check": "K-2", "reason": "example_service_account has no such argument"}
]
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
def test_an_unimplementable_entry_needs_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-2", "unimplementable": ""},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json[0]: needs unimplementable",
"implementation.json: Check K-2 is not accounted for",
)
def verdicts(**changes) -> str:
data = json.loads(SAMPLE_DATA["verdicts.json"])
data.update(changes)
return json.dumps(data)
def validation_missing(text: str) -> tuple[str, ...]:
args = sample_submission(K.VALIDATION_RESULTS).args
return check_submission(
K.VALIDATION_RESULTS,
args["content"],
{"verdicts.json": text},
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
)
def test_validation_gives_exactly_one_verdict_per_check():
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
assert validation_missing(verdicts(checks=[one, one, invented])) == (
"verdicts.json.checks: check K-1 is used more than once",
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
"""What it notices beyond the Checks is a proposed addition, never a failure."""
addition = {
"statement": "Cloud Armor rate limiting is configured.",
"applies_to": "E-1",
"reason": "NFR-1",
}
text = verdicts(proposed_additions=[addition])
assert validation_missing(text) == ()
outcome = validation_outcome(text)
assert outcome["failed"] == []
assert outcome["proposed_additions"] == [addition]
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
vague = [
{"check": "K-1", "verdict": "probably", "evidence": "x"},
{"check": "K-2", "verdict": "pass", "evidence": " "},
]
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
"verdicts.json.checks[1]: needs evidence",
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
checks = [
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
]
outcome = validation_outcome(verdicts(checks=checks))
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
assert outcome["not_verifiable"] == [
{"check": "K-2", "evidence": "No such argument."}
]
assert outcome["passed"] == []
assert outcome["dry_run"]["verdict"] == "pass"
# --- the diagram is a view of the graph ------------------------------------------------
async def test_the_diagram_shows_every_element_and_relationship_once():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
diagram = render_mermaid(graph)
assert diagram == render_mermaid(graph) # deterministic
assert diagram.count("Ingress service") == 1
assert diagram.count("Ingress identity") == 1
assert diagram.count("-->") == 1
assert "runs as" in diagram
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
async def test_names_with_characters_mermaid_reserves_still_render():
data = graph_data()
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
data["elements"][0]["id"] = "svc/ingest.v2"
data["relationships"][0]["from"] = "svc/ingest.v2"
data["checks"][0]["applies_to"] = "svc/ingest.v2"
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
diagram = render_mermaid(parse_graph(json.dumps(data)))
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
assert "svc/ingest.v2" not in diagram
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
submit = specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=upstream(*DESIGN),
)[0]
args = sample_submission(K.TARGET_ARCHITECTURE).args
result = await submit.handler(**args)
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
parse_graph(SAMPLE_DATA["target-architecture.json"])
)
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
built_on = upstream(
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
)
submit = specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=built_on,
)[0]
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
assert result.outcome is None
assert result.message == (
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
)
# --- through the engine ----------------------------------------------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def to_design_approval() -> list[list[Call]]:
return [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
]
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
return engine, runtime
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
# Tries to start the code before asking: refused, then asks.
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
],
},
)
first = await engine.start("Ingest webhooks")
paused = await engine.resume(
first.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert paused.prompt["approval"]["phase"] == "solution_architecture"
assert paused.prompt["approval"]["target_architecture"] == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
refused = runtime.runs[-1].results[0]
assert refused[0] == "delegate" and refused[1].startswith("Refused")
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
failing = json.loads(SAMPLE_DATA["verdicts.json"])
failing["checks"][0] = {
"check": "K-1",
"verdict": "fail",
"evidence": "min instances is 1",
}
failed_validation = sample_submission(K.VALIDATION_RESULTS)
failed_validation = Call(
failed_validation.tool,
{
**failed_validation.args,
"attachments": {"verdicts.json": json.dumps(failing)},
},
)
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
*to_design_approval(),
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{
"kind": "deployment_guidance",
"reason": "K-1 failed: min instances is 1",
},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
},
)
result = await engine.start("Ingest webhooks", pre_approved=True)
turns = [run for run in runtime.runs if run.role == "orchestrator"]
before, after = turns[6].task.context, turns[7].task.context
assert before["validation"] is None
assert after["validation"]["failed"] == [
{"check": "K-1", "evidence": "min instances is 1"}
]
assert after["validation"]["proposed_additions"] == []
# Once the code is reopened those verdicts are about work that no longer exists,
# so the next turn must not see them and reopen the same thing again.
assert turns[8].task.context["validation"] is None
# Revising the code leaves the approved design standing.
phases = result.state["ledger"]["phases"]
assert phases["solution_architecture"] == "approved"
revised = [
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
# --- constraints list their conditions --------------------------------------------------
def test_a_non_functional_requirement_must_list_its_conditions():
requirements = [
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
{"id": "NFR-1", "kind": "non_functional", "text": "Encrypted and retained."},
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Available.",
"conditions": [],
},
]
args = sample_submission(K.REQUIREMENTS).args
missing = check_submission(
K.REQUIREMENTS,
args["content"],
{"requirements.json": json.dumps(requirements)},
)
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"requirements.json: NFR-1 {needs}",
f"requirements.json: NFR-2 {needs}",
]
def test_a_recommendation_must_list_its_conditions():
recommendations = json.loads(SAMPLE_DATA["recommendations.json"])
del recommendations[0]["conditions"]
args = sample_submission(K.DESIGN_RECOMMENDATIONS).args
missing = check_submission(
K.DESIGN_RECOMMENDATIONS,
args["content"],
{"recommendations.json": json.dumps(recommendations)},
)
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"recommendations.json: DR-1 {needs}"
]
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
"""An outcome measured in operation is recorded, and no Check is asked for it."""
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Pages load within one second at p95.",
"conditions": [],
"outside_code": ["Measured p95 page load is under one second."],
}
)
args = sample_submission(K.REQUIREMENTS).args
accepted = check_submission(
K.REQUIREMENTS,
args["content"],
{"requirements.json": json.dumps(requirements)},
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert accepted == ()
assert gaps == ()
def test_a_constraint_with_conditions_still_needs_a_check():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Two zones, 99.9%.",
"conditions": ["The service runs in two zones."],
"outside_code": ["Measured availability is 99.9%."],
}
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert gaps == ("Requirement NFR-2 has no Check",)
# --- what the customer stated reaches the requirements (ADR-0008) ------------------------
def scoped_upstream(**scope) -> dict:
import json
return {"scope": {"attachments": {"scope.json": json.dumps(scope)}}}
def requirement_problems(requirements: list[dict], **scope) -> list[str]:
import json
from tests.reference_workflow.chain import check_requirements
return check_requirements(
{"requirements.json": json.dumps(requirements)}, scoped_upstream(**scope)
)
STATED = {
"stated": {
"intent": {
"requirements": [{"id": "REQ-01", "statement": "Ready in 10 weeks."}]
},
"networking": {"topology": "hub_and_spoke"},
},
"binding": ["networking.topology"],
}
FUNCTIONAL = {"id": "FR-1", "kind": "functional", "text": "Accept events."}
CARRIED = {
"id": "NFR-1",
"kind": "non_functional",
"text": "The network is hub and spoke, ready in ten weeks.",
"conditions": ["A hub network is connected to each spoke network."],
"from": ["REQ-01", "networking.topology"],
}
def test_requirements_that_carry_everything_the_customer_stated_are_accepted():
assert requirement_problems([FUNCTIONAL, CARRIED], **STATED) == []
def test_a_customers_own_requirement_must_be_covered_by_name():
uncited = {**CARRIED, "from": ["networking.topology"]}
(problem,) = requirement_problems([FUNCTIONAL, uncited], **STATED)
assert "the customer's requirement REQ-01 is covered by no requirement" in problem
def test_a_stated_choice_must_be_carried_by_a_requirement():
"""Case 08: a stated topology was named in the design and not built, and every
Check passed, because nothing asked for it."""
uncarried = {**CARRIED, "from": ["REQ-01"]}
(problem,) = requirement_problems([FUNCTIONAL, uncarried], **STATED)
assert "the customer stated networking.topology and no requirement carries it" in (
problem
)
def test_a_stated_choice_is_carried_as_a_constraint_the_design_is_checked_against():
as_functional = {**CARRIED, "kind": "functional", "conditions": []}
outside_code_only = {
**CARRIED,
"conditions": [],
"outside_code": ["The topology is reviewed by the network team."],
}
for weak in (as_functional, outside_code_only):
problems = requirement_problems([FUNCTIONAL, weak], **STATED)
assert any(
"must be non_functional with at least one entry in `conditions`" in p
for p in problems
), weak
def test_with_no_scope_or_nothing_binding_requirements_are_as_before():
from tests.reference_workflow.chain import check_requirements
plain = [FUNCTIONAL]
import json
assert check_requirements({"requirements.json": json.dumps(plain)}, None) == []
assert requirement_problems(plain, stated={"intent": {"statement": "x"}}) == []
def test_what_was_assumed_for_the_customer_does_not_bind_the_design():
from execution_engine.scope import Catalogue, FieldSpec, Scope
catalogue = Catalogue(
1,
("workload",),
{
"net.topology": FieldSpec("net.topology", binds=True),
"net.region": FieldSpec("net.region", binds=True),
"notes": FieldSpec("notes"),
},
)
scope = Scope(
catalogue,
"workload",
values={"net.topology": "hub", "net.region": "x", "notes": "n"},
sources={"net.region": "assumed"},
)
assert scope.for_context()["binding"] == ["net.topology"]