889 lines
29 KiB
Python
889 lines
29 KiB
Python
"""The Requirements chain: the Target architecture graph and the data files around it.
|
|
|
|
ADR-0005: the outcome is understood before code is written, and validation answers
|
|
the graph's Checks and nothing else.
|
|
"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.runtime import RoleSpec
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from execution_engine.tools.execution import specialist_tools
|
|
from tests.reference_workflow.chain import unimplementable_checks, validation_outcome
|
|
from tests.reference_workflow.local import mermaid_check
|
|
from tests.reference_workflow.target_architecture import (
|
|
GraphError,
|
|
chain_gaps,
|
|
parse_graph,
|
|
render_mermaid,
|
|
)
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
SAMPLE_DATA,
|
|
WORKFLOW,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
check_submission,
|
|
sample_submission,
|
|
)
|
|
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
|
|
|
K = DeliverableKind
|
|
P = PhaseId
|
|
|
|
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
|
|
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
|
|
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
|
|
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
|
|
|
|
|
|
def graph_data() -> dict:
|
|
return json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
|
|
def problems(data: dict) -> tuple[str, ...]:
|
|
with pytest.raises(GraphError) as raised:
|
|
parse_graph(json.dumps(data))
|
|
return raised.value.problems
|
|
|
|
|
|
def gaps(data: dict, **overrides) -> tuple[str, ...]:
|
|
links = {
|
|
"requirements": REQUIREMENTS,
|
|
"components": COMPONENTS,
|
|
"products": PRODUCTS,
|
|
"recommendations": RECOMMENDATIONS,
|
|
**overrides,
|
|
}
|
|
return chain_gaps(parse_graph(json.dumps(data)), **links)
|
|
|
|
|
|
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
|
|
"""The Deliverables a submission builds on, from the coherent sample."""
|
|
built = {}
|
|
for kind in kinds:
|
|
args = sample_submission(kind).args
|
|
attachments = {
|
|
name: replaced.get(name, text) for name, text in args["attachments"].items()
|
|
}
|
|
built[kind.value] = {"content": args["content"], "attachments": attachments}
|
|
return built
|
|
|
|
|
|
DESIGN = (
|
|
K.REQUIREMENTS,
|
|
K.TECHNICAL_DECOMPOSITION,
|
|
K.PRODUCT_SELECTION,
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
)
|
|
|
|
|
|
# --- the graph is well-formed ----------------------------------------------------------
|
|
|
|
|
|
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
|
|
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
|
|
assert graph.elements[1].component is None # a supporting Element
|
|
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
|
|
assert graph.check_ids == ("K-1", "K-2")
|
|
assert graph.summary() == {
|
|
"elements": 2,
|
|
"relationships": 1,
|
|
"checks": 2,
|
|
"not_applicable": [],
|
|
}
|
|
|
|
|
|
def test_a_relationship_must_join_two_elements():
|
|
data = graph_data()
|
|
data["relationships"][0]["to"] = "E-9"
|
|
|
|
assert problems(data) == (
|
|
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
|
|
)
|
|
|
|
|
|
def test_a_check_must_apply_to_something_in_the_graph():
|
|
data = graph_data()
|
|
data["checks"][0]["applies_to"] = "E-9"
|
|
|
|
assert problems(data) == (
|
|
(
|
|
"target-architecture.json: Check K-1 applies to E-9, "
|
|
"which is not an Element or Relationship"
|
|
),
|
|
"target-architecture.json: Element E-1 has no Check",
|
|
)
|
|
|
|
|
|
def test_every_element_needs_a_check_and_every_id_is_used_once():
|
|
data = graph_data()
|
|
data["checks"] = [data["checks"][0]]
|
|
data["relationships"][0]["id"] = "E-1"
|
|
|
|
assert problems(data) == (
|
|
"target-architecture.json: id E-1 is used more than once",
|
|
"target-architecture.json: Element E-2 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_check_without_a_statement_is_named():
|
|
data = graph_data()
|
|
data["checks"][1]["statement"] = " "
|
|
|
|
assert (
|
|
"target-architecture.json.checks[1]: needs a non-empty statement"
|
|
in problems(data)
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
|
|
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
|
|
with pytest.raises(GraphError):
|
|
parse_graph(text)
|
|
|
|
|
|
# --- the Requirements chain has no gap -------------------------------------------------
|
|
|
|
|
|
def test_the_sample_chain_is_complete():
|
|
assert gaps(graph_data()) == ()
|
|
|
|
|
|
def test_a_non_functional_requirement_with_no_check_is_a_gap():
|
|
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
|
|
|
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
|
|
"Requirement NFR-2 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_functional_requirement_is_carried_by_its_components_element():
|
|
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
|
|
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
|
|
|
|
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
|
|
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
|
|
|
|
|
|
def test_a_component_with_no_element_is_a_gap():
|
|
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
|
|
|
|
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
|
|
"Component C-2 has no Element",
|
|
)
|
|
|
|
|
|
def test_a_selected_product_that_no_element_uses_is_a_gap():
|
|
armour = [{"component": "C-1", "product": "Example Armor"}]
|
|
|
|
assert gaps(graph_data(), products=armour) == (
|
|
"Product Example Armor selected for C-1 appears on no Element",
|
|
)
|
|
|
|
|
|
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
|
|
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
|
recommendations = [*RECOMMENDATIONS, cmek]
|
|
|
|
assert gaps(graph_data(), recommendations=recommendations) == (
|
|
"Recommendation DR-2 has no Check",
|
|
)
|
|
|
|
excused = graph_data()
|
|
excused["not_applicable"] = [
|
|
{"recommendation": "DR-2", "reason": "No regulated data."}
|
|
]
|
|
assert gaps(excused, recommendations=recommendations) == ()
|
|
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
|
|
{"recommendation": "DR-2", "reason": "No regulated data."}
|
|
]
|
|
|
|
|
|
def test_a_check_must_come_from_a_requirement_or_recommendation():
|
|
data = graph_data()
|
|
data["checks"][0]["source"] = "my own idea"
|
|
|
|
assert gaps(data) == (
|
|
(
|
|
"Check K-1 has source my own idea, which is not a requirement or design "
|
|
"recommendation"
|
|
),
|
|
"Requirement NFR-1 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
|
|
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
|
built_on = upstream(
|
|
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
|
|
)
|
|
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
|
|
|
assert check_submission(
|
|
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
|
|
) == ("Requirement NFR-2 has no Check",)
|
|
assert (
|
|
check_submission(
|
|
K.TARGET_ARCHITECTURE,
|
|
args["content"],
|
|
args["attachments"],
|
|
upstream(*DESIGN),
|
|
)
|
|
== ()
|
|
)
|
|
|
|
|
|
# --- the links before the graph --------------------------------------------------------
|
|
|
|
|
|
def test_requirements_need_ids_and_a_kind():
|
|
def missing(items) -> tuple[str, ...]:
|
|
args = sample_submission(K.REQUIREMENTS).args
|
|
return check_submission(
|
|
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
|
|
)
|
|
|
|
assert missing(REQUIREMENTS) == ()
|
|
assert missing([]) == ("requirements.json: lists no requirements",)
|
|
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
|
|
"requirements.json: id FR-1 is used more than once",
|
|
"requirements.json: FR-1 kind must be one of functional, non_functional",
|
|
)
|
|
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
|
|
"requirements.json[0]: needs text",
|
|
)
|
|
|
|
|
|
def test_every_functional_requirement_is_served_by_a_component():
|
|
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
|
|
|
|
def missing(components) -> tuple[str, ...]:
|
|
return check_submission(
|
|
K.TECHNICAL_DECOMPOSITION,
|
|
args["content"],
|
|
{"components.json": json.dumps(components)},
|
|
upstream(K.REQUIREMENTS),
|
|
)
|
|
|
|
assert missing(COMPONENTS) == ()
|
|
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
|
|
"components.json: C-1 serves NFR-9, which is not a requirement",
|
|
"components.json: functional requirement FR-1 is served by no component",
|
|
)
|
|
|
|
|
|
def test_every_component_gets_a_product_and_recommendations_name_real_components():
|
|
selection = sample_submission(K.PRODUCT_SELECTION).args
|
|
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
|
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
|
|
|
assert check_submission(
|
|
K.PRODUCT_SELECTION,
|
|
selection["content"],
|
|
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
|
|
built_on,
|
|
) == (
|
|
"products.json: C-7 is not a component",
|
|
"products.json: component C-1 has no product selected",
|
|
)
|
|
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
|
|
assert check_submission(
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
advice["content"],
|
|
{"recommendations.json": json.dumps(wandering)},
|
|
built_on,
|
|
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
|
|
|
|
|
|
# --- code and validation answer the graph ----------------------------------------------
|
|
|
|
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
|
|
|
|
|
|
def test_deployment_guidance_must_account_for_every_check():
|
|
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
|
partial = json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
|
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
|
|
]
|
|
)
|
|
|
|
missing = check_submission(
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
args["content"],
|
|
{**args["attachments"], "implementation.json": partial},
|
|
upstream(*EVERYTHING_BEFORE_CODE),
|
|
)
|
|
|
|
assert missing == (
|
|
"implementation.json: K-7 is not a Check in the Target architecture",
|
|
"implementation.json: Check K-2 is not accounted for",
|
|
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
|
|
)
|
|
|
|
|
|
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
|
|
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
|
implementation = json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
|
{
|
|
"check": "K-2",
|
|
"unimplementable": "example_service_account has no such argument",
|
|
},
|
|
]
|
|
)
|
|
|
|
missing = check_submission(
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
args["content"],
|
|
{**args["attachments"], "implementation.json": implementation},
|
|
upstream(*EVERYTHING_BEFORE_CODE),
|
|
)
|
|
|
|
assert missing == ()
|
|
assert unimplementable_checks(implementation) == [
|
|
{"check": "K-2", "reason": "example_service_account has no such argument"}
|
|
]
|
|
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
|
|
|
|
|
|
def test_an_unimplementable_entry_needs_its_reason():
|
|
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
|
implementation = json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
|
{"check": "K-2", "unimplementable": ""},
|
|
]
|
|
)
|
|
|
|
missing = check_submission(
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
args["content"],
|
|
{**args["attachments"], "implementation.json": implementation},
|
|
upstream(*EVERYTHING_BEFORE_CODE),
|
|
)
|
|
|
|
assert missing == (
|
|
"implementation.json[0]: needs unimplementable",
|
|
"implementation.json: Check K-2 is not accounted for",
|
|
)
|
|
|
|
|
|
def verdicts(**changes) -> str:
|
|
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
data.update(changes)
|
|
return json.dumps(data)
|
|
|
|
|
|
def validation_missing(text: str) -> tuple[str, ...]:
|
|
args = sample_submission(K.VALIDATION_RESULTS).args
|
|
return check_submission(
|
|
K.VALIDATION_RESULTS,
|
|
args["content"],
|
|
{"verdicts.json": text},
|
|
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
|
|
)
|
|
|
|
|
|
def test_validation_gives_exactly_one_verdict_per_check():
|
|
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
|
|
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
|
|
|
|
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
|
|
assert validation_missing(verdicts(checks=[one, one, invented])) == (
|
|
"verdicts.json.checks: check K-1 is used more than once",
|
|
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
|
|
"verdicts.json.checks: Check K-2 is not accounted for",
|
|
)
|
|
|
|
|
|
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
|
|
"""What it notices beyond the Checks is a proposed addition, never a failure."""
|
|
addition = {
|
|
"statement": "Cloud Armor rate limiting is configured.",
|
|
"applies_to": "E-1",
|
|
"reason": "NFR-1",
|
|
}
|
|
text = verdicts(proposed_additions=[addition])
|
|
|
|
assert validation_missing(text) == ()
|
|
outcome = validation_outcome(text)
|
|
assert outcome["failed"] == []
|
|
assert outcome["proposed_additions"] == [addition]
|
|
|
|
|
|
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
|
|
vague = [
|
|
{"check": "K-1", "verdict": "probably", "evidence": "x"},
|
|
{"check": "K-2", "verdict": "pass", "evidence": " "},
|
|
]
|
|
|
|
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
|
|
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
|
|
"verdicts.json.checks[1]: needs evidence",
|
|
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
|
|
"verdicts.json.checks: Check K-2 is not accounted for",
|
|
)
|
|
|
|
|
|
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
|
|
checks = [
|
|
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
|
|
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
|
|
]
|
|
|
|
outcome = validation_outcome(verdicts(checks=checks))
|
|
|
|
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
|
|
assert outcome["not_verifiable"] == [
|
|
{"check": "K-2", "evidence": "No such argument."}
|
|
]
|
|
assert outcome["passed"] == []
|
|
assert outcome["dry_run"]["verdict"] == "pass"
|
|
|
|
|
|
# --- the diagram is a view of the graph ------------------------------------------------
|
|
|
|
|
|
async def test_the_diagram_shows_every_element_and_relationship_once():
|
|
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
diagram = render_mermaid(graph)
|
|
|
|
assert diagram == render_mermaid(graph) # deterministic
|
|
assert diagram.count("Ingress service") == 1
|
|
assert diagram.count("Ingress identity") == 1
|
|
assert diagram.count("-->") == 1
|
|
assert "runs as" in diagram
|
|
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
|
|
|
|
|
async def test_names_with_characters_mermaid_reserves_still_render():
|
|
data = graph_data()
|
|
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
|
|
data["elements"][0]["id"] = "svc/ingest.v2"
|
|
data["relationships"][0]["from"] = "svc/ingest.v2"
|
|
data["checks"][0]["applies_to"] = "svc/ingest.v2"
|
|
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
|
|
|
|
diagram = render_mermaid(parse_graph(json.dumps(data)))
|
|
|
|
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
|
assert "svc/ingest.v2" not in diagram
|
|
|
|
|
|
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
|
|
submit = specialist_tools(
|
|
WORKFLOW,
|
|
K.TARGET_ARCHITECTURE,
|
|
extra=[],
|
|
allow_cloud_read=False,
|
|
upstream=upstream(*DESIGN),
|
|
)[0]
|
|
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
|
|
|
result = await submit.handler(**args)
|
|
|
|
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
|
|
parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
)
|
|
|
|
|
|
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
|
|
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
|
built_on = upstream(
|
|
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
|
|
)
|
|
submit = specialist_tools(
|
|
WORKFLOW,
|
|
K.TARGET_ARCHITECTURE,
|
|
extra=[],
|
|
allow_cloud_read=False,
|
|
upstream=built_on,
|
|
)[0]
|
|
|
|
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
|
|
|
|
assert result.outcome is None
|
|
assert result.message == (
|
|
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
|
|
)
|
|
|
|
|
|
# --- through the engine ----------------------------------------------------------------
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def delegate(*kinds: DeliverableKind) -> Call:
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
|
|
def approve(phase: PhaseId) -> Call:
|
|
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def to_design_approval() -> list[list[Call]]:
|
|
return [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
]
|
|
|
|
|
|
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
)
|
|
return engine, runtime
|
|
|
|
|
|
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
|
|
once = {
|
|
specialist(kind): [[sample_submission(kind)]]
|
|
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
|
|
}
|
|
engine, runtime = engine_for(
|
|
tmp_path,
|
|
{
|
|
**once,
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY)],
|
|
[delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
# Tries to start the code before asking: refused, then asks.
|
|
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
|
|
],
|
|
},
|
|
)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
paused = await engine.resume(
|
|
first.execution_id,
|
|
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
|
)
|
|
|
|
assert paused.prompt["approval"]["phase"] == "solution_architecture"
|
|
assert paused.prompt["approval"]["target_architecture"] == {
|
|
"elements": 2,
|
|
"relationships": 1,
|
|
"checks": 2,
|
|
"not_applicable": [],
|
|
}
|
|
refused = runtime.runs[-1].results[0]
|
|
assert refused[0] == "delegate" and refused[1].startswith("Refused")
|
|
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
|
|
|
|
|
|
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
|
|
failing = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
failing["checks"][0] = {
|
|
"check": "K-1",
|
|
"verdict": "fail",
|
|
"evidence": "min instances is 1",
|
|
}
|
|
failed_validation = sample_submission(K.VALIDATION_RESULTS)
|
|
failed_validation = Call(
|
|
failed_validation.tool,
|
|
{
|
|
**failed_validation.args,
|
|
"attachments": {"verdicts.json": json.dumps(failing)},
|
|
},
|
|
)
|
|
once = {
|
|
specialist(kind): [[sample_submission(kind)]]
|
|
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
|
|
}
|
|
engine, runtime = engine_for(
|
|
tmp_path,
|
|
{
|
|
**once,
|
|
"orchestrator": [
|
|
*to_design_approval(),
|
|
[
|
|
approve(P.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[
|
|
Call(
|
|
"reopen",
|
|
{
|
|
"kind": "deployment_guidance",
|
|
"reason": "K-1 failed: min instances is 1",
|
|
},
|
|
),
|
|
delegate(K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[],
|
|
],
|
|
specialist(K.DEPLOYMENT_GUIDANCE): [
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
],
|
|
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
|
|
},
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
turns = [run for run in runtime.runs if run.role == "orchestrator"]
|
|
before, after = turns[6].task.context, turns[7].task.context
|
|
assert before["validation"] is None
|
|
assert after["validation"]["failed"] == [
|
|
{"check": "K-1", "evidence": "min instances is 1"}
|
|
]
|
|
assert after["validation"]["proposed_additions"] == []
|
|
# Once the code is reopened those verdicts are about work that no longer exists,
|
|
# so the next turn must not see them and reopen the same thing again.
|
|
assert turns[8].task.context["validation"] is None
|
|
# Revising the code leaves the approved design standing.
|
|
phases = result.state["ledger"]["phases"]
|
|
assert phases["solution_architecture"] == "approved"
|
|
revised = [
|
|
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
|
][1]
|
|
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
|
|
|
|
|
|
# --- constraints list their conditions --------------------------------------------------
|
|
|
|
|
|
def test_a_non_functional_requirement_must_list_its_conditions():
|
|
requirements = [
|
|
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
|
|
{"id": "NFR-1", "kind": "non_functional", "text": "Encrypted and retained."},
|
|
{
|
|
"id": "NFR-2",
|
|
"kind": "non_functional",
|
|
"text": "Available.",
|
|
"conditions": [],
|
|
},
|
|
]
|
|
args = sample_submission(K.REQUIREMENTS).args
|
|
|
|
missing = check_submission(
|
|
K.REQUIREMENTS,
|
|
args["content"],
|
|
{"requirements.json": json.dumps(requirements)},
|
|
)
|
|
|
|
needs = "needs `conditions`:"
|
|
assert [line.split(" a list")[0] for line in missing] == [
|
|
f"requirements.json: NFR-1 {needs}",
|
|
f"requirements.json: NFR-2 {needs}",
|
|
]
|
|
|
|
|
|
def test_a_recommendation_must_list_its_conditions():
|
|
recommendations = json.loads(SAMPLE_DATA["recommendations.json"])
|
|
del recommendations[0]["conditions"]
|
|
args = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
|
|
|
missing = check_submission(
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
args["content"],
|
|
{"recommendations.json": json.dumps(recommendations)},
|
|
)
|
|
|
|
needs = "needs `conditions`:"
|
|
assert [line.split(" a list")[0] for line in missing] == [
|
|
f"recommendations.json: DR-1 {needs}"
|
|
]
|
|
|
|
|
|
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
|
|
"""An outcome measured in operation is recorded, and no Check is asked for it."""
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
requirements.append(
|
|
{
|
|
"id": "NFR-2",
|
|
"kind": "non_functional",
|
|
"text": "Pages load within one second at p95.",
|
|
"conditions": [],
|
|
"outside_code": ["Measured p95 page load is under one second."],
|
|
}
|
|
)
|
|
args = sample_submission(K.REQUIREMENTS).args
|
|
|
|
accepted = check_submission(
|
|
K.REQUIREMENTS,
|
|
args["content"],
|
|
{"requirements.json": json.dumps(requirements)},
|
|
)
|
|
gaps = chain_gaps(
|
|
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
|
requirements=requirements,
|
|
components=json.loads(SAMPLE_DATA["components.json"]),
|
|
products=json.loads(SAMPLE_DATA["products.json"]),
|
|
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
|
)
|
|
|
|
assert accepted == ()
|
|
assert gaps == ()
|
|
|
|
|
|
def test_a_constraint_with_conditions_still_needs_a_check():
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
requirements.append(
|
|
{
|
|
"id": "NFR-2",
|
|
"kind": "non_functional",
|
|
"text": "Two zones, 99.9%.",
|
|
"conditions": ["The service runs in two zones."],
|
|
"outside_code": ["Measured availability is 99.9%."],
|
|
}
|
|
)
|
|
|
|
gaps = chain_gaps(
|
|
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
|
requirements=requirements,
|
|
components=json.loads(SAMPLE_DATA["components.json"]),
|
|
products=json.loads(SAMPLE_DATA["products.json"]),
|
|
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
|
)
|
|
|
|
assert gaps == ("Requirement NFR-2 has no Check",)
|
|
|
|
|
|
# --- what the customer stated reaches the requirements (ADR-0008) ------------------------
|
|
|
|
|
|
def scoped_upstream(**scope) -> dict:
|
|
import json
|
|
|
|
return {"scope": {"attachments": {"scope.json": json.dumps(scope)}}}
|
|
|
|
|
|
def requirement_problems(requirements: list[dict], **scope) -> list[str]:
|
|
import json
|
|
|
|
from tests.reference_workflow.chain import check_requirements
|
|
|
|
return check_requirements(
|
|
{"requirements.json": json.dumps(requirements)}, scoped_upstream(**scope)
|
|
)
|
|
|
|
|
|
STATED = {
|
|
"stated": {
|
|
"intent": {
|
|
"requirements": [{"id": "REQ-01", "statement": "Ready in 10 weeks."}]
|
|
},
|
|
"networking": {"topology": "hub_and_spoke"},
|
|
},
|
|
"binding": ["networking.topology"],
|
|
}
|
|
FUNCTIONAL = {"id": "FR-1", "kind": "functional", "text": "Accept events."}
|
|
CARRIED = {
|
|
"id": "NFR-1",
|
|
"kind": "non_functional",
|
|
"text": "The network is hub and spoke, ready in ten weeks.",
|
|
"conditions": ["A hub network is connected to each spoke network."],
|
|
"from": ["REQ-01", "networking.topology"],
|
|
}
|
|
|
|
|
|
def test_requirements_that_carry_everything_the_customer_stated_are_accepted():
|
|
assert requirement_problems([FUNCTIONAL, CARRIED], **STATED) == []
|
|
|
|
|
|
def test_a_customers_own_requirement_must_be_covered_by_name():
|
|
uncited = {**CARRIED, "from": ["networking.topology"]}
|
|
|
|
(problem,) = requirement_problems([FUNCTIONAL, uncited], **STATED)
|
|
|
|
assert "the customer's requirement REQ-01 is covered by no requirement" in problem
|
|
|
|
|
|
def test_a_stated_choice_must_be_carried_by_a_requirement():
|
|
"""Case 08: a stated topology was named in the design and not built, and every
|
|
Check passed, because nothing asked for it."""
|
|
uncarried = {**CARRIED, "from": ["REQ-01"]}
|
|
|
|
(problem,) = requirement_problems([FUNCTIONAL, uncarried], **STATED)
|
|
|
|
assert "the customer stated networking.topology and no requirement carries it" in (
|
|
problem
|
|
)
|
|
|
|
|
|
def test_a_stated_choice_is_carried_as_a_constraint_the_design_is_checked_against():
|
|
as_functional = {**CARRIED, "kind": "functional", "conditions": []}
|
|
outside_code_only = {
|
|
**CARRIED,
|
|
"conditions": [],
|
|
"outside_code": ["The topology is reviewed by the network team."],
|
|
}
|
|
|
|
for weak in (as_functional, outside_code_only):
|
|
problems = requirement_problems([FUNCTIONAL, weak], **STATED)
|
|
assert any(
|
|
"must be non_functional with at least one entry in `conditions`" in p
|
|
for p in problems
|
|
), weak
|
|
|
|
|
|
def test_with_no_scope_or_nothing_binding_requirements_are_as_before():
|
|
from tests.reference_workflow.chain import check_requirements
|
|
|
|
plain = [FUNCTIONAL]
|
|
import json
|
|
|
|
assert check_requirements({"requirements.json": json.dumps(plain)}, None) == []
|
|
assert requirement_problems(plain, stated={"intent": {"statement": "x"}}) == []
|
|
|
|
|
|
def test_what_was_assumed_for_the_customer_does_not_bind_the_design():
|
|
from execution_engine.scope import Catalogue, FieldSpec, Scope
|
|
|
|
catalogue = Catalogue(
|
|
1,
|
|
("workload",),
|
|
{
|
|
"net.topology": FieldSpec("net.topology", binds=True),
|
|
"net.region": FieldSpec("net.region", binds=True),
|
|
"notes": FieldSpec("notes"),
|
|
},
|
|
)
|
|
scope = Scope(
|
|
catalogue,
|
|
"workload",
|
|
values={"net.topology": "hub", "net.region": "x", "notes": "n"},
|
|
sources={"net.region": "assumed"},
|
|
)
|
|
|
|
assert scope.for_context()["binding"] == ["net.topology"]
|