The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
254 lines
8.4 KiB
Python
254 lines
8.4 KiB
Python
"""The Target operating model graph: what is refused, and the views code writes."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from app.operating_model import graph as model
|
|
from app.operating_model.names import (
|
|
FINDINGS_FILE,
|
|
GRAPH_FILE,
|
|
ORGANISATION_FILE,
|
|
STAFFING_FILE,
|
|
)
|
|
from tests.support import CARD, CATALOGUE, GRAPH, as_file, graph, upstream
|
|
|
|
check = model.check(CARD)
|
|
add_views = model.views(CATALOGUE, CARD)
|
|
FINDING = {
|
|
"id": "AF-1",
|
|
"about": ["S-2"],
|
|
"needed_by": ["C-3"],
|
|
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
|
|
"proposed": "Add an alert on a change to the retention lock.",
|
|
}
|
|
|
|
|
|
def faults(changed, findings=None) -> list[str]:
|
|
attachments = as_file(GRAPH_FILE, changed)
|
|
if findings is not None:
|
|
attachments[FINDINGS_FILE] = json.dumps(findings)
|
|
return check(attachments, upstream())
|
|
|
|
|
|
def test_a_graph_that_holds_together_is_accepted():
|
|
assert faults(GRAPH) == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("change", "problem"),
|
|
[
|
|
(
|
|
lambda g: g["parties"][0].update(kind="friend"),
|
|
"Party P-1: kind must be one of",
|
|
),
|
|
(
|
|
lambda g: g["roles"][0].update(party="P-9"),
|
|
"Role R-1 is in P-9, which is not a Party",
|
|
),
|
|
(
|
|
lambda g: g["roles"][1].update(country="FR"),
|
|
"Role R-2 sits in FR, which the rate card does not have",
|
|
),
|
|
(
|
|
lambda g: g["roles"][1].update(country="US"),
|
|
"Role R-2 sits in US; the customer allows only GB, PL, IN",
|
|
),
|
|
(
|
|
lambda g: g["roles"][1].update(level="wizard"),
|
|
"Role R-2 is at level wizard, which the rate card does not have",
|
|
),
|
|
(
|
|
lambda g: g["roles"][1].update(hours="nights"),
|
|
"Role R-2: hours must be one of",
|
|
),
|
|
(lambda g: g["roles"][1].pop("country"), "(R-2) needs country"),
|
|
(
|
|
lambda g: g["responsibilities"][0].update(kind="owner"),
|
|
"Responsibility RS-1: kind must be one of",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"][0].update(role="R-9"),
|
|
"RS-1 names R-9, which is not a Role",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"][0].update(capability="C-9"),
|
|
"RS-1 names C-9, which is not a Capability",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"].pop(0),
|
|
"Capability C-1 has 0 accountable Roles; it needs exactly one",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"][2].update(kind="accountable"),
|
|
"Capability C-1 has 2 accountable Roles; it needs exactly one",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"].pop(1),
|
|
"Capability C-1 has no responsible Role",
|
|
),
|
|
(
|
|
lambda g: g["roles"][1].update(hours="business"),
|
|
"Capability C-1 is needed 24x7, and no Role responsible for it works those hours",
|
|
),
|
|
(lambda g: g["checks"].pop(1), "Condition K-1 is carried by no Check"),
|
|
(
|
|
lambda g: g["checks"].pop(0),
|
|
"Operational requirement OR-1 is verified by no Check",
|
|
),
|
|
(
|
|
lambda g: g["checks"][0].update(on="C-9"),
|
|
"Check OC-1 is on C-9, which is not",
|
|
),
|
|
(
|
|
lambda g: g["checks"][0].update(source="OR-9"),
|
|
"Check OC-1: source OR-9 is not an Operational requirement",
|
|
),
|
|
(
|
|
lambda g: g["checks"][0].update(depends_on_finding="AF-1"),
|
|
"Check OC-1 depends on AF-1, which is not a finding",
|
|
),
|
|
(
|
|
lambda g: g["roles"].append(dict(g["roles"][0])),
|
|
"id R-1 is used more than once in roles",
|
|
),
|
|
],
|
|
)
|
|
def test_a_graph_that_does_not_hold_together_is_refused(change, problem):
|
|
changed = graph()
|
|
change(changed)
|
|
|
|
found = faults(changed)
|
|
|
|
assert any(problem in fault for fault in found), found
|
|
|
|
|
|
def test_a_requirement_is_verified_through_the_conditions_it_comes_from():
|
|
# OR-3 comes from K-1 and K-2: the Checks carrying them verify it.
|
|
changed = graph()
|
|
assert not any(c["source"] == "OR-3" for c in changed["checks"])
|
|
|
|
assert not any("OR-3" in fault for fault in faults(changed))
|
|
|
|
|
|
def test_a_file_that_is_not_a_graph_is_named():
|
|
assert "not valid JSON" in check({GRAPH_FILE: "{"}, upstream())[0]
|
|
assert "must be an object with" in check({GRAPH_FILE: "[]"}, upstream())[0]
|
|
|
|
|
|
def test_a_finding_is_recorded_and_a_check_may_wait_on_it():
|
|
changed = graph()
|
|
changed["checks"][2]["depends_on_finding"] = "AF-1"
|
|
|
|
assert faults(changed, [FINDING]) == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("change", "problem"),
|
|
[
|
|
(lambda f: f.update(lacks=""), "AF-1 needs lacks"),
|
|
(lambda f: f.update(proposed=""), "AF-1 needs proposed"),
|
|
(lambda f: f.update(needed_by=[]), "AF-1 must say what needs it"),
|
|
(lambda f: f.update(about=["S-9"]), "AF-1 is about S-9, which is not an Item"),
|
|
(
|
|
lambda f: f.update(about=[]),
|
|
"AF-1 names no Item of the solution under about",
|
|
),
|
|
(
|
|
lambda f: f.update(needed_by=["C-9"]),
|
|
"AF-1 is needed by C-9, which is not in the operating model",
|
|
),
|
|
],
|
|
)
|
|
def test_a_finding_that_does_not_hold_together_is_refused(change, problem):
|
|
finding = dict(FINDING)
|
|
change(finding)
|
|
|
|
found = faults(GRAPH, [finding])
|
|
|
|
assert any(problem in fault for fault in found), found
|
|
|
|
|
|
def viewed(changed=GRAPH, findings=None, spec=None):
|
|
attachments = as_file(GRAPH_FILE, changed)
|
|
if findings is not None:
|
|
attachments[FINDINGS_FILE] = json.dumps(findings)
|
|
built = upstream() if spec is None else upstream(spec=spec)
|
|
return add_views(
|
|
"# Target operating model\n\n## Design rationale\n\nWhy.\n", attachments, built
|
|
)
|
|
|
|
|
|
def test_the_matrix_is_written_from_the_graph():
|
|
content, _ = viewed()
|
|
|
|
assert "## Responsibility matrix" in content
|
|
assert "| Capability | Platform owner | Platform SRE | Service manager |" in content
|
|
assert "| Incident management (C-1) | C | R | A |" in content
|
|
assert "| Change enablement (C-2) | A | R | |" in content
|
|
# One Role may be answerable for a Capability and do it: both parts are shown.
|
|
assert "| Regulatory evidence and reporting (C-3) | C | | A/R |" in content
|
|
|
|
|
|
def test_staffing_and_cost_are_written_with_the_prices_they_came_from():
|
|
content, attachments = viewed()
|
|
|
|
assert (
|
|
"| Platform SRE | Managed service provider | PL | senior | 24x7 | 100.0 h | 5.0 | 41,640 GBP |"
|
|
in content
|
|
)
|
|
assert (
|
|
"| Platform owner | Northwind platform team | GB | lead | business | 0.0 h | 0.0 | not priced |"
|
|
in content
|
|
)
|
|
assert "Cost a month: 43,410 GBP." in content
|
|
assert "The monthly budget is 120,000 GBP: this is within it." in content
|
|
assert "version `draft-2026-10-13`" in content
|
|
assert "Its figures are placeholders" in content
|
|
assert "Converted from USD at 0.79, a rate set on 2026-10-13." in content
|
|
assert json.loads(attachments[STAFFING_FILE])["cost_per_month"] == 43410
|
|
|
|
|
|
def test_a_cost_over_the_budget_is_shown_as_over_not_refused():
|
|
from tests.support import SPEC
|
|
|
|
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
|
|
|
|
content, _ = viewed(spec=tight)
|
|
|
|
assert "this is over it by 3,410 GBP." in content
|
|
assert check(as_file(GRAPH_FILE, GRAPH), upstream(spec=tight)) == []
|
|
|
|
|
|
def test_the_organisation_chart_is_drawn_from_the_graph():
|
|
_, attachments = viewed()
|
|
|
|
chart = attachments[ORGANISATION_FILE]
|
|
assert chart.startswith("flowchart TB\n")
|
|
assert 'subgraph P_2["Managed service provider (provider)"]' in chart
|
|
assert 'R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
|
|
|
|
|
|
def test_findings_are_listed_for_whoever_owns_the_design():
|
|
content, attachments = viewed(findings=[FINDING])
|
|
|
|
assert "## Findings" in content
|
|
assert "**AF-1** (S-2): Nothing alerts when" in content
|
|
assert json.loads(attachments[FINDINGS_FILE]) == [FINDING]
|
|
|
|
|
|
def test_with_no_findings_the_file_is_empty_and_no_section_is_written():
|
|
content, attachments = viewed()
|
|
|
|
assert "## Findings" not in content
|
|
assert attachments[FINDINGS_FILE] == "[]"
|
|
|
|
|
|
def test_a_revision_replaces_the_views_and_never_stacks_them():
|
|
first, attachments = viewed()
|
|
|
|
second, _ = add_views(first, dict(attachments), upstream())
|
|
|
|
assert second == first
|
|
assert second.count("## Responsibility matrix") == 1
|