219 lines
6.8 KiB
Python
219 lines
6.8 KiB
Python
"""Measures tied to the customer's business outcomes."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from app.operating_model import digest, guide
|
|
from app.operating_model import graph as model
|
|
from app.operating_model.names import GRAPH_FILE
|
|
from tests.support import CAPABILITIES, CARD, CATALOGUE, SPEC, as_file, graph, upstream
|
|
|
|
OUTCOMES = SPEC.replace(
|
|
" - {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}\n"
|
|
" - {id: BO-2, outcome: Regulators always get the evidence they ask for.}\n",
|
|
" - {id: BO-1, outcome: New products reach customers sooner., target: Daily releases.}\n"
|
|
" - {id: BO-2, outcome: Payments are never down for long.}\n",
|
|
)
|
|
# The same spec with no business outcome stated: only the customer can say none.
|
|
NO_OUTCOMES = (
|
|
OUTCOMES.split(" business_outcomes:\n")[0]
|
|
+ " business_outcomes: not_applicable\n"
|
|
+ " requirements:\n"
|
|
+ OUTCOMES.split(" requirements:\n")[1]
|
|
)
|
|
check = model.check(CARD, CATALOGUE.frameworks)
|
|
|
|
|
|
def measured() -> dict:
|
|
changed = graph()
|
|
changed["measures"] = [
|
|
{
|
|
"id": "M-1",
|
|
"name": "Time to restore service",
|
|
"layer": "outcome",
|
|
"outcome": "BO-2",
|
|
"measured_by": "C-1",
|
|
"framework": "dora-metrics",
|
|
"target": "Under 4 hours",
|
|
},
|
|
{
|
|
"id": "M-2",
|
|
"name": "Change lead time",
|
|
"layer": "outcome",
|
|
"outcome": "BO-1",
|
|
"measured_by": "C-2",
|
|
"framework": "dora-metrics",
|
|
},
|
|
]
|
|
return changed
|
|
|
|
|
|
def faults(changed, text=OUTCOMES, capabilities=None) -> list[str]:
|
|
return check(
|
|
as_file(GRAPH_FILE, changed), upstream(spec=text, capabilities=capabilities)
|
|
)
|
|
|
|
|
|
def test_every_outcome_shown_by_a_measure_is_accepted():
|
|
assert faults(measured()) == []
|
|
|
|
|
|
def test_with_no_outcomes_stated_no_measures_are_needed():
|
|
unmeasured = graph()
|
|
unmeasured["measures"] = []
|
|
assert faults(unmeasured, NO_OUTCOMES) == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("change", "problem"),
|
|
[
|
|
(
|
|
lambda g: g["measures"].pop(1),
|
|
"business outcome BO-1 (New products reach customers sooner.) is shown by no Measure",
|
|
),
|
|
(
|
|
lambda g: g["measures"][0].update(outcome="BO-9"),
|
|
"Measure M-1 shows BO-9, which is not a business outcome the customer stated",
|
|
),
|
|
(
|
|
lambda g: g["measures"][0].update(layer="vanity"),
|
|
"Measure M-1: layer must be one of",
|
|
),
|
|
(
|
|
lambda g: g["measures"][0].update(measured_by="C-9"),
|
|
"Measure M-1 is measured by C-9, which is not a Capability",
|
|
),
|
|
(
|
|
lambda g: g["measures"][0].update(framework="okr"),
|
|
"Measure M-1 names framework okr, which is not in the register",
|
|
),
|
|
(
|
|
lambda g: g["responsibilities"].pop(1),
|
|
"Measure M-1 is measured by C-1, and nobody is responsible for it",
|
|
),
|
|
],
|
|
)
|
|
def test_a_measure_that_serves_no_stated_outcome_or_nobody_takes_is_refused(
|
|
change, problem
|
|
):
|
|
changed = measured()
|
|
change(changed)
|
|
|
|
found = faults(changed)
|
|
|
|
assert any(problem in fault for fault in found), found
|
|
|
|
|
|
def test_a_measure_is_refused_when_the_customer_stated_no_outcomes():
|
|
found = faults(measured(), NO_OUTCOMES)
|
|
|
|
assert any("none were stated" in fault for fault in found), found
|
|
|
|
|
|
def test_measuring_how_work_flows_measures_all_three_layers():
|
|
capabilities = [dict(c) for c in CAPABILITIES]
|
|
capabilities[2]["catalogue"] = "delivery-and-developer-experience-measurement"
|
|
changed = measured()
|
|
changed["measures"].append(
|
|
{
|
|
"id": "M-3",
|
|
"name": "Deployment frequency",
|
|
"layer": "outcome",
|
|
"outcome": "BO-1",
|
|
"measured_by": "C-3",
|
|
"framework": "dora-metrics",
|
|
}
|
|
)
|
|
|
|
found = faults(changed, capabilities=capabilities)
|
|
|
|
assert any(
|
|
"C-3 measures how work flows and has no Measure at the guardrail, leading layer"
|
|
in fault
|
|
for fault in found
|
|
), found
|
|
|
|
changed["measures"] += [
|
|
{
|
|
"id": "M-4",
|
|
"name": "Developer satisfaction",
|
|
"layer": "guardrail",
|
|
"outcome": "BO-1",
|
|
"measured_by": "C-3",
|
|
"framework": "space-framework",
|
|
},
|
|
{
|
|
"id": "M-5",
|
|
"name": "Perceived build wait",
|
|
"layer": "leading",
|
|
"outcome": "BO-1",
|
|
"measured_by": "C-3",
|
|
"framework": "devex-framework",
|
|
},
|
|
]
|
|
assert faults(changed, capabilities=capabilities) == []
|
|
|
|
|
|
def test_the_guide_shows_each_outcome_with_what_shows_it_and_who_answers():
|
|
built = upstream(spec=OUTCOMES, model=measured())
|
|
|
|
text = guide.business_outcomes(built, CATALOGUE)
|
|
|
|
assert (
|
|
"**BO-1: New products reach customers sooner.** Target: Daily releases." in text
|
|
)
|
|
assert (
|
|
"| Time to restore service | outcome | dora-metrics | Under 4 hours | "
|
|
"Incident management | Service manager |" in text
|
|
)
|
|
|
|
|
|
def test_the_guide_says_when_no_outcome_was_stated():
|
|
text = guide.business_outcomes(upstream(spec=NO_OUTCOMES), CATALOGUE)
|
|
|
|
assert text.startswith("The customer stated no business outcomes")
|
|
|
|
|
|
def test_an_operational_requirement_may_be_traced_to_a_business_outcome():
|
|
from app.operating_model import chain
|
|
from app.operating_model.names import REQUIREMENTS_FILE
|
|
from tests.support import REQUIREMENTS
|
|
|
|
requirements = json.loads(json.dumps(REQUIREMENTS))
|
|
requirements[3]["source"] = ["BO-2", "service_levels.recovery_time_hours"]
|
|
|
|
assert (
|
|
chain.check_requirements(
|
|
as_file(REQUIREMENTS_FILE, requirements), upstream(spec=OUTCOMES)
|
|
)
|
|
== []
|
|
)
|
|
|
|
|
|
def test_the_architectures_stated_success_criteria_become_business_outcomes():
|
|
from pathlib import Path
|
|
|
|
fixture = (
|
|
Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
|
)
|
|
sources = digest.read_folder(fixture)
|
|
scope = json.loads(sources[digest.SCOPE])
|
|
scope["stated"]["intent"]["success_criteria"] = [
|
|
"Payments engine onboarded within 10 weeks.",
|
|
"Audit findings closed before go-live.",
|
|
]
|
|
sources[digest.SCOPE] = json.dumps(scope)
|
|
|
|
digested = digest.architecture(
|
|
sources, digest.item_kinds(digest.Path(fixture).parents[3], "gcp")
|
|
)
|
|
|
|
assert digested.spec["intent"]["business_outcomes"] == [
|
|
{"id": "BO-1", "outcome": "Payments engine onboarded within 10 weeks."},
|
|
{"id": "BO-2", "outcome": "Audit findings closed before go-live."},
|
|
]
|
|
# In the real run they were assumed for the customer, so none are carried.
|
|
real = digest.architecture(digest.read_folder(fixture), {})
|
|
assert "business_outcomes" not in real.spec["intent"]
|