Files
tom_analysis_agent/tests/test_outcomes.py

219 lines
6.8 KiB
Python

"""Measures tied to the customer's business outcomes."""
import json
import pytest
from app.operating_model import digest, guide
from app.operating_model import graph as model
from app.operating_model.names import GRAPH_FILE
from tests.support import CAPABILITIES, CARD, CATALOGUE, SPEC, as_file, graph, upstream
OUTCOMES = SPEC.replace(
" - {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}\n"
" - {id: BO-2, outcome: Regulators always get the evidence they ask for.}\n",
" - {id: BO-1, outcome: New products reach customers sooner., target: Daily releases.}\n"
" - {id: BO-2, outcome: Payments are never down for long.}\n",
)
# The same spec with no business outcome stated: only the customer can say none.
NO_OUTCOMES = (
OUTCOMES.split(" business_outcomes:\n")[0]
+ " business_outcomes: not_applicable\n"
+ " requirements:\n"
+ OUTCOMES.split(" requirements:\n")[1]
)
check = model.check(CARD, CATALOGUE.frameworks)
def measured() -> dict:
changed = graph()
changed["measures"] = [
{
"id": "M-1",
"name": "Time to restore service",
"layer": "outcome",
"outcome": "BO-2",
"measured_by": "C-1",
"framework": "dora-metrics",
"target": "Under 4 hours",
},
{
"id": "M-2",
"name": "Change lead time",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-2",
"framework": "dora-metrics",
},
]
return changed
def faults(changed, text=OUTCOMES, capabilities=None) -> list[str]:
return check(
as_file(GRAPH_FILE, changed), upstream(spec=text, capabilities=capabilities)
)
def test_every_outcome_shown_by_a_measure_is_accepted():
assert faults(measured()) == []
def test_with_no_outcomes_stated_no_measures_are_needed():
unmeasured = graph()
unmeasured["measures"] = []
assert faults(unmeasured, NO_OUTCOMES) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda g: g["measures"].pop(1),
"business outcome BO-1 (New products reach customers sooner.) is shown by no Measure",
),
(
lambda g: g["measures"][0].update(outcome="BO-9"),
"Measure M-1 shows BO-9, which is not a business outcome the customer stated",
),
(
lambda g: g["measures"][0].update(layer="vanity"),
"Measure M-1: layer must be one of",
),
(
lambda g: g["measures"][0].update(measured_by="C-9"),
"Measure M-1 is measured by C-9, which is not a Capability",
),
(
lambda g: g["measures"][0].update(framework="okr"),
"Measure M-1 names framework okr, which is not in the register",
),
(
lambda g: g["responsibilities"].pop(1),
"Measure M-1 is measured by C-1, and nobody is responsible for it",
),
],
)
def test_a_measure_that_serves_no_stated_outcome_or_nobody_takes_is_refused(
change, problem
):
changed = measured()
change(changed)
found = faults(changed)
assert any(problem in fault for fault in found), found
def test_a_measure_is_refused_when_the_customer_stated_no_outcomes():
found = faults(measured(), NO_OUTCOMES)
assert any("none were stated" in fault for fault in found), found
def test_measuring_how_work_flows_measures_all_three_layers():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[2]["catalogue"] = "delivery-and-developer-experience-measurement"
changed = measured()
changed["measures"].append(
{
"id": "M-3",
"name": "Deployment frequency",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "dora-metrics",
}
)
found = faults(changed, capabilities=capabilities)
assert any(
"C-3 measures how work flows and has no Measure at the guardrail, leading layer"
in fault
for fault in found
), found
changed["measures"] += [
{
"id": "M-4",
"name": "Developer satisfaction",
"layer": "guardrail",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "space-framework",
},
{
"id": "M-5",
"name": "Perceived build wait",
"layer": "leading",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "devex-framework",
},
]
assert faults(changed, capabilities=capabilities) == []
def test_the_guide_shows_each_outcome_with_what_shows_it_and_who_answers():
built = upstream(spec=OUTCOMES, model=measured())
text = guide.business_outcomes(built, CATALOGUE)
assert (
"**BO-1: New products reach customers sooner.** Target: Daily releases." in text
)
assert (
"| Time to restore service | outcome | dora-metrics | Under 4 hours | "
"Incident management | Service manager |" in text
)
def test_the_guide_says_when_no_outcome_was_stated():
text = guide.business_outcomes(upstream(spec=NO_OUTCOMES), CATALOGUE)
assert text.startswith("The customer stated no business outcomes")
def test_an_operational_requirement_may_be_traced_to_a_business_outcome():
from app.operating_model import chain
from app.operating_model.names import REQUIREMENTS_FILE
from tests.support import REQUIREMENTS
requirements = json.loads(json.dumps(REQUIREMENTS))
requirements[3]["source"] = ["BO-2", "service_levels.recovery_time_hours"]
assert (
chain.check_requirements(
as_file(REQUIREMENTS_FILE, requirements), upstream(spec=OUTCOMES)
)
== []
)
def test_the_architectures_stated_success_criteria_become_business_outcomes():
from pathlib import Path
fixture = (
Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
)
sources = digest.read_folder(fixture)
scope = json.loads(sources[digest.SCOPE])
scope["stated"]["intent"]["success_criteria"] = [
"Payments engine onboarded within 10 weeks.",
"Audit findings closed before go-live.",
]
sources[digest.SCOPE] = json.dumps(scope)
digested = digest.architecture(
sources, digest.item_kinds(digest.Path(fixture).parents[3], "gcp")
)
assert digested.spec["intent"]["business_outcomes"] == [
{"id": "BO-1", "outcome": "Payments engine onboarded within 10 weeks."},
{"id": "BO-2", "outcome": "Audit findings closed before go-live."},
]
# In the real run they were assumed for the customer, so none are carried.
real = digest.architecture(digest.read_folder(fixture), {})
assert "business_outcomes" not in real.spec["intent"]