"""Measures tied to the customer's business outcomes.""" import json import pytest from app.operating_model import digest, guide from app.operating_model import graph as model from app.operating_model.names import GRAPH_FILE from tests.support import CAPABILITIES, CARD, CATALOGUE, SPEC, as_file, graph, upstream OUTCOMES = SPEC.replace( " - {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}\n" " - {id: BO-2, outcome: Regulators always get the evidence they ask for.}\n", " - {id: BO-1, outcome: New products reach customers sooner., target: Daily releases.}\n" " - {id: BO-2, outcome: Payments are never down for long.}\n", ) # The same spec with no business outcome stated: only the customer can say none. NO_OUTCOMES = ( OUTCOMES.split(" business_outcomes:\n")[0] + " business_outcomes: not_applicable\n" + " requirements:\n" + OUTCOMES.split(" requirements:\n")[1] ) check = model.check(CARD, CATALOGUE.frameworks) def measured() -> dict: changed = graph() changed["measures"] = [ { "id": "M-1", "name": "Time to restore service", "layer": "outcome", "outcome": "BO-2", "measured_by": "C-1", "framework": "dora-metrics", "target": "Under 4 hours", }, { "id": "M-2", "name": "Change lead time", "layer": "outcome", "outcome": "BO-1", "measured_by": "C-2", "framework": "dora-metrics", }, ] return changed def faults(changed, text=OUTCOMES, capabilities=None) -> list[str]: return check( as_file(GRAPH_FILE, changed), upstream(spec=text, capabilities=capabilities) ) def test_every_outcome_shown_by_a_measure_is_accepted(): assert faults(measured()) == [] def test_with_no_outcomes_stated_no_measures_are_needed(): unmeasured = graph() unmeasured["measures"] = [] assert faults(unmeasured, NO_OUTCOMES) == [] @pytest.mark.parametrize( ("change", "problem"), [ ( lambda g: g["measures"].pop(1), "business outcome BO-1 (New products reach customers sooner.) is shown by no Measure", ), ( lambda g: g["measures"][0].update(outcome="BO-9"), "Measure M-1 shows BO-9, which is not a business outcome the customer stated", ), ( lambda g: g["measures"][0].update(layer="vanity"), "Measure M-1: layer must be one of", ), ( lambda g: g["measures"][0].update(measured_by="C-9"), "Measure M-1 is measured by C-9, which is not a Capability", ), ( lambda g: g["measures"][0].update(framework="okr"), "Measure M-1 names framework okr, which is not in the register", ), ( lambda g: g["responsibilities"].pop(1), "Measure M-1 is measured by C-1, and nobody is responsible for it", ), ], ) def test_a_measure_that_serves_no_stated_outcome_or_nobody_takes_is_refused( change, problem ): changed = measured() change(changed) found = faults(changed) assert any(problem in fault for fault in found), found def test_a_measure_is_refused_when_the_customer_stated_no_outcomes(): found = faults(measured(), NO_OUTCOMES) assert any("none were stated" in fault for fault in found), found def test_measuring_how_work_flows_measures_all_three_layers(): capabilities = [dict(c) for c in CAPABILITIES] capabilities[2]["catalogue"] = "delivery-and-developer-experience-measurement" changed = measured() changed["measures"].append( { "id": "M-3", "name": "Deployment frequency", "layer": "outcome", "outcome": "BO-1", "measured_by": "C-3", "framework": "dora-metrics", } ) found = faults(changed, capabilities=capabilities) assert any( "C-3 measures how work flows and has no Measure at the guardrail, leading layer" in fault for fault in found ), found changed["measures"] += [ { "id": "M-4", "name": "Developer satisfaction", "layer": "guardrail", "outcome": "BO-1", "measured_by": "C-3", "framework": "space-framework", }, { "id": "M-5", "name": "Perceived build wait", "layer": "leading", "outcome": "BO-1", "measured_by": "C-3", "framework": "devex-framework", }, ] assert faults(changed, capabilities=capabilities) == [] def test_the_guide_shows_each_outcome_with_what_shows_it_and_who_answers(): built = upstream(spec=OUTCOMES, model=measured()) text = guide.business_outcomes(built, CATALOGUE) assert ( "**BO-1: New products reach customers sooner.** Target: Daily releases." in text ) assert ( "| Time to restore service | outcome | dora-metrics | Under 4 hours | " "Incident management | Service manager |" in text ) def test_the_guide_says_when_no_outcome_was_stated(): text = guide.business_outcomes(upstream(spec=NO_OUTCOMES), CATALOGUE) assert text.startswith("The customer stated no business outcomes") def test_an_operational_requirement_may_be_traced_to_a_business_outcome(): from app.operating_model import chain from app.operating_model.names import REQUIREMENTS_FILE from tests.support import REQUIREMENTS requirements = json.loads(json.dumps(REQUIREMENTS)) requirements[3]["source"] = ["BO-2", "service_levels.recovery_time_hours"] assert ( chain.check_requirements( as_file(REQUIREMENTS_FILE, requirements), upstream(spec=OUTCOMES) ) == [] ) def test_the_architectures_stated_success_criteria_become_business_outcomes(): from pathlib import Path fixture = ( Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone" ) sources = digest.read_folder(fixture) scope = json.loads(sources[digest.SCOPE]) scope["stated"]["intent"]["success_criteria"] = [ "Payments engine onboarded within 10 weeks.", "Audit findings closed before go-live.", ] sources[digest.SCOPE] = json.dumps(scope) digested = digest.architecture( sources, digest.item_kinds(digest.Path(fixture).parents[3], "gcp") ) assert digested.spec["intent"]["business_outcomes"] == [ {"id": "BO-1", "outcome": "Payments engine onboarded within 10 weeks."}, {"id": "BO-2", "outcome": "Audit findings closed before go-live."}, ] # In the real run they were assumed for the customer, so none are carried. real = digest.architecture(digest.read_folder(fixture), {}) assert "business_outcomes" not in real.spec["intent"]