"""Scenario walk-throughs: generated and traced by code, in place of a dry run.""" import json from app.operating_model import scenarios, spec, staffing from app.operating_model.names import VERDICTS_FILE from tests.support import ( CAPABILITIES, CARD, CATALOGUE, GRAPH, SPEC, VERDICTS, graph, scope_data, ) def walked(model=GRAPH, capabilities=CAPABILITIES, text=SPEC) -> dict[str, dict]: scope = scope_data(text) currency = spec.value(scope, "commercial.currency") cost = staffing.compute(model, capabilities, CARD, currency) return { item["id"]: item for item in scenarios.walk(model, capabilities, scope, CATALOGUE, cost) } def test_every_scenario_of_a_sound_model_has_a_path(): found = walked() assert set(found) == { "SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3", "SC-change-S-1", "SC-change-S-2", "SC-change-S-3", "SC-condition-K-1", "SC-condition-K-2", "SC-party-P-2", "SC-budget", } assert all(item["passes"] for item in found.values()) assert found["SC-fault-S-1"]["path"] == ( "Platform SRE responds under C-1, and Service manager is answerable." ) assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, within the budget." def test_a_fault_out_of_hours_fails_when_nobody_responsible_is_on(): model = graph() model["roles"][1]["hours"] = "business" found = walked(model) assert not found["SC-fault-S-1"]["passes"] assert found["SC-fault-S-1"]["path"] == ( "No Role responsible for responding works 24x7 hours." ) # Changes are made in business hours: that path still holds. assert found["SC-change-S-1"]["passes"] def test_an_item_no_responding_capability_covers_fails(): capabilities = [dict(c) for c in CAPABILITIES] capabilities[0]["items"] = ["S-1", "S-2"] found = walked(capabilities=capabilities) assert ( found["SC-fault-S-3"]["path"] == "No Capability that responds to faults covers it." ) assert found["SC-fault-S-1"]["passes"] def test_an_item_nobody_can_change_fails(): capabilities = [dict(c) for c in CAPABILITIES] capabilities[1]["items"] = ["S-1", "S-2"] assert not walked(capabilities=capabilities)["SC-change-S-3"]["passes"] def test_a_condition_nothing_keeps_true_fails(): model = graph() model["checks"].pop(1) found = walked(model) assert found["SC-condition-K-1"]["path"] == "No Check carries it." assert found["SC-condition-K-2"]["passes"] def test_a_party_with_nobody_else_to_turn_to_fails(): model = graph() model["responsibilities"] = [ r for r in model["responsibilities"] if r["id"] not in ("RS-3", "RS-8") ] found = walked(model) assert not found["SC-party-P-2"]["passes"] assert "C-1, C-3" in found["SC-party-P-2"]["path"] def test_a_cost_over_the_budget_fails_the_budget_scenario(): tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000") found = walked(text=tight) assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, over the budget." def test_with_no_budget_stated_there_is_no_budget_scenario(): assert "SC-budget" not in walked( text=SPEC.replace(" monthly_run_budget: 120000\n", "") ) def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code(): found = list(walked().values()) merged = json.loads(scenarios.merge(json.dumps(VERDICTS), found)) assert merged["dry_run"] == { "verdict": "pass", "evidence": "10 of 10 scenario walk-throughs pass.", } assert [c["check"] for c in merged["checks"]][:4] == [ "OC-1", "OC-2", "OC-3", "OC-4", ] assert len(merged["checks"]) == 14 # Merging again replaces the scenarios' verdicts; it never adds them twice. again = json.loads(scenarios.merge(json.dumps(merged), found)) assert again == merged def test_a_failed_scenario_fails_the_run(): model = graph() model["roles"][1]["hours"] = "business" merged = json.loads( scenarios.merge(json.dumps(VERDICTS), list(walked(model).values())) ) assert merged["dry_run"]["verdict"] == "fail" failed = [c["check"] for c in merged["checks"] if c["verdict"] == "fail"] assert failed == ["SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3"] def verdict_faults(verdicts, model=GRAPH) -> list[str]: return scenarios.check_verdicts( {VERDICTS_FILE: json.dumps(verdicts)}, model, VERDICTS_FILE ) def test_a_verdict_on_every_check_is_accepted(): assert verdict_faults(VERDICTS) == [] def test_a_check_without_a_verdict_is_refused(): missing = {"checks": VERDICTS["checks"][:-1]} assert verdict_faults(missing) == ["verdicts.json: Check OC-4 has no verdict"] def test_a_check_that_waits_on_a_finding_cannot_pass(): model = graph() model["checks"][2]["depends_on_finding"] = "AF-1" assert ( "Check OC-3 depends on finding AF-1. It is not_verifiable while the finding stands" in verdict_faults(VERDICTS, model)[0] ) waiting = json.loads(json.dumps(VERDICTS)) waiting["checks"][2]["verdict"] = "not_verifiable" assert verdict_faults(waiting, model) == [] def test_a_verdict_on_something_that_is_not_a_check_is_refused(): extra = { "checks": [ *VERDICTS["checks"], {"check": "OC-9", "verdict": "pass", "evidence": "x"}, ] } assert verdict_faults(extra) == [ "verdicts.json: OC-9 is not a Check of the Target operating model" ] def test_a_check_whose_finding_the_model_could_meet_itself_may_fail(): model = graph() model["checks"][2]["depends_on_finding"] = "AF-1" judged = json.loads(json.dumps(VERDICTS)) judged["checks"][2].update( verdict="fail", evidence="Staff the review 24x7; this is not the solution's." ) assert verdict_faults(judged, model) == [] merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"})) # The failure stands as a failure: the router sends the model back. assert merged["checks"][2]["verdict"] == "fail" def test_a_check_left_not_verifiable_on_its_finding_is_recorded_as_waiting(): judged = json.loads(json.dumps(VERDICTS)) judged["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.") merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"})) assert merged["checks"][2]["verdict"] == "waits_on_finding" assert merged["checks"][2]["finding"] == "AF-1"