Files
tom_analysis_agent/tests/test_scenarios.py

220 lines
6.5 KiB
Python

"""Scenario walk-throughs: generated and traced by code, in place of a dry run."""
import json
from app.operating_model import scenarios, spec, staffing
from app.operating_model.names import VERDICTS_FILE
from tests.support import (
CAPABILITIES,
CARD,
CATALOGUE,
GRAPH,
SPEC,
VERDICTS,
graph,
scope_data,
)
def walked(model=GRAPH, capabilities=CAPABILITIES, text=SPEC) -> dict[str, dict]:
scope = scope_data(text)
currency = spec.value(scope, "commercial.currency")
cost = staffing.compute(model, capabilities, CARD, currency)
return {
item["id"]: item
for item in scenarios.walk(model, capabilities, scope, CATALOGUE, cost)
}
def test_every_scenario_of_a_sound_model_has_a_path():
found = walked()
assert set(found) == {
"SC-fault-S-1",
"SC-fault-S-2",
"SC-fault-S-3",
"SC-change-S-1",
"SC-change-S-2",
"SC-change-S-3",
"SC-condition-K-1",
"SC-condition-K-2",
"SC-party-P-2",
"SC-budget",
}
assert all(item["passes"] for item in found.values())
assert found["SC-fault-S-1"]["path"] == (
"Platform SRE responds under C-1, and Service manager is answerable."
)
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, within the budget."
def test_a_fault_out_of_hours_fails_when_nobody_responsible_is_on():
model = graph()
model["roles"][1]["hours"] = "business"
found = walked(model)
assert not found["SC-fault-S-1"]["passes"]
assert found["SC-fault-S-1"]["path"] == (
"No Role responsible for responding works 24x7 hours."
)
# Changes are made in business hours: that path still holds.
assert found["SC-change-S-1"]["passes"]
def test_an_item_no_responding_capability_covers_fails():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[0]["items"] = ["S-1", "S-2"]
found = walked(capabilities=capabilities)
assert (
found["SC-fault-S-3"]["path"]
== "No Capability that responds to faults covers it."
)
assert found["SC-fault-S-1"]["passes"]
def test_an_item_nobody_can_change_fails():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[1]["items"] = ["S-1", "S-2"]
assert not walked(capabilities=capabilities)["SC-change-S-3"]["passes"]
def test_a_condition_nothing_keeps_true_fails():
model = graph()
model["checks"].pop(1)
found = walked(model)
assert found["SC-condition-K-1"]["path"] == "No Check carries it."
assert found["SC-condition-K-2"]["passes"]
def test_a_party_with_nobody_else_to_turn_to_fails():
model = graph()
model["responsibilities"] = [
r for r in model["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
]
found = walked(model)
assert not found["SC-party-P-2"]["passes"]
assert "C-1, C-3" in found["SC-party-P-2"]["path"]
def test_a_cost_over_the_budget_fails_the_budget_scenario():
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
found = walked(text=tight)
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, over the budget."
def test_with_no_budget_stated_there_is_no_budget_scenario():
assert "SC-budget" not in walked(
text=SPEC.replace(" monthly_run_budget: 120000\n", "")
)
def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code():
found = list(walked().values())
merged = json.loads(scenarios.merge(json.dumps(VERDICTS), found))
assert merged["dry_run"] == {
"verdict": "pass",
"evidence": "10 of 10 scenario walk-throughs pass.",
}
assert [c["check"] for c in merged["checks"]][:4] == [
"OC-1",
"OC-2",
"OC-3",
"OC-4",
]
assert len(merged["checks"]) == 14
# Merging again replaces the scenarios' verdicts; it never adds them twice.
again = json.loads(scenarios.merge(json.dumps(merged), found))
assert again == merged
def test_a_failed_scenario_fails_the_run():
model = graph()
model["roles"][1]["hours"] = "business"
merged = json.loads(
scenarios.merge(json.dumps(VERDICTS), list(walked(model).values()))
)
assert merged["dry_run"]["verdict"] == "fail"
failed = [c["check"] for c in merged["checks"] if c["verdict"] == "fail"]
assert failed == ["SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3"]
def verdict_faults(verdicts, model=GRAPH) -> list[str]:
return scenarios.check_verdicts(
{VERDICTS_FILE: json.dumps(verdicts)}, model, VERDICTS_FILE
)
def test_a_verdict_on_every_check_is_accepted():
assert verdict_faults(VERDICTS) == []
def test_a_check_without_a_verdict_is_refused():
missing = {"checks": VERDICTS["checks"][:-1]}
assert verdict_faults(missing) == ["verdicts.json: Check OC-4 has no verdict"]
def test_a_check_that_waits_on_a_finding_cannot_pass():
model = graph()
model["checks"][2]["depends_on_finding"] = "AF-1"
assert (
"Check OC-3 depends on finding AF-1. It is not_verifiable while the finding stands"
in verdict_faults(VERDICTS, model)[0]
)
waiting = json.loads(json.dumps(VERDICTS))
waiting["checks"][2]["verdict"] = "not_verifiable"
assert verdict_faults(waiting, model) == []
def test_a_verdict_on_something_that_is_not_a_check_is_refused():
extra = {
"checks": [
*VERDICTS["checks"],
{"check": "OC-9", "verdict": "pass", "evidence": "x"},
]
}
assert verdict_faults(extra) == [
"verdicts.json: OC-9 is not a Check of the Target operating model"
]
def test_a_check_whose_finding_the_model_could_meet_itself_may_fail():
model = graph()
model["checks"][2]["depends_on_finding"] = "AF-1"
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(
verdict="fail", evidence="Staff the review 24x7; this is not the solution's."
)
assert verdict_faults(judged, model) == []
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
# The failure stands as a failure: the router sends the model back.
assert merged["checks"][2]["verdict"] == "fail"
def test_a_check_left_not_verifiable_on_its_finding_is_recorded_as_waiting():
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
assert merged["checks"][2]["verdict"] == "waits_on_finding"
assert merged["checks"][2]["finding"] == "AF-1"