The operating model is refused while an Operational requirement is verified by no Check, from it or from a Condition it comes from; the architect is told so, and the operating-chain Skill lists it. Six of nine earlier runs left a requirement unverified. The evaluation checks it too, runs each case N times (--repeat), reports the spread of people, cost, Roles, Checks and Capabilities (spread.json), and writes report.md to publish. Result folders are relative to the run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
220 lines
6.5 KiB
Python
220 lines
6.5 KiB
Python
"""Scenario walk-throughs: generated and traced by code, in place of a dry run."""
|
|
|
|
import json
|
|
|
|
from app.operating_model import scenarios, spec, staffing
|
|
from app.operating_model.names import VERDICTS_FILE
|
|
from tests.support import (
|
|
CAPABILITIES,
|
|
CARD,
|
|
CATALOGUE,
|
|
GRAPH,
|
|
SPEC,
|
|
VERDICTS,
|
|
graph,
|
|
scope_data,
|
|
)
|
|
|
|
|
|
def walked(model=GRAPH, capabilities=CAPABILITIES, text=SPEC) -> dict[str, dict]:
|
|
scope = scope_data(text)
|
|
currency = spec.value(scope, "commercial.currency")
|
|
cost = staffing.compute(model, capabilities, CARD, currency)
|
|
return {
|
|
item["id"]: item
|
|
for item in scenarios.walk(model, capabilities, scope, CATALOGUE, cost)
|
|
}
|
|
|
|
|
|
def test_every_scenario_of_a_sound_model_has_a_path():
|
|
found = walked()
|
|
|
|
assert set(found) == {
|
|
"SC-fault-S-1",
|
|
"SC-fault-S-2",
|
|
"SC-fault-S-3",
|
|
"SC-change-S-1",
|
|
"SC-change-S-2",
|
|
"SC-change-S-3",
|
|
"SC-condition-K-1",
|
|
"SC-condition-K-2",
|
|
"SC-party-P-2",
|
|
"SC-budget",
|
|
}
|
|
assert all(item["passes"] for item in found.values())
|
|
assert found["SC-fault-S-1"]["path"] == (
|
|
"Platform SRE responds under C-1, and Service manager is answerable."
|
|
)
|
|
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, within the budget."
|
|
|
|
|
|
def test_a_fault_out_of_hours_fails_when_nobody_responsible_is_on():
|
|
model = graph()
|
|
model["roles"][1]["hours"] = "business"
|
|
|
|
found = walked(model)
|
|
|
|
assert not found["SC-fault-S-1"]["passes"]
|
|
assert found["SC-fault-S-1"]["path"] == (
|
|
"No Role responsible for responding works 24x7 hours."
|
|
)
|
|
# Changes are made in business hours: that path still holds.
|
|
assert found["SC-change-S-1"]["passes"]
|
|
|
|
|
|
def test_an_item_no_responding_capability_covers_fails():
|
|
capabilities = [dict(c) for c in CAPABILITIES]
|
|
capabilities[0]["items"] = ["S-1", "S-2"]
|
|
|
|
found = walked(capabilities=capabilities)
|
|
|
|
assert (
|
|
found["SC-fault-S-3"]["path"]
|
|
== "No Capability that responds to faults covers it."
|
|
)
|
|
assert found["SC-fault-S-1"]["passes"]
|
|
|
|
|
|
def test_an_item_nobody_can_change_fails():
|
|
capabilities = [dict(c) for c in CAPABILITIES]
|
|
capabilities[1]["items"] = ["S-1", "S-2"]
|
|
|
|
assert not walked(capabilities=capabilities)["SC-change-S-3"]["passes"]
|
|
|
|
|
|
def test_a_condition_nothing_keeps_true_fails():
|
|
model = graph()
|
|
model["checks"].pop(1)
|
|
|
|
found = walked(model)
|
|
|
|
assert found["SC-condition-K-1"]["path"] == "No Check carries it."
|
|
assert found["SC-condition-K-2"]["passes"]
|
|
|
|
|
|
def test_a_party_with_nobody_else_to_turn_to_fails():
|
|
model = graph()
|
|
model["responsibilities"] = [
|
|
r for r in model["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
|
|
]
|
|
|
|
found = walked(model)
|
|
|
|
assert not found["SC-party-P-2"]["passes"]
|
|
assert "C-1, C-3" in found["SC-party-P-2"]["path"]
|
|
|
|
|
|
def test_a_cost_over_the_budget_fails_the_budget_scenario():
|
|
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
|
|
|
|
found = walked(text=tight)
|
|
|
|
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, over the budget."
|
|
|
|
|
|
def test_with_no_budget_stated_there_is_no_budget_scenario():
|
|
assert "SC-budget" not in walked(
|
|
text=SPEC.replace(" monthly_run_budget: 120000\n", "")
|
|
)
|
|
|
|
|
|
def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code():
|
|
found = list(walked().values())
|
|
|
|
merged = json.loads(scenarios.merge(json.dumps(VERDICTS), found))
|
|
|
|
assert merged["dry_run"] == {
|
|
"verdict": "pass",
|
|
"evidence": "10 of 10 scenario walk-throughs pass.",
|
|
}
|
|
assert [c["check"] for c in merged["checks"]][:4] == [
|
|
"OC-1",
|
|
"OC-2",
|
|
"OC-3",
|
|
"OC-4",
|
|
]
|
|
assert len(merged["checks"]) == 16
|
|
# Merging again replaces the scenarios' verdicts; it never adds them twice.
|
|
again = json.loads(scenarios.merge(json.dumps(merged), found))
|
|
assert again == merged
|
|
|
|
|
|
def test_a_failed_scenario_fails_the_run():
|
|
model = graph()
|
|
model["roles"][1]["hours"] = "business"
|
|
|
|
merged = json.loads(
|
|
scenarios.merge(json.dumps(VERDICTS), list(walked(model).values()))
|
|
)
|
|
|
|
assert merged["dry_run"]["verdict"] == "fail"
|
|
failed = [c["check"] for c in merged["checks"] if c["verdict"] == "fail"]
|
|
assert failed == ["SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3"]
|
|
|
|
|
|
def verdict_faults(verdicts, model=GRAPH) -> list[str]:
|
|
return scenarios.check_verdicts(
|
|
{VERDICTS_FILE: json.dumps(verdicts)}, model, VERDICTS_FILE
|
|
)
|
|
|
|
|
|
def test_a_verdict_on_every_check_is_accepted():
|
|
assert verdict_faults(VERDICTS) == []
|
|
|
|
|
|
def test_a_check_without_a_verdict_is_refused():
|
|
missing = {"checks": VERDICTS["checks"][:-1]}
|
|
|
|
assert verdict_faults(missing) == ["verdicts.json: Check OC-6 has no verdict"]
|
|
|
|
|
|
def test_a_check_that_waits_on_a_finding_cannot_pass():
|
|
model = graph()
|
|
model["checks"][2]["depends_on_finding"] = "AF-1"
|
|
|
|
assert (
|
|
"Check OC-3 depends on finding AF-1. It is not_verifiable while the finding stands"
|
|
in verdict_faults(VERDICTS, model)[0]
|
|
)
|
|
|
|
waiting = json.loads(json.dumps(VERDICTS))
|
|
waiting["checks"][2]["verdict"] = "not_verifiable"
|
|
assert verdict_faults(waiting, model) == []
|
|
|
|
|
|
def test_a_verdict_on_something_that_is_not_a_check_is_refused():
|
|
extra = {
|
|
"checks": [
|
|
*VERDICTS["checks"],
|
|
{"check": "OC-9", "verdict": "pass", "evidence": "x"},
|
|
]
|
|
}
|
|
|
|
assert verdict_faults(extra) == [
|
|
"verdicts.json: OC-9 is not a Check of the Target operating model"
|
|
]
|
|
|
|
|
|
def test_a_check_whose_finding_the_model_could_meet_itself_may_fail():
|
|
model = graph()
|
|
model["checks"][2]["depends_on_finding"] = "AF-1"
|
|
judged = json.loads(json.dumps(VERDICTS))
|
|
judged["checks"][2].update(
|
|
verdict="fail", evidence="Staff the review 24x7; this is not the solution's."
|
|
)
|
|
|
|
assert verdict_faults(judged, model) == []
|
|
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
|
|
# The failure stands as a failure: the router sends the model back.
|
|
assert merged["checks"][2]["verdict"] == "fail"
|
|
|
|
|
|
def test_a_check_left_not_verifiable_on_its_finding_is_recorded_as_waiting():
|
|
judged = json.loads(json.dumps(VERDICTS))
|
|
judged["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
|
|
|
|
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
|
|
|
|
assert merged["checks"][2]["verdict"] == "waits_on_finding"
|
|
assert merged["checks"][2]["finding"] == "AF-1"
|