347 lines
12 KiB
Python
347 lines
12 KiB
Python
"""Scenario walk-throughs: events the operating model must be able to handle.
|
|
|
|
Infrastructure code can be validated by running it. An operating model cannot, so
|
|
each scenario is generated from the graph and the operating spec and traced through
|
|
the graph by code: is there a Capability for it, is someone responsible at the hours
|
|
it happens, is one Role answerable. A broken path is a failed Check. No model writes
|
|
a scenario or judges one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
from app.operating_model import spec
|
|
from app.operating_model.catalogue import Catalogue
|
|
from app.operating_model.names import (
|
|
ACCOUNTABLE,
|
|
CONSULTED,
|
|
CUSTOMER,
|
|
HOURS,
|
|
RESPONSIBLE,
|
|
wider,
|
|
)
|
|
|
|
PREFIX = "SC-" # a scenario's id; the validator gives no verdict under it
|
|
# The verdict on a Check that cannot hold until a finding is met. It is neither a
|
|
# failure of the model nor a gap in it: nothing is revised for it, and it is
|
|
# reported with the finding it waits on.
|
|
WAITING = "waits_on_finding"
|
|
HEADING = "Scenario walk-throughs"
|
|
CHANGE_DOMAIN = "change"
|
|
|
|
|
|
def _scenario(
|
|
identifier: str, scenario: str, about: str, fault: str | None, path: str
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": f"{PREFIX}{identifier}",
|
|
"scenario": scenario,
|
|
"about": about,
|
|
"passes": fault is None,
|
|
"path": fault or path,
|
|
}
|
|
|
|
|
|
def walk(
|
|
graph: Mapping[str, Sequence[Mapping[str, Any]]],
|
|
capabilities: Sequence[Mapping[str, Any]],
|
|
scope: Mapping[str, Any],
|
|
catalogue: Catalogue,
|
|
staffing: Mapping[str, Any] | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
"""Every scenario for this operating model, traced."""
|
|
roles = {str(r["id"]): r for r in graph["roles"]}
|
|
parties = {str(p["id"]): p for p in graph["parties"]}
|
|
by_capability: dict[str, list[Mapping[str, Any]]] = {}
|
|
for item in graph["responsibilities"]:
|
|
by_capability.setdefault(str(item.get("capability")), []).append(item)
|
|
|
|
def holders(capability: str, kind: str) -> list[Mapping[str, Any]]:
|
|
return [
|
|
roles[str(r["role"])]
|
|
for r in by_capability.get(capability, [])
|
|
if r.get("kind") == kind and str(r.get("role")) in roles
|
|
]
|
|
|
|
def entry_of(capability: Mapping[str, Any]) -> Any:
|
|
return catalogue.entries.get(str(capability.get("catalogue")))
|
|
|
|
def covering(item: str) -> list[Mapping[str, Any]]:
|
|
return [c for c in capabilities if item in (c.get("items") or [])]
|
|
|
|
support = str(spec.value(scope, "operations.support_hours") or HOURS[0])
|
|
found = []
|
|
for item in spec.items(scope):
|
|
item_id, label = str(item["id"]), f"{item.get('name')} ({item['id']})"
|
|
|
|
responders = [
|
|
c
|
|
for c in covering(item_id)
|
|
if entry_of(c) and entry_of(c).follows_support_hours
|
|
]
|
|
fault = None
|
|
if not responders:
|
|
fault = "No Capability that responds to faults covers it."
|
|
else:
|
|
on_hours = [
|
|
(c, role)
|
|
for c in responders
|
|
for role in holders(str(c["id"]), RESPONSIBLE)
|
|
if str(role.get("hours")) in HOURS
|
|
and not wider(support, str(role.get("hours")))
|
|
]
|
|
answerable = [c for c, _ in on_hours if holders(str(c["id"]), ACCOUNTABLE)]
|
|
if not on_hours:
|
|
fault = f"No Role responsible for responding works {support} hours."
|
|
elif not answerable:
|
|
fault = "The Capability that responds has no accountable Role."
|
|
chosen = None if fault else on_hours[0]
|
|
found.append(
|
|
_scenario(
|
|
f"fault-{item_id}",
|
|
f"{label} fails at an hour faults are responded to ({support}).",
|
|
item_id,
|
|
fault,
|
|
""
|
|
if chosen is None
|
|
else f"{chosen[1].get('name')} responds under {chosen[0]['id']}, and "
|
|
f"{holders(str(chosen[0]['id']), ACCOUNTABLE)[0].get('name')} is "
|
|
"answerable.",
|
|
)
|
|
)
|
|
|
|
changers = [
|
|
c
|
|
for c in covering(item_id)
|
|
if entry_of(c)
|
|
and entry_of(c).domain == CHANGE_DOMAIN
|
|
and holders(str(c["id"]), RESPONSIBLE)
|
|
]
|
|
found.append(
|
|
_scenario(
|
|
f"change-{item_id}",
|
|
f"{label} must be changed.",
|
|
item_id,
|
|
None
|
|
if changers
|
|
else "No Capability for making changes covers it with a responsible Role.",
|
|
""
|
|
if not changers
|
|
else f"{holders(str(changers[0]['id']), RESPONSIBLE)[0].get('name')} "
|
|
f"makes the change under {changers[0]['id']}.",
|
|
)
|
|
)
|
|
|
|
capability_ids = {str(c["id"]) for c in capabilities}
|
|
for condition in spec.conditions(scope):
|
|
condition_id = str(condition["id"])
|
|
carrying = [k for k in graph["checks"] if k.get("source") == condition_id]
|
|
kept = [
|
|
k
|
|
for k in carrying
|
|
if str(k.get("on")) not in capability_ids
|
|
or holders(str(k.get("on")), RESPONSIBLE)
|
|
]
|
|
found.append(
|
|
_scenario(
|
|
f"condition-{condition_id}",
|
|
f"Condition {condition_id} must be shown to hold: "
|
|
f"{condition.get('condition')}",
|
|
condition_id,
|
|
None
|
|
if kept
|
|
else (
|
|
"No Check carries it."
|
|
if not carrying
|
|
else "The Check that carries it is on a Capability nobody is responsible for."
|
|
),
|
|
""
|
|
if not kept
|
|
else f"Check {kept[0]['id']} on {kept[0].get('on')} keeps it true.",
|
|
)
|
|
)
|
|
|
|
for party in graph["parties"]:
|
|
if party.get("kind") == CUSTOMER:
|
|
continue
|
|
party_id = str(party["id"])
|
|
stranded = [
|
|
str(c["id"])
|
|
for c in capabilities
|
|
if any(
|
|
a.get("party") == party_id for a in holders(str(c["id"]), ACCOUNTABLE)
|
|
)
|
|
and not any(
|
|
parties.get(str(role.get("party")), {}).get("id") != party_id
|
|
for kind in (RESPONSIBLE, CONSULTED)
|
|
for role in holders(str(c["id"]), kind)
|
|
)
|
|
]
|
|
found.append(
|
|
_scenario(
|
|
f"party-{party_id}",
|
|
f"{party.get('name')} cannot be reached.",
|
|
party_id,
|
|
None
|
|
if not stranded
|
|
else "No Role in another Party is responsible or consulted for "
|
|
+ ", ".join(stranded)
|
|
+ ".",
|
|
"Every Capability it is answerable for has a Role in another Party to turn to.",
|
|
)
|
|
)
|
|
|
|
budget = spec.value(scope, "commercial.monthly_run_budget")
|
|
if (
|
|
staffing is not None
|
|
and isinstance(budget, (int, float))
|
|
and not isinstance(budget, bool)
|
|
):
|
|
cost, currency = staffing["cost_per_month"], staffing["currency"]
|
|
found.append(
|
|
_scenario(
|
|
"budget",
|
|
f"A month's cost of operating is compared with the budget of {round(budget):,} {currency}.",
|
|
"commercial.monthly_run_budget",
|
|
None
|
|
if cost <= budget
|
|
else f"The cost is {cost:,} {currency}, over the budget.",
|
|
f"The cost is {cost:,} {currency}, within the budget.",
|
|
)
|
|
)
|
|
return found
|
|
|
|
|
|
def as_verdicts(scenarios: Sequence[Mapping[str, Any]]) -> list[dict[str, str]]:
|
|
return [
|
|
{
|
|
"check": str(item["id"]),
|
|
"verdict": "pass" if item["passes"] else "fail",
|
|
"evidence": f"{item['scenario']} {item['path']}",
|
|
}
|
|
for item in scenarios
|
|
]
|
|
|
|
|
|
def merge(
|
|
verdicts_text: str,
|
|
scenarios: Sequence[Mapping[str, Any]],
|
|
waiting: Mapping[str, str] | None = None,
|
|
) -> str:
|
|
"""verdicts.json with the scenarios' verdicts in it, written by code.
|
|
|
|
The validator's verdicts on the Checks are kept as given, but for one thing:
|
|
a Check in `waiting`, which names the finding it depends on, is recorded as
|
|
waiting on that finding. The overall run is the walk-through: it passes when
|
|
every scenario does.
|
|
"""
|
|
try:
|
|
data = json.loads(verdicts_text)
|
|
except ValueError:
|
|
data = {}
|
|
if not isinstance(data, dict):
|
|
data = {}
|
|
given = [
|
|
(
|
|
{**item, "verdict": WAITING, "finding": waiting[str(item.get("check"))]}
|
|
# Only where the validator found the finding stands. One it failed is
|
|
# something the operating model can provide itself, and is a fault in it.
|
|
if waiting
|
|
and str(item.get("check")) in waiting
|
|
and item.get("verdict") in ("not_verifiable", WAITING)
|
|
else item
|
|
)
|
|
for item in data.get("checks") or []
|
|
if isinstance(item, dict) and not str(item.get("check", "")).startswith(PREFIX)
|
|
]
|
|
failed = [item for item in scenarios if not item["passes"]]
|
|
return json.dumps(
|
|
{
|
|
"dry_run": {
|
|
"verdict": "fail" if failed else "pass",
|
|
"evidence": f"{len(scenarios) - len(failed)} of {len(scenarios)} "
|
|
"scenario walk-throughs pass.",
|
|
},
|
|
"checks": [*given, *as_verdicts(scenarios)],
|
|
"proposed_additions": data.get("proposed_additions") or [],
|
|
},
|
|
indent=2,
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
|
|
def render(scenarios: Sequence[Mapping[str, Any]]) -> str:
|
|
lines = [
|
|
f"## {HEADING}",
|
|
"",
|
|
(
|
|
"Each scenario is generated from the operating model and traced through "
|
|
"it by code."
|
|
),
|
|
"",
|
|
"| Scenario | Result | Path |",
|
|
"| --- | --- | --- |",
|
|
]
|
|
lines += [
|
|
f"| {item['scenario']} | {'pass' if item['passes'] else '**fail**'} | {item['path']} |"
|
|
for item in scenarios
|
|
]
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def check_verdicts(
|
|
attachments: Mapping[str, str],
|
|
graph: Mapping[str, Sequence[Mapping[str, Any]]] | None,
|
|
name: str,
|
|
) -> list[str]:
|
|
"""What is wrong with the validator's verdicts: one on every Check, none on a
|
|
scenario, and never a pass on a Check that depends on a finding."""
|
|
try:
|
|
data = json.loads(attachments[name])
|
|
except ValueError as exc:
|
|
return [f"{name}: not valid JSON ({exc})"]
|
|
if not isinstance(data, dict) or not isinstance(data.get("checks"), list):
|
|
return [f"{name}: must be an object with a list of checks"]
|
|
problems = []
|
|
given: dict[str, str] = {}
|
|
for index, item in enumerate(data["checks"]):
|
|
where = f"{name}: checks[{index}]"
|
|
if not isinstance(item, dict) or not item.get("check"):
|
|
problems.append(f"{where} needs a check")
|
|
continue
|
|
identifier = str(item["check"])
|
|
if identifier.startswith(PREFIX):
|
|
continue # the walk-throughs' own, written by code on an earlier attempt
|
|
if item.get("verdict") == WAITING:
|
|
# Recorded by code on an earlier attempt; the validator's own verdict
|
|
# on such a Check is not_verifiable.
|
|
item = {**item, "verdict": "not_verifiable"}
|
|
if item.get("verdict") not in ("pass", "fail", "not_verifiable"):
|
|
problems.append(
|
|
f"{where} ({identifier}): verdict must be pass, fail or not_verifiable"
|
|
)
|
|
if not str(item.get("evidence") or "").strip():
|
|
problems.append(f"{where} ({identifier}) needs evidence")
|
|
given[identifier] = str(item.get("verdict"))
|
|
for check in (graph or {}).get("checks", []):
|
|
identifier = str(check["id"])
|
|
if identifier not in given:
|
|
problems.append(f"{name}: Check {identifier} has no verdict")
|
|
elif check.get("depends_on_finding") and given[identifier] == "pass":
|
|
problems.append(
|
|
f"{name}: Check {identifier} depends on finding "
|
|
f"{check['depends_on_finding']}. It is not_verifiable while the finding "
|
|
"stands, or fail when the operating model could meet it itself"
|
|
)
|
|
known = {str(check["id"]) for check in (graph or {}).get("checks", [])}
|
|
if graph is not None:
|
|
problems += [
|
|
f"{name}: {identifier} is not a Check of the Target operating model"
|
|
for identifier in given
|
|
if identifier not in known
|
|
]
|
|
return problems
|