Files
tom_analysis_agent/app/operating_model/scenarios.py

347 lines
12 KiB
Python

"""Scenario walk-throughs: events the operating model must be able to handle.
Infrastructure code can be validated by running it. An operating model cannot, so
each scenario is generated from the graph and the operating spec and traced through
the graph by code: is there a Capability for it, is someone responsible at the hours
it happens, is one Role answerable. A broken path is a failed Check. No model writes
a scenario or judges one.
"""
from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from typing import Any
from app.operating_model import spec
from app.operating_model.catalogue import Catalogue
from app.operating_model.names import (
ACCOUNTABLE,
CONSULTED,
CUSTOMER,
HOURS,
RESPONSIBLE,
wider,
)
PREFIX = "SC-" # a scenario's id; the validator gives no verdict under it
# The verdict on a Check that cannot hold until a finding is met. It is neither a
# failure of the model nor a gap in it: nothing is revised for it, and it is
# reported with the finding it waits on.
WAITING = "waits_on_finding"
HEADING = "Scenario walk-throughs"
CHANGE_DOMAIN = "change"
def _scenario(
identifier: str, scenario: str, about: str, fault: str | None, path: str
) -> dict[str, Any]:
return {
"id": f"{PREFIX}{identifier}",
"scenario": scenario,
"about": about,
"passes": fault is None,
"path": fault or path,
}
def walk(
graph: Mapping[str, Sequence[Mapping[str, Any]]],
capabilities: Sequence[Mapping[str, Any]],
scope: Mapping[str, Any],
catalogue: Catalogue,
staffing: Mapping[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""Every scenario for this operating model, traced."""
roles = {str(r["id"]): r for r in graph["roles"]}
parties = {str(p["id"]): p for p in graph["parties"]}
by_capability: dict[str, list[Mapping[str, Any]]] = {}
for item in graph["responsibilities"]:
by_capability.setdefault(str(item.get("capability")), []).append(item)
def holders(capability: str, kind: str) -> list[Mapping[str, Any]]:
return [
roles[str(r["role"])]
for r in by_capability.get(capability, [])
if r.get("kind") == kind and str(r.get("role")) in roles
]
def entry_of(capability: Mapping[str, Any]) -> Any:
return catalogue.entries.get(str(capability.get("catalogue")))
def covering(item: str) -> list[Mapping[str, Any]]:
return [c for c in capabilities if item in (c.get("items") or [])]
support = str(spec.value(scope, "operations.support_hours") or HOURS[0])
found = []
for item in spec.items(scope):
item_id, label = str(item["id"]), f"{item.get('name')} ({item['id']})"
responders = [
c
for c in covering(item_id)
if entry_of(c) and entry_of(c).follows_support_hours
]
fault = None
if not responders:
fault = "No Capability that responds to faults covers it."
else:
on_hours = [
(c, role)
for c in responders
for role in holders(str(c["id"]), RESPONSIBLE)
if str(role.get("hours")) in HOURS
and not wider(support, str(role.get("hours")))
]
answerable = [c for c, _ in on_hours if holders(str(c["id"]), ACCOUNTABLE)]
if not on_hours:
fault = f"No Role responsible for responding works {support} hours."
elif not answerable:
fault = "The Capability that responds has no accountable Role."
chosen = None if fault else on_hours[0]
found.append(
_scenario(
f"fault-{item_id}",
f"{label} fails at an hour faults are responded to ({support}).",
item_id,
fault,
""
if chosen is None
else f"{chosen[1].get('name')} responds under {chosen[0]['id']}, and "
f"{holders(str(chosen[0]['id']), ACCOUNTABLE)[0].get('name')} is "
"answerable.",
)
)
changers = [
c
for c in covering(item_id)
if entry_of(c)
and entry_of(c).domain == CHANGE_DOMAIN
and holders(str(c["id"]), RESPONSIBLE)
]
found.append(
_scenario(
f"change-{item_id}",
f"{label} must be changed.",
item_id,
None
if changers
else "No Capability for making changes covers it with a responsible Role.",
""
if not changers
else f"{holders(str(changers[0]['id']), RESPONSIBLE)[0].get('name')} "
f"makes the change under {changers[0]['id']}.",
)
)
capability_ids = {str(c["id"]) for c in capabilities}
for condition in spec.conditions(scope):
condition_id = str(condition["id"])
carrying = [k for k in graph["checks"] if k.get("source") == condition_id]
kept = [
k
for k in carrying
if str(k.get("on")) not in capability_ids
or holders(str(k.get("on")), RESPONSIBLE)
]
found.append(
_scenario(
f"condition-{condition_id}",
f"Condition {condition_id} must be shown to hold: "
f"{condition.get('condition')}",
condition_id,
None
if kept
else (
"No Check carries it."
if not carrying
else "The Check that carries it is on a Capability nobody is responsible for."
),
""
if not kept
else f"Check {kept[0]['id']} on {kept[0].get('on')} keeps it true.",
)
)
for party in graph["parties"]:
if party.get("kind") == CUSTOMER:
continue
party_id = str(party["id"])
stranded = [
str(c["id"])
for c in capabilities
if any(
a.get("party") == party_id for a in holders(str(c["id"]), ACCOUNTABLE)
)
and not any(
parties.get(str(role.get("party")), {}).get("id") != party_id
for kind in (RESPONSIBLE, CONSULTED)
for role in holders(str(c["id"]), kind)
)
]
found.append(
_scenario(
f"party-{party_id}",
f"{party.get('name')} cannot be reached.",
party_id,
None
if not stranded
else "No Role in another Party is responsible or consulted for "
+ ", ".join(stranded)
+ ".",
"Every Capability it is answerable for has a Role in another Party to turn to.",
)
)
budget = spec.value(scope, "commercial.monthly_run_budget")
if (
staffing is not None
and isinstance(budget, (int, float))
and not isinstance(budget, bool)
):
cost, currency = staffing["cost_per_month"], staffing["currency"]
found.append(
_scenario(
"budget",
f"A month's cost of operating is compared with the budget of {round(budget):,} {currency}.",
"commercial.monthly_run_budget",
None
if cost <= budget
else f"The cost is {cost:,} {currency}, over the budget.",
f"The cost is {cost:,} {currency}, within the budget.",
)
)
return found
def as_verdicts(scenarios: Sequence[Mapping[str, Any]]) -> list[dict[str, str]]:
return [
{
"check": str(item["id"]),
"verdict": "pass" if item["passes"] else "fail",
"evidence": f"{item['scenario']} {item['path']}",
}
for item in scenarios
]
def merge(
verdicts_text: str,
scenarios: Sequence[Mapping[str, Any]],
waiting: Mapping[str, str] | None = None,
) -> str:
"""verdicts.json with the scenarios' verdicts in it, written by code.
The validator's verdicts on the Checks are kept as given, but for one thing:
a Check in `waiting`, which names the finding it depends on, is recorded as
waiting on that finding. The overall run is the walk-through: it passes when
every scenario does.
"""
try:
data = json.loads(verdicts_text)
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
given = [
(
{**item, "verdict": WAITING, "finding": waiting[str(item.get("check"))]}
# Only where the validator found the finding stands. One it failed is
# something the operating model can provide itself, and is a fault in it.
if waiting
and str(item.get("check")) in waiting
and item.get("verdict") in ("not_verifiable", WAITING)
else item
)
for item in data.get("checks") or []
if isinstance(item, dict) and not str(item.get("check", "")).startswith(PREFIX)
]
failed = [item for item in scenarios if not item["passes"]]
return json.dumps(
{
"dry_run": {
"verdict": "fail" if failed else "pass",
"evidence": f"{len(scenarios) - len(failed)} of {len(scenarios)} "
"scenario walk-throughs pass.",
},
"checks": [*given, *as_verdicts(scenarios)],
"proposed_additions": data.get("proposed_additions") or [],
},
indent=2,
ensure_ascii=False,
)
def render(scenarios: Sequence[Mapping[str, Any]]) -> str:
lines = [
f"## {HEADING}",
"",
(
"Each scenario is generated from the operating model and traced through "
"it by code."
),
"",
"| Scenario | Result | Path |",
"| --- | --- | --- |",
]
lines += [
f"| {item['scenario']} | {'pass' if item['passes'] else '**fail**'} | {item['path']} |"
for item in scenarios
]
return "\n".join(lines) + "\n"
def check_verdicts(
attachments: Mapping[str, str],
graph: Mapping[str, Sequence[Mapping[str, Any]]] | None,
name: str,
) -> list[str]:
"""What is wrong with the validator's verdicts: one on every Check, none on a
scenario, and never a pass on a Check that depends on a finding."""
try:
data = json.loads(attachments[name])
except ValueError as exc:
return [f"{name}: not valid JSON ({exc})"]
if not isinstance(data, dict) or not isinstance(data.get("checks"), list):
return [f"{name}: must be an object with a list of checks"]
problems = []
given: dict[str, str] = {}
for index, item in enumerate(data["checks"]):
where = f"{name}: checks[{index}]"
if not isinstance(item, dict) or not item.get("check"):
problems.append(f"{where} needs a check")
continue
identifier = str(item["check"])
if identifier.startswith(PREFIX):
continue # the walk-throughs' own, written by code on an earlier attempt
if item.get("verdict") == WAITING:
# Recorded by code on an earlier attempt; the validator's own verdict
# on such a Check is not_verifiable.
item = {**item, "verdict": "not_verifiable"}
if item.get("verdict") not in ("pass", "fail", "not_verifiable"):
problems.append(
f"{where} ({identifier}): verdict must be pass, fail or not_verifiable"
)
if not str(item.get("evidence") or "").strip():
problems.append(f"{where} ({identifier}) needs evidence")
given[identifier] = str(item.get("verdict"))
for check in (graph or {}).get("checks", []):
identifier = str(check["id"])
if identifier not in given:
problems.append(f"{name}: Check {identifier} has no verdict")
elif check.get("depends_on_finding") and given[identifier] == "pass":
problems.append(
f"{name}: Check {identifier} depends on finding "
f"{check['depends_on_finding']}. It is not_verifiable while the finding "
"stands, or fail when the operating model could meet it itself"
)
known = {str(check["id"]) for check in (graph or {}).get("checks", [])}
if graph is not None:
problems += [
f"{name}: {identifier} is not a Check of the Target operating model"
for identifier in given
if identifier not in known
]
return problems