Files
tom_analysis_agent/app/operating_model/digest.py

414 lines
15 KiB
Python

"""Digesting a solution architecture into an operating spec, in code.
The solution architecture agent delivers its design as data files. This reads the
ones an operating model depends on and writes the operating spec's fields from
them: nothing is summarised and nothing is guessed, so what is digested can be
counted against what was delivered.
components.json, products.json each component becomes an Item to be operated,
target-architecture.json with the Elements that realise it
requirements.json, what each says can only be shown in operation
recommendations.json (`outside_code`) becomes a Condition
verdicts.json a Check that failed or could not be verified
becomes an inherited risk
scope.json what the customer stated of operations, and of
the solution, is carried across as stated
work-breakdown.json, how long the build takes, and what the customer
execution.json owns of it
An Item is a component, not a resource: an organisation operates a hybrid
connectivity gateway, not each of its six routers and attachments. A Condition is
what the architecture itself says infrastructure code cannot verify: an
availability measured in operation, a recovery rehearsed. The settings its Checks
verified at build are kept true by configuration management, and are not Conditions.
Only what the customer stated in the architecture's scope is carried across as
stated. What that agent assumed for them is left to be asked again.
"""
from __future__ import annotations
import json
import math
import re
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
import yaml
from execution_engine.workflows.workflow import Digested
GRAPH = "target-architecture.json"
COMPONENTS = "components.json"
PRODUCTS = "products.json"
REQUIREMENTS = "requirements.json"
RECOMMENDATIONS = "recommendations.json"
VERDICTS = "verdicts.json"
SCOPE = "scope.json"
PLAN = "work-breakdown.json"
MANIFEST = "execution.json"
# Every file the digest reads. A folder is read for these and nothing else.
FILES = (
MANIFEST,
SCOPE,
REQUIREMENTS,
RECOMMENDATIONS,
COMPONENTS,
PRODUCTS,
GRAPH,
VERDICTS,
PLAN,
)
DEFAULT_KIND = "service"
SOURCE_PREFIX = "architecture:"
_HOURS = {"business_hours": "business", "extended": "extended", "always": "24x7"}
_NUMBER = re.compile(r"\d+(?:\.\d+)?")
def _load(sources: Mapping[str, str], name: str) -> Any:
if name not in sources:
return None
try:
return json.loads(sources[name])
except ValueError as exc:
raise ValueError(f"{name}: not valid JSON ({exc})") from exc
def _records(data: Any) -> list[dict[str, Any]]:
return (
[item for item in data if isinstance(item, dict)]
if isinstance(data, list)
else []
)
def _within(document: Any, path: str) -> Any:
for part in path.split("."):
if not isinstance(document, Mapping) or part not in document:
return None
document = document[part]
return document
def kind_of(product: str, item_kinds: Mapping[str, Sequence[str]]) -> str:
"""What kind of thing a product is, by the words the provider bundle lists."""
lowered = product.lower()
for kind, words in item_kinds.items():
if any(str(word).lower() in lowered for word in words):
return kind
return DEFAULT_KIND
def item_kinds(root: Path, provider: str) -> dict[str, list[str]]:
"""The provider bundle's table of what kind of thing each product is."""
path = root / "providers" / f"{provider}.yaml"
data = yaml.safe_load(path.read_text(encoding="utf-8")) if path.is_file() else {}
kinds = ((data or {}).get("operating") or {}).get("item_kinds") or {}
return {
str(kind): [str(word) for word in words or []] for kind, words in kinds.items()
}
def architecture(
sources: Mapping[str, str], kinds: Mapping[str, Sequence[str]] | None = None
) -> Digested:
"""The operating spec's fields, read from a solution architecture's data files.
Returns nothing digested when the sources are not a solution architecture.
Raises ValueError, saying what is wrong, when they are one that cannot be used.
"""
if GRAPH not in sources:
return Digested({})
graph = _load(sources, GRAPH)
if not isinstance(graph, dict) or not isinstance(graph.get("elements"), list):
# A ValueError is what the engine turns into a refused start.
raise ValueError( # noqa: TRY004
f"{GRAPH}: must be an object with a list of elements"
)
manifest = _load(sources, MANIFEST) or {}
if manifest and manifest.get("status", "completed") != "completed":
raise ValueError(
f"{MANIFEST}: the Execution did not complete ({manifest.get('status')}); "
"an operating model is designed for an architecture that was approved"
)
elements = _records(graph["elements"])
components = _records(_load(sources, COMPONENTS))
product_of = {
str(item.get("component")): str(item.get("product"))
for item in _records(_load(sources, PRODUCTS))
}
kinds = kinds or {}
notes: dict[str, str] = {}
# --- Items: one a component, with the Elements that realise it -------------------
realised: dict[str, list[dict[str, Any]]] = {}
for element in elements:
realised.setdefault(str(element.get("component") or ""), []).append(element)
if not components:
components = [
{"id": identifier, "name": identifier, "requirements": []}
for identifier in realised
if identifier
]
items: list[dict[str, Any]] = []
item_of: dict[str, str] = {}
for component in components:
component_id = str(component.get("id"))
own = realised.get(component_id, [])
products = [str(e.get("product")) for e in own if e.get("product")]
product = product_of.get(component_id) or (
max(set(products), key=products.count) if products else ""
)
item_id = f"S-{len(items) + 1}"
item_of[component_id] = item_id
items.append(
{
"id": item_id,
"name": str(component.get("name") or component_id),
"kind": kind_of(product, kinds),
**({"product": product} if product else {}),
"source_ref": component_id,
"elements": [str(e.get("id")) for e in own],
}
)
loose = realised.get("", [])
if loose:
# Elements the architecture filed under no component are still operated.
item_id = f"S-{len(items) + 1}"
item_of[""] = item_id
items.append(
{
"id": item_id,
"name": "Resources outside any component",
"kind": DEFAULT_KIND,
"elements": [str(e.get("id")) for e in loose],
}
)
if not items:
raise ValueError(f"{GRAPH}: lists no Elements to operate")
placed = sum(len(item["elements"]) for item in items)
if placed != len(elements):
unplaced = sorted(
str(e.get("id"))
for e in elements
if str(e.get("component") or "") not in item_of
)
raise ValueError(
f"{GRAPH}: these Elements name a component that {COMPONENTS} does not "
f"have: {', '.join(unplaced)}"
)
notes["solution.items"] = (
f"{len(items)} components of the solution architecture, standing for "
f"{len(elements)} Elements"
)
# --- Conditions: what can only be shown to hold in operation --------------------
serving: dict[str, list[str]] = {}
for component in components:
for requirement in component.get("requirements") or []:
serving.setdefault(str(requirement), []).append(str(component.get("id")))
conditions: list[dict[str, Any]] = []
stated = 0
for name in (REQUIREMENTS, RECOMMENDATIONS):
for entry in _records(_load(sources, name)):
if entry.get("change") == "removed":
continue
entry_id = str(entry.get("id"))
behind = [str(c) for c in entry.get("components") or []] or serving.get(
entry_id, []
)
on = [item_of[c] for c in behind if c in item_of] or [
str(i["id"]) for i in items
]
for statement in entry.get("outside_code") or []:
stated += 1
conditions.append(
{
"id": f"K-{len(conditions) + 1}",
"on": on,
"condition": " ".join(str(statement).split()),
"source_ref": entry_id,
}
)
if len(conditions) != stated:
raise ValueError(
"a condition the architecture leaves to operation was not placed"
)
if conditions:
notes["solution.conditions"] = (
f"{len(conditions)} conditions the solution architecture says cannot be "
"verified from infrastructure code"
)
# --- Inherited risks: Checks that did not pass ------------------------------------
element_item = {
str(e.get("id")): item_of.get(str(e.get("component") or ""), items[0]["id"])
for e in elements
}
check_on = {
str(check.get("id")): element_item.get(
str(check.get("applies_to")), items[0]["id"]
)
for check in _records(graph.get("checks"))
}
statement_of = {
str(check.get("id")): str(check.get("statement") or "")
for check in _records(graph.get("checks"))
}
verdicts = _load(sources, VERDICTS) or {}
risks = [
{
"on": check_on.get(str(item.get("check")), items[0]["id"]),
"risk": (
f"Check {item.get('check')} "
+ (
"failed"
if item.get("verdict") == "fail"
else "could not be verified"
)
+ f" in validation: {statement_of.get(str(item.get('check')), '')} "
f"({' '.join(str(item.get('evidence') or '').split())})"
).strip(),
"source_ref": str(item.get("check")),
}
for item in _records(
verdicts.get("checks") if isinstance(verdicts, dict) else None
)
if item.get("verdict") in ("fail", "not_verifiable")
]
# --- What the customer stated, carried across as stated --------------------------
scope = _load(sources, SCOPE) or {}
said = scope.get("stated") if isinstance(scope, dict) else {}
spec: dict[str, Any] = {
"solution": {
"items": items,
**({"conditions": conditions} if conditions else {}),
}
}
def put(path: str, value: Any, note: str = "") -> None:
if value in (None, "", [], {}):
return
node = spec
*parents, last = path.split(".")
for part in parents:
node = node.setdefault(part, {})
node[last] = value
if note:
notes[path] = note
if risks:
put("solution.inherited_risks", risks, f"{VERDICTS}: Checks that did not pass")
execution_id = manifest.get("execution_id")
put("metadata.customer_case", manifest.get("customer_case"))
put(
"metadata.source",
f"{SOURCE_PREFIX}{execution_id}"
if execution_id
else f"{SOURCE_PREFIX}unverified",
"the Execution these files are of"
if execution_id
else f"no {MANIFEST} came with the files, so they are not known to be one "
"completed Execution",
)
put("engagement.kind", scope.get("kind") if isinstance(scope, dict) else None)
put(
"intent.statement",
manifest.get("workflow_request") or _within(said, "intent.statement"),
"the request the solution architecture was designed for",
)
criteria = _within(said, "intent.success_criteria")
put(
"intent.business_outcomes",
[
{"id": f"BO-{n}", "outcome": " ".join(str(text).split())}
for n, text in enumerate(criteria or [], start=1)
if str(text).strip()
]
if isinstance(criteria, list)
else None,
f"{SCOPE}: the success criteria the customer stated",
)
put("solution.cloud_provider", _within(said, "estate.cloud_provider"))
hours = _within(said, "operations.support_hours")
put("operations.support_hours", _HOURS.get(str(hours)), f"{SCOPE}: {hours}")
available = _within(said, "workload.availability.hours")
put(
"operations.availability_hours",
_HOURS.get(str(available)),
f"{SCOPE}: {available}",
)
put("operations.change_process", _within(said, "operations.change_process"))
put("operations.monitoring", _within(said, "operations.monitoring"))
put("compliance.frameworks", _within(said, "compliance.frameworks"))
recovery = _NUMBER.search(
str(_within(said, "workload.availability.recovery_time") or "")
)
put(
"service_levels.recovery_time_hours",
float(recovery.group()) if recovery else None,
)
currency = _within(said, "cost.currency")
put("commercial.currency", currency if isinstance(currency, str) else None)
plan = manifest.get("delivery_plan") if isinstance(manifest, dict) else None
days = (plan or {}).get("working_days")
if isinstance(days, (int, float)) and not isinstance(days, bool):
put(
"solution.build.duration_days",
math.ceil(days),
"the build's computed length, in working days",
)
activities = _records((_load(sources, PLAN) or {}).get("activities"))
put(
"solution.build.customer_owned",
[str(a.get("name")) for a in activities if a.get("owner") == "customer"],
f"{PLAN}: the activities the customer owns",
)
return Digested(
spec=spec, used=tuple(n for n in FILES if n in sources), notes=notes
)
def read_folder(folder: Path) -> dict[str, str]:
"""The data files of an architecture Execution's folder that the digest reads."""
return {
name: (folder / name).read_text(encoding="utf-8")
for name in FILES
if (folder / name).is_file()
}
def main(arguments: Sequence[str]) -> int:
"""`python -m app.operating_model.digest <folder> [provider]`: print the operating
spec digested from an architecture Execution's folder, and what is left to settle."""
from app.operating_model.workflow import ROOT
if not arguments:
print(main.__doc__, file=sys.stderr)
return 2
folder = Path(arguments[0])
provider = arguments[1] if len(arguments) > 1 else "gcp"
sources = read_folder(folder)
try:
digested = architecture(sources, item_kinds(ROOT, provider))
except ValueError as exc:
print(f"Cannot digest {folder}: {exc}", file=sys.stderr)
return 1
if not digested.spec:
print(
f"{folder} holds no {GRAPH}: it is not a solution architecture.",
file=sys.stderr,
)
return 1
print(yaml.safe_dump(dict(digested.spec), sort_keys=False, allow_unicode=True))
for path, note in digested.notes.items():
print(f"# {path}: {note}", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))