414 lines
15 KiB
Python
414 lines
15 KiB
Python
"""Digesting a solution architecture into an operating spec, in code.
|
|
|
|
The solution architecture agent delivers its design as data files. This reads the
|
|
ones an operating model depends on and writes the operating spec's fields from
|
|
them: nothing is summarised and nothing is guessed, so what is digested can be
|
|
counted against what was delivered.
|
|
|
|
components.json, products.json each component becomes an Item to be operated,
|
|
target-architecture.json with the Elements that realise it
|
|
requirements.json, what each says can only be shown in operation
|
|
recommendations.json (`outside_code`) becomes a Condition
|
|
verdicts.json a Check that failed or could not be verified
|
|
becomes an inherited risk
|
|
scope.json what the customer stated of operations, and of
|
|
the solution, is carried across as stated
|
|
work-breakdown.json, how long the build takes, and what the customer
|
|
execution.json owns of it
|
|
|
|
An Item is a component, not a resource: an organisation operates a hybrid
|
|
connectivity gateway, not each of its six routers and attachments. A Condition is
|
|
what the architecture itself says infrastructure code cannot verify: an
|
|
availability measured in operation, a recovery rehearsed. The settings its Checks
|
|
verified at build are kept true by configuration management, and are not Conditions.
|
|
|
|
Only what the customer stated in the architecture's scope is carried across as
|
|
stated. What that agent assumed for them is left to be asked again.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
from collections.abc import Mapping, Sequence
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
from execution_engine.workflows.workflow import Digested
|
|
|
|
GRAPH = "target-architecture.json"
|
|
COMPONENTS = "components.json"
|
|
PRODUCTS = "products.json"
|
|
REQUIREMENTS = "requirements.json"
|
|
RECOMMENDATIONS = "recommendations.json"
|
|
VERDICTS = "verdicts.json"
|
|
SCOPE = "scope.json"
|
|
PLAN = "work-breakdown.json"
|
|
MANIFEST = "execution.json"
|
|
# Every file the digest reads. A folder is read for these and nothing else.
|
|
FILES = (
|
|
MANIFEST,
|
|
SCOPE,
|
|
REQUIREMENTS,
|
|
RECOMMENDATIONS,
|
|
COMPONENTS,
|
|
PRODUCTS,
|
|
GRAPH,
|
|
VERDICTS,
|
|
PLAN,
|
|
)
|
|
DEFAULT_KIND = "service"
|
|
SOURCE_PREFIX = "architecture:"
|
|
_HOURS = {"business_hours": "business", "extended": "extended", "always": "24x7"}
|
|
_NUMBER = re.compile(r"\d+(?:\.\d+)?")
|
|
|
|
|
|
def _load(sources: Mapping[str, str], name: str) -> Any:
|
|
if name not in sources:
|
|
return None
|
|
try:
|
|
return json.loads(sources[name])
|
|
except ValueError as exc:
|
|
raise ValueError(f"{name}: not valid JSON ({exc})") from exc
|
|
|
|
|
|
def _records(data: Any) -> list[dict[str, Any]]:
|
|
return (
|
|
[item for item in data if isinstance(item, dict)]
|
|
if isinstance(data, list)
|
|
else []
|
|
)
|
|
|
|
|
|
def _within(document: Any, path: str) -> Any:
|
|
for part in path.split("."):
|
|
if not isinstance(document, Mapping) or part not in document:
|
|
return None
|
|
document = document[part]
|
|
return document
|
|
|
|
|
|
def kind_of(product: str, item_kinds: Mapping[str, Sequence[str]]) -> str:
|
|
"""What kind of thing a product is, by the words the provider bundle lists."""
|
|
lowered = product.lower()
|
|
for kind, words in item_kinds.items():
|
|
if any(str(word).lower() in lowered for word in words):
|
|
return kind
|
|
return DEFAULT_KIND
|
|
|
|
|
|
def item_kinds(root: Path, provider: str) -> dict[str, list[str]]:
|
|
"""The provider bundle's table of what kind of thing each product is."""
|
|
path = root / "providers" / f"{provider}.yaml"
|
|
data = yaml.safe_load(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
kinds = ((data or {}).get("operating") or {}).get("item_kinds") or {}
|
|
return {
|
|
str(kind): [str(word) for word in words or []] for kind, words in kinds.items()
|
|
}
|
|
|
|
|
|
def architecture(
|
|
sources: Mapping[str, str], kinds: Mapping[str, Sequence[str]] | None = None
|
|
) -> Digested:
|
|
"""The operating spec's fields, read from a solution architecture's data files.
|
|
|
|
Returns nothing digested when the sources are not a solution architecture.
|
|
Raises ValueError, saying what is wrong, when they are one that cannot be used.
|
|
"""
|
|
if GRAPH not in sources:
|
|
return Digested({})
|
|
graph = _load(sources, GRAPH)
|
|
if not isinstance(graph, dict) or not isinstance(graph.get("elements"), list):
|
|
# A ValueError is what the engine turns into a refused start.
|
|
raise ValueError( # noqa: TRY004
|
|
f"{GRAPH}: must be an object with a list of elements"
|
|
)
|
|
manifest = _load(sources, MANIFEST) or {}
|
|
if manifest and manifest.get("status", "completed") != "completed":
|
|
raise ValueError(
|
|
f"{MANIFEST}: the Execution did not complete ({manifest.get('status')}); "
|
|
"an operating model is designed for an architecture that was approved"
|
|
)
|
|
elements = _records(graph["elements"])
|
|
components = _records(_load(sources, COMPONENTS))
|
|
product_of = {
|
|
str(item.get("component")): str(item.get("product"))
|
|
for item in _records(_load(sources, PRODUCTS))
|
|
}
|
|
kinds = kinds or {}
|
|
notes: dict[str, str] = {}
|
|
|
|
# --- Items: one a component, with the Elements that realise it -------------------
|
|
realised: dict[str, list[dict[str, Any]]] = {}
|
|
for element in elements:
|
|
realised.setdefault(str(element.get("component") or ""), []).append(element)
|
|
if not components:
|
|
components = [
|
|
{"id": identifier, "name": identifier, "requirements": []}
|
|
for identifier in realised
|
|
if identifier
|
|
]
|
|
items: list[dict[str, Any]] = []
|
|
item_of: dict[str, str] = {}
|
|
for component in components:
|
|
component_id = str(component.get("id"))
|
|
own = realised.get(component_id, [])
|
|
products = [str(e.get("product")) for e in own if e.get("product")]
|
|
product = product_of.get(component_id) or (
|
|
max(set(products), key=products.count) if products else ""
|
|
)
|
|
item_id = f"S-{len(items) + 1}"
|
|
item_of[component_id] = item_id
|
|
items.append(
|
|
{
|
|
"id": item_id,
|
|
"name": str(component.get("name") or component_id),
|
|
"kind": kind_of(product, kinds),
|
|
**({"product": product} if product else {}),
|
|
"source_ref": component_id,
|
|
"elements": [str(e.get("id")) for e in own],
|
|
}
|
|
)
|
|
loose = realised.get("", [])
|
|
if loose:
|
|
# Elements the architecture filed under no component are still operated.
|
|
item_id = f"S-{len(items) + 1}"
|
|
item_of[""] = item_id
|
|
items.append(
|
|
{
|
|
"id": item_id,
|
|
"name": "Resources outside any component",
|
|
"kind": DEFAULT_KIND,
|
|
"elements": [str(e.get("id")) for e in loose],
|
|
}
|
|
)
|
|
if not items:
|
|
raise ValueError(f"{GRAPH}: lists no Elements to operate")
|
|
placed = sum(len(item["elements"]) for item in items)
|
|
if placed != len(elements):
|
|
unplaced = sorted(
|
|
str(e.get("id"))
|
|
for e in elements
|
|
if str(e.get("component") or "") not in item_of
|
|
)
|
|
raise ValueError(
|
|
f"{GRAPH}: these Elements name a component that {COMPONENTS} does not "
|
|
f"have: {', '.join(unplaced)}"
|
|
)
|
|
notes["solution.items"] = (
|
|
f"{len(items)} components of the solution architecture, standing for "
|
|
f"{len(elements)} Elements"
|
|
)
|
|
|
|
# --- Conditions: what can only be shown to hold in operation --------------------
|
|
serving: dict[str, list[str]] = {}
|
|
for component in components:
|
|
for requirement in component.get("requirements") or []:
|
|
serving.setdefault(str(requirement), []).append(str(component.get("id")))
|
|
conditions: list[dict[str, Any]] = []
|
|
stated = 0
|
|
for name in (REQUIREMENTS, RECOMMENDATIONS):
|
|
for entry in _records(_load(sources, name)):
|
|
if entry.get("change") == "removed":
|
|
continue
|
|
entry_id = str(entry.get("id"))
|
|
behind = [str(c) for c in entry.get("components") or []] or serving.get(
|
|
entry_id, []
|
|
)
|
|
on = [item_of[c] for c in behind if c in item_of] or [
|
|
str(i["id"]) for i in items
|
|
]
|
|
for statement in entry.get("outside_code") or []:
|
|
stated += 1
|
|
conditions.append(
|
|
{
|
|
"id": f"K-{len(conditions) + 1}",
|
|
"on": on,
|
|
"condition": " ".join(str(statement).split()),
|
|
"source_ref": entry_id,
|
|
}
|
|
)
|
|
if len(conditions) != stated:
|
|
raise ValueError(
|
|
"a condition the architecture leaves to operation was not placed"
|
|
)
|
|
if conditions:
|
|
notes["solution.conditions"] = (
|
|
f"{len(conditions)} conditions the solution architecture says cannot be "
|
|
"verified from infrastructure code"
|
|
)
|
|
|
|
# --- Inherited risks: Checks that did not pass ------------------------------------
|
|
element_item = {
|
|
str(e.get("id")): item_of.get(str(e.get("component") or ""), items[0]["id"])
|
|
for e in elements
|
|
}
|
|
check_on = {
|
|
str(check.get("id")): element_item.get(
|
|
str(check.get("applies_to")), items[0]["id"]
|
|
)
|
|
for check in _records(graph.get("checks"))
|
|
}
|
|
statement_of = {
|
|
str(check.get("id")): str(check.get("statement") or "")
|
|
for check in _records(graph.get("checks"))
|
|
}
|
|
verdicts = _load(sources, VERDICTS) or {}
|
|
risks = [
|
|
{
|
|
"on": check_on.get(str(item.get("check")), items[0]["id"]),
|
|
"risk": (
|
|
f"Check {item.get('check')} "
|
|
+ (
|
|
"failed"
|
|
if item.get("verdict") == "fail"
|
|
else "could not be verified"
|
|
)
|
|
+ f" in validation: {statement_of.get(str(item.get('check')), '')} "
|
|
f"({' '.join(str(item.get('evidence') or '').split())})"
|
|
).strip(),
|
|
"source_ref": str(item.get("check")),
|
|
}
|
|
for item in _records(
|
|
verdicts.get("checks") if isinstance(verdicts, dict) else None
|
|
)
|
|
if item.get("verdict") in ("fail", "not_verifiable")
|
|
]
|
|
|
|
# --- What the customer stated, carried across as stated --------------------------
|
|
scope = _load(sources, SCOPE) or {}
|
|
said = scope.get("stated") if isinstance(scope, dict) else {}
|
|
spec: dict[str, Any] = {
|
|
"solution": {
|
|
"items": items,
|
|
**({"conditions": conditions} if conditions else {}),
|
|
}
|
|
}
|
|
|
|
def put(path: str, value: Any, note: str = "") -> None:
|
|
if value in (None, "", [], {}):
|
|
return
|
|
node = spec
|
|
*parents, last = path.split(".")
|
|
for part in parents:
|
|
node = node.setdefault(part, {})
|
|
node[last] = value
|
|
if note:
|
|
notes[path] = note
|
|
|
|
if risks:
|
|
put("solution.inherited_risks", risks, f"{VERDICTS}: Checks that did not pass")
|
|
execution_id = manifest.get("execution_id")
|
|
put("metadata.customer_case", manifest.get("customer_case"))
|
|
put(
|
|
"metadata.source",
|
|
f"{SOURCE_PREFIX}{execution_id}"
|
|
if execution_id
|
|
else f"{SOURCE_PREFIX}unverified",
|
|
"the Execution these files are of"
|
|
if execution_id
|
|
else f"no {MANIFEST} came with the files, so they are not known to be one "
|
|
"completed Execution",
|
|
)
|
|
put("engagement.kind", scope.get("kind") if isinstance(scope, dict) else None)
|
|
put(
|
|
"intent.statement",
|
|
manifest.get("workflow_request") or _within(said, "intent.statement"),
|
|
"the request the solution architecture was designed for",
|
|
)
|
|
criteria = _within(said, "intent.success_criteria")
|
|
put(
|
|
"intent.business_outcomes",
|
|
[
|
|
{"id": f"BO-{n}", "outcome": " ".join(str(text).split())}
|
|
for n, text in enumerate(criteria or [], start=1)
|
|
if str(text).strip()
|
|
]
|
|
if isinstance(criteria, list)
|
|
else None,
|
|
f"{SCOPE}: the success criteria the customer stated",
|
|
)
|
|
put("solution.cloud_provider", _within(said, "estate.cloud_provider"))
|
|
hours = _within(said, "operations.support_hours")
|
|
put("operations.support_hours", _HOURS.get(str(hours)), f"{SCOPE}: {hours}")
|
|
available = _within(said, "workload.availability.hours")
|
|
put(
|
|
"operations.availability_hours",
|
|
_HOURS.get(str(available)),
|
|
f"{SCOPE}: {available}",
|
|
)
|
|
put("operations.change_process", _within(said, "operations.change_process"))
|
|
put("operations.monitoring", _within(said, "operations.monitoring"))
|
|
put("compliance.frameworks", _within(said, "compliance.frameworks"))
|
|
recovery = _NUMBER.search(
|
|
str(_within(said, "workload.availability.recovery_time") or "")
|
|
)
|
|
put(
|
|
"service_levels.recovery_time_hours",
|
|
float(recovery.group()) if recovery else None,
|
|
)
|
|
currency = _within(said, "cost.currency")
|
|
put("commercial.currency", currency if isinstance(currency, str) else None)
|
|
|
|
plan = manifest.get("delivery_plan") if isinstance(manifest, dict) else None
|
|
days = (plan or {}).get("working_days")
|
|
if isinstance(days, (int, float)) and not isinstance(days, bool):
|
|
put(
|
|
"solution.build.duration_days",
|
|
math.ceil(days),
|
|
"the build's computed length, in working days",
|
|
)
|
|
activities = _records((_load(sources, PLAN) or {}).get("activities"))
|
|
put(
|
|
"solution.build.customer_owned",
|
|
[str(a.get("name")) for a in activities if a.get("owner") == "customer"],
|
|
f"{PLAN}: the activities the customer owns",
|
|
)
|
|
return Digested(
|
|
spec=spec, used=tuple(n for n in FILES if n in sources), notes=notes
|
|
)
|
|
|
|
|
|
def read_folder(folder: Path) -> dict[str, str]:
|
|
"""The data files of an architecture Execution's folder that the digest reads."""
|
|
return {
|
|
name: (folder / name).read_text(encoding="utf-8")
|
|
for name in FILES
|
|
if (folder / name).is_file()
|
|
}
|
|
|
|
|
|
def main(arguments: Sequence[str]) -> int:
|
|
"""`python -m app.operating_model.digest <folder> [provider]`: print the operating
|
|
spec digested from an architecture Execution's folder, and what is left to settle."""
|
|
from app.operating_model.workflow import ROOT
|
|
|
|
if not arguments:
|
|
print(main.__doc__, file=sys.stderr)
|
|
return 2
|
|
folder = Path(arguments[0])
|
|
provider = arguments[1] if len(arguments) > 1 else "gcp"
|
|
sources = read_folder(folder)
|
|
try:
|
|
digested = architecture(sources, item_kinds(ROOT, provider))
|
|
except ValueError as exc:
|
|
print(f"Cannot digest {folder}: {exc}", file=sys.stderr)
|
|
return 1
|
|
if not digested.spec:
|
|
print(
|
|
f"{folder} holds no {GRAPH}: it is not a solution architecture.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
print(yaml.safe_dump(dict(digested.spec), sort_keys=False, allow_unicode=True))
|
|
for path, note in digested.notes.items():
|
|
print(f"# {path}: {note}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|