feat: working TOM agent
This commit is contained in:
413
app/operating_model/digest.py
Normal file
413
app/operating_model/digest.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""Digesting a solution architecture into an operating spec, in code.
|
||||
|
||||
The solution architecture agent delivers its design as data files. This reads the
|
||||
ones an operating model depends on and writes the operating spec's fields from
|
||||
them: nothing is summarised and nothing is guessed, so what is digested can be
|
||||
counted against what was delivered.
|
||||
|
||||
components.json, products.json each component becomes an Item to be operated,
|
||||
target-architecture.json with the Elements that realise it
|
||||
requirements.json, what each says can only be shown in operation
|
||||
recommendations.json (`outside_code`) becomes a Condition
|
||||
verdicts.json a Check that failed or could not be verified
|
||||
becomes an inherited risk
|
||||
scope.json what the customer stated of operations, and of
|
||||
the solution, is carried across as stated
|
||||
work-breakdown.json, how long the build takes, and what the customer
|
||||
execution.json owns of it
|
||||
|
||||
An Item is a component, not a resource: an organisation operates a hybrid
|
||||
connectivity gateway, not each of its six routers and attachments. A Condition is
|
||||
what the architecture itself says infrastructure code cannot verify: an
|
||||
availability measured in operation, a recovery rehearsed. The settings its Checks
|
||||
verified at build are kept true by configuration management, and are not Conditions.
|
||||
|
||||
Only what the customer stated in the architecture's scope is carried across as
|
||||
stated. What that agent assumed for them is left to be asked again.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from execution_engine.workflows.workflow import Digested
|
||||
|
||||
GRAPH = "target-architecture.json"
|
||||
COMPONENTS = "components.json"
|
||||
PRODUCTS = "products.json"
|
||||
REQUIREMENTS = "requirements.json"
|
||||
RECOMMENDATIONS = "recommendations.json"
|
||||
VERDICTS = "verdicts.json"
|
||||
SCOPE = "scope.json"
|
||||
PLAN = "work-breakdown.json"
|
||||
MANIFEST = "execution.json"
|
||||
# Every file the digest reads. A folder is read for these and nothing else.
|
||||
FILES = (
|
||||
MANIFEST,
|
||||
SCOPE,
|
||||
REQUIREMENTS,
|
||||
RECOMMENDATIONS,
|
||||
COMPONENTS,
|
||||
PRODUCTS,
|
||||
GRAPH,
|
||||
VERDICTS,
|
||||
PLAN,
|
||||
)
|
||||
DEFAULT_KIND = "service"
|
||||
SOURCE_PREFIX = "architecture:"
|
||||
_HOURS = {"business_hours": "business", "extended": "extended", "always": "24x7"}
|
||||
_NUMBER = re.compile(r"\d+(?:\.\d+)?")
|
||||
|
||||
|
||||
def _load(sources: Mapping[str, str], name: str) -> Any:
|
||||
if name not in sources:
|
||||
return None
|
||||
try:
|
||||
return json.loads(sources[name])
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{name}: not valid JSON ({exc})") from exc
|
||||
|
||||
|
||||
def _records(data: Any) -> list[dict[str, Any]]:
|
||||
return (
|
||||
[item for item in data if isinstance(item, dict)]
|
||||
if isinstance(data, list)
|
||||
else []
|
||||
)
|
||||
|
||||
|
||||
def _within(document: Any, path: str) -> Any:
|
||||
for part in path.split("."):
|
||||
if not isinstance(document, Mapping) or part not in document:
|
||||
return None
|
||||
document = document[part]
|
||||
return document
|
||||
|
||||
|
||||
def kind_of(product: str, item_kinds: Mapping[str, Sequence[str]]) -> str:
|
||||
"""What kind of thing a product is, by the words the provider bundle lists."""
|
||||
lowered = product.lower()
|
||||
for kind, words in item_kinds.items():
|
||||
if any(str(word).lower() in lowered for word in words):
|
||||
return kind
|
||||
return DEFAULT_KIND
|
||||
|
||||
|
||||
def item_kinds(root: Path, provider: str) -> dict[str, list[str]]:
|
||||
"""The provider bundle's table of what kind of thing each product is."""
|
||||
path = root / "providers" / f"{provider}.yaml"
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
||||
kinds = ((data or {}).get("operating") or {}).get("item_kinds") or {}
|
||||
return {
|
||||
str(kind): [str(word) for word in words or []] for kind, words in kinds.items()
|
||||
}
|
||||
|
||||
|
||||
def architecture(
|
||||
sources: Mapping[str, str], kinds: Mapping[str, Sequence[str]] | None = None
|
||||
) -> Digested:
|
||||
"""The operating spec's fields, read from a solution architecture's data files.
|
||||
|
||||
Returns nothing digested when the sources are not a solution architecture.
|
||||
Raises ValueError, saying what is wrong, when they are one that cannot be used.
|
||||
"""
|
||||
if GRAPH not in sources:
|
||||
return Digested({})
|
||||
graph = _load(sources, GRAPH)
|
||||
if not isinstance(graph, dict) or not isinstance(graph.get("elements"), list):
|
||||
# A ValueError is what the engine turns into a refused start.
|
||||
raise ValueError( # noqa: TRY004
|
||||
f"{GRAPH}: must be an object with a list of elements"
|
||||
)
|
||||
manifest = _load(sources, MANIFEST) or {}
|
||||
if manifest and manifest.get("status", "completed") != "completed":
|
||||
raise ValueError(
|
||||
f"{MANIFEST}: the Execution did not complete ({manifest.get('status')}); "
|
||||
"an operating model is designed for an architecture that was approved"
|
||||
)
|
||||
elements = _records(graph["elements"])
|
||||
components = _records(_load(sources, COMPONENTS))
|
||||
product_of = {
|
||||
str(item.get("component")): str(item.get("product"))
|
||||
for item in _records(_load(sources, PRODUCTS))
|
||||
}
|
||||
kinds = kinds or {}
|
||||
notes: dict[str, str] = {}
|
||||
|
||||
# --- Items: one a component, with the Elements that realise it -------------------
|
||||
realised: dict[str, list[dict[str, Any]]] = {}
|
||||
for element in elements:
|
||||
realised.setdefault(str(element.get("component") or ""), []).append(element)
|
||||
if not components:
|
||||
components = [
|
||||
{"id": identifier, "name": identifier, "requirements": []}
|
||||
for identifier in realised
|
||||
if identifier
|
||||
]
|
||||
items: list[dict[str, Any]] = []
|
||||
item_of: dict[str, str] = {}
|
||||
for component in components:
|
||||
component_id = str(component.get("id"))
|
||||
own = realised.get(component_id, [])
|
||||
products = [str(e.get("product")) for e in own if e.get("product")]
|
||||
product = product_of.get(component_id) or (
|
||||
max(set(products), key=products.count) if products else ""
|
||||
)
|
||||
item_id = f"S-{len(items) + 1}"
|
||||
item_of[component_id] = item_id
|
||||
items.append(
|
||||
{
|
||||
"id": item_id,
|
||||
"name": str(component.get("name") or component_id),
|
||||
"kind": kind_of(product, kinds),
|
||||
**({"product": product} if product else {}),
|
||||
"source_ref": component_id,
|
||||
"elements": [str(e.get("id")) for e in own],
|
||||
}
|
||||
)
|
||||
loose = realised.get("", [])
|
||||
if loose:
|
||||
# Elements the architecture filed under no component are still operated.
|
||||
item_id = f"S-{len(items) + 1}"
|
||||
item_of[""] = item_id
|
||||
items.append(
|
||||
{
|
||||
"id": item_id,
|
||||
"name": "Resources outside any component",
|
||||
"kind": DEFAULT_KIND,
|
||||
"elements": [str(e.get("id")) for e in loose],
|
||||
}
|
||||
)
|
||||
if not items:
|
||||
raise ValueError(f"{GRAPH}: lists no Elements to operate")
|
||||
placed = sum(len(item["elements"]) for item in items)
|
||||
if placed != len(elements):
|
||||
unplaced = sorted(
|
||||
str(e.get("id"))
|
||||
for e in elements
|
||||
if str(e.get("component") or "") not in item_of
|
||||
)
|
||||
raise ValueError(
|
||||
f"{GRAPH}: these Elements name a component that {COMPONENTS} does not "
|
||||
f"have: {', '.join(unplaced)}"
|
||||
)
|
||||
notes["solution.items"] = (
|
||||
f"{len(items)} components of the solution architecture, standing for "
|
||||
f"{len(elements)} Elements"
|
||||
)
|
||||
|
||||
# --- Conditions: what can only be shown to hold in operation --------------------
|
||||
serving: dict[str, list[str]] = {}
|
||||
for component in components:
|
||||
for requirement in component.get("requirements") or []:
|
||||
serving.setdefault(str(requirement), []).append(str(component.get("id")))
|
||||
conditions: list[dict[str, Any]] = []
|
||||
stated = 0
|
||||
for name in (REQUIREMENTS, RECOMMENDATIONS):
|
||||
for entry in _records(_load(sources, name)):
|
||||
if entry.get("change") == "removed":
|
||||
continue
|
||||
entry_id = str(entry.get("id"))
|
||||
behind = [str(c) for c in entry.get("components") or []] or serving.get(
|
||||
entry_id, []
|
||||
)
|
||||
on = [item_of[c] for c in behind if c in item_of] or [
|
||||
str(i["id"]) for i in items
|
||||
]
|
||||
for statement in entry.get("outside_code") or []:
|
||||
stated += 1
|
||||
conditions.append(
|
||||
{
|
||||
"id": f"K-{len(conditions) + 1}",
|
||||
"on": on,
|
||||
"condition": " ".join(str(statement).split()),
|
||||
"source_ref": entry_id,
|
||||
}
|
||||
)
|
||||
if len(conditions) != stated:
|
||||
raise ValueError(
|
||||
"a condition the architecture leaves to operation was not placed"
|
||||
)
|
||||
if conditions:
|
||||
notes["solution.conditions"] = (
|
||||
f"{len(conditions)} conditions the solution architecture says cannot be "
|
||||
"verified from infrastructure code"
|
||||
)
|
||||
|
||||
# --- Inherited risks: Checks that did not pass ------------------------------------
|
||||
element_item = {
|
||||
str(e.get("id")): item_of.get(str(e.get("component") or ""), items[0]["id"])
|
||||
for e in elements
|
||||
}
|
||||
check_on = {
|
||||
str(check.get("id")): element_item.get(
|
||||
str(check.get("applies_to")), items[0]["id"]
|
||||
)
|
||||
for check in _records(graph.get("checks"))
|
||||
}
|
||||
statement_of = {
|
||||
str(check.get("id")): str(check.get("statement") or "")
|
||||
for check in _records(graph.get("checks"))
|
||||
}
|
||||
verdicts = _load(sources, VERDICTS) or {}
|
||||
risks = [
|
||||
{
|
||||
"on": check_on.get(str(item.get("check")), items[0]["id"]),
|
||||
"risk": (
|
||||
f"Check {item.get('check')} "
|
||||
+ (
|
||||
"failed"
|
||||
if item.get("verdict") == "fail"
|
||||
else "could not be verified"
|
||||
)
|
||||
+ f" in validation: {statement_of.get(str(item.get('check')), '')} "
|
||||
f"({' '.join(str(item.get('evidence') or '').split())})"
|
||||
).strip(),
|
||||
"source_ref": str(item.get("check")),
|
||||
}
|
||||
for item in _records(
|
||||
verdicts.get("checks") if isinstance(verdicts, dict) else None
|
||||
)
|
||||
if item.get("verdict") in ("fail", "not_verifiable")
|
||||
]
|
||||
|
||||
# --- What the customer stated, carried across as stated --------------------------
|
||||
scope = _load(sources, SCOPE) or {}
|
||||
said = scope.get("stated") if isinstance(scope, dict) else {}
|
||||
spec: dict[str, Any] = {
|
||||
"solution": {
|
||||
"items": items,
|
||||
**({"conditions": conditions} if conditions else {}),
|
||||
}
|
||||
}
|
||||
|
||||
def put(path: str, value: Any, note: str = "") -> None:
|
||||
if value in (None, "", [], {}):
|
||||
return
|
||||
node = spec
|
||||
*parents, last = path.split(".")
|
||||
for part in parents:
|
||||
node = node.setdefault(part, {})
|
||||
node[last] = value
|
||||
if note:
|
||||
notes[path] = note
|
||||
|
||||
if risks:
|
||||
put("solution.inherited_risks", risks, f"{VERDICTS}: Checks that did not pass")
|
||||
execution_id = manifest.get("execution_id")
|
||||
put("metadata.customer_case", manifest.get("customer_case"))
|
||||
put(
|
||||
"metadata.source",
|
||||
f"{SOURCE_PREFIX}{execution_id}"
|
||||
if execution_id
|
||||
else f"{SOURCE_PREFIX}unverified",
|
||||
"the Execution these files are of"
|
||||
if execution_id
|
||||
else f"no {MANIFEST} came with the files, so they are not known to be one "
|
||||
"completed Execution",
|
||||
)
|
||||
put("engagement.kind", scope.get("kind") if isinstance(scope, dict) else None)
|
||||
put(
|
||||
"intent.statement",
|
||||
manifest.get("workflow_request") or _within(said, "intent.statement"),
|
||||
"the request the solution architecture was designed for",
|
||||
)
|
||||
criteria = _within(said, "intent.success_criteria")
|
||||
put(
|
||||
"intent.business_outcomes",
|
||||
[
|
||||
{"id": f"BO-{n}", "outcome": " ".join(str(text).split())}
|
||||
for n, text in enumerate(criteria or [], start=1)
|
||||
if str(text).strip()
|
||||
]
|
||||
if isinstance(criteria, list)
|
||||
else None,
|
||||
f"{SCOPE}: the success criteria the customer stated",
|
||||
)
|
||||
put("solution.cloud_provider", _within(said, "estate.cloud_provider"))
|
||||
hours = _within(said, "operations.support_hours")
|
||||
put("operations.support_hours", _HOURS.get(str(hours)), f"{SCOPE}: {hours}")
|
||||
available = _within(said, "workload.availability.hours")
|
||||
put(
|
||||
"operations.availability_hours",
|
||||
_HOURS.get(str(available)),
|
||||
f"{SCOPE}: {available}",
|
||||
)
|
||||
put("operations.change_process", _within(said, "operations.change_process"))
|
||||
put("operations.monitoring", _within(said, "operations.monitoring"))
|
||||
put("compliance.frameworks", _within(said, "compliance.frameworks"))
|
||||
recovery = _NUMBER.search(
|
||||
str(_within(said, "workload.availability.recovery_time") or "")
|
||||
)
|
||||
put(
|
||||
"service_levels.recovery_time_hours",
|
||||
float(recovery.group()) if recovery else None,
|
||||
)
|
||||
currency = _within(said, "cost.currency")
|
||||
put("commercial.currency", currency if isinstance(currency, str) else None)
|
||||
|
||||
plan = manifest.get("delivery_plan") if isinstance(manifest, dict) else None
|
||||
days = (plan or {}).get("working_days")
|
||||
if isinstance(days, (int, float)) and not isinstance(days, bool):
|
||||
put(
|
||||
"solution.build.duration_days",
|
||||
math.ceil(days),
|
||||
"the build's computed length, in working days",
|
||||
)
|
||||
activities = _records((_load(sources, PLAN) or {}).get("activities"))
|
||||
put(
|
||||
"solution.build.customer_owned",
|
||||
[str(a.get("name")) for a in activities if a.get("owner") == "customer"],
|
||||
f"{PLAN}: the activities the customer owns",
|
||||
)
|
||||
return Digested(
|
||||
spec=spec, used=tuple(n for n in FILES if n in sources), notes=notes
|
||||
)
|
||||
|
||||
|
||||
def read_folder(folder: Path) -> dict[str, str]:
|
||||
"""The data files of an architecture Execution's folder that the digest reads."""
|
||||
return {
|
||||
name: (folder / name).read_text(encoding="utf-8")
|
||||
for name in FILES
|
||||
if (folder / name).is_file()
|
||||
}
|
||||
|
||||
|
||||
def main(arguments: Sequence[str]) -> int:
|
||||
"""`python -m app.operating_model.digest <folder> [provider]`: print the operating
|
||||
spec digested from an architecture Execution's folder, and what is left to settle."""
|
||||
from app.operating_model.workflow import ROOT
|
||||
|
||||
if not arguments:
|
||||
print(main.__doc__, file=sys.stderr)
|
||||
return 2
|
||||
folder = Path(arguments[0])
|
||||
provider = arguments[1] if len(arguments) > 1 else "gcp"
|
||||
sources = read_folder(folder)
|
||||
try:
|
||||
digested = architecture(sources, item_kinds(ROOT, provider))
|
||||
except ValueError as exc:
|
||||
print(f"Cannot digest {folder}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not digested.spec:
|
||||
print(
|
||||
f"{folder} holds no {GRAPH}: it is not a solution architecture.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print(yaml.safe_dump(dict(digested.spec), sort_keys=False, allow_unicode=True))
|
||||
for path, note in digested.notes.items():
|
||||
print(f"# {path}: {note}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user