199 lines
6.7 KiB
Python
199 lines
6.7 KiB
Python
"""Checks applied to a finished Execution of the target operating model Workflow.
|
|
|
|
Every check here is structural: it reads the Execution's own data and asks whether
|
|
the chain holds and the result is what the Workflow promises. None judges whether
|
|
the operating model is a good one; that needs a person, or a judge calibrated
|
|
against one, and is not built.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping
|
|
from dataclasses import asdict, dataclass, field
|
|
from typing import Any
|
|
|
|
from execution_engine.workflows.workflow import Workflow
|
|
|
|
from app.operating_model import guide, scenarios, spec
|
|
from app.operating_model.names import (
|
|
GUIDE,
|
|
SCENARIOS_FILE,
|
|
STAFFING_FILE,
|
|
TARGET_OPERATING_MODEL,
|
|
VALIDATION_RESULTS,
|
|
VERDICTS_FILE,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Check:
|
|
name: str
|
|
passed: bool
|
|
detail: str = ""
|
|
|
|
|
|
@dataclass
|
|
class CaseResult:
|
|
case: str
|
|
status: str
|
|
checks: list[Check] = field(default_factory=list)
|
|
minutes: float = 0.0
|
|
revisions: dict[str, int] = field(default_factory=dict)
|
|
tokens: dict[str, dict[str, int]] = field(default_factory=dict)
|
|
figures: dict[str, Any] = field(default_factory=dict)
|
|
warnings: list[str] = field(default_factory=list)
|
|
folder: str = ""
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return all(check.passed for check in self.checks)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {**asdict(self), "passed": self.passed}
|
|
|
|
|
|
def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
|
|
text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "")
|
|
try:
|
|
return json.loads(text) if text else None
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def score(
|
|
workflow: Workflow,
|
|
state: Mapping[str, Any],
|
|
*,
|
|
status: str,
|
|
max_revisions: int = 2,
|
|
) -> tuple[list[Check], dict[str, int], dict[str, Any]]:
|
|
"""The checks on a finished Execution, its Revisions, and its key figures."""
|
|
deliverables: Mapping[str, Any] = state.get("deliverables") or {}
|
|
ledger = (state.get("ledger") or {}).get("deliverables") or {}
|
|
revisions = {
|
|
kind: int(entry.get("revisions", 0))
|
|
for kind, entry in ledger.items()
|
|
if entry.get("revisions")
|
|
}
|
|
checks = [
|
|
Check("completed", status == "completed", status),
|
|
Check(
|
|
"every Deliverable was submitted",
|
|
set(deliverables) == set(workflow.deliverables),
|
|
", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all",
|
|
),
|
|
]
|
|
|
|
# The chain, checked again on what was delivered: each Deliverable's data
|
|
# against the ones it builds on, as the engine checked it when it was submitted.
|
|
broken = []
|
|
for kind in workflow.deliverables:
|
|
if kind not in deliverables or kind not in workflow.data_checks:
|
|
continue
|
|
name, check = workflow.data_checks[kind]
|
|
attachments = deliverables[kind].get("attachments", {})
|
|
upstream = {
|
|
k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables
|
|
}
|
|
broken += (
|
|
[f"{kind}: {fault}" for fault in check(attachments, upstream)]
|
|
if name in attachments
|
|
else [f"{kind}: {name} is missing"]
|
|
)
|
|
checks.append(Check("the chain holds", not broken, "; ".join(broken[:5])))
|
|
|
|
verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {}
|
|
given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)]
|
|
own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)]
|
|
failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"]
|
|
checks.append(Check("no Check failed", not failed, ", ".join(failed)))
|
|
walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or []
|
|
checks.append(
|
|
Check(
|
|
"every scenario has a path",
|
|
bool(walked) and all(item.get("passes") for item in walked),
|
|
f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}",
|
|
)
|
|
)
|
|
|
|
content = str((deliverables.get(GUIDE) or {}).get("content", ""))
|
|
headings = [
|
|
line[3:].strip() for line in content.splitlines() if line.startswith("## ")
|
|
]
|
|
checks.append(
|
|
Check(
|
|
"the guide has its sections",
|
|
headings == list(guide.SECTIONS),
|
|
", ".join(headings),
|
|
)
|
|
)
|
|
summary = content.split(f"## {guide.SUMMARY}")[-1].split(
|
|
f"### {guide.KEY_FIGURES}"
|
|
)[0]
|
|
checks.append(
|
|
Check(
|
|
"the packager wrote a summary",
|
|
len(summary.split()) >= 30,
|
|
f"{len(summary.split())} words",
|
|
)
|
|
)
|
|
|
|
most = max(revisions.values(), default=0)
|
|
checks.append(
|
|
Check(
|
|
f"no Deliverable was revised more than {max_revisions} times",
|
|
most <= max_revisions,
|
|
", ".join(f"{kind} x {count}" for kind, count in revisions.items())
|
|
or "none",
|
|
)
|
|
)
|
|
|
|
scope = spec.agreed(deliverables)
|
|
cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {}
|
|
model = (
|
|
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
|
|
or {}
|
|
)
|
|
figures = {
|
|
"items": len(spec.items(scope)),
|
|
"conditions": len(spec.conditions(scope)),
|
|
"operational_requirements": len(
|
|
_attachment(
|
|
deliverables,
|
|
"operational_requirements",
|
|
"operational-requirements.json",
|
|
)
|
|
or []
|
|
),
|
|
"capabilities": len(
|
|
_attachment(deliverables, "capability_model", "capabilities.json") or []
|
|
),
|
|
"custom_capabilities": sum(
|
|
1
|
|
for c in _attachment(deliverables, "capability_model", "capabilities.json")
|
|
or []
|
|
if c.get("catalogue") == "custom"
|
|
),
|
|
"parties": len(model.get("parties") or []),
|
|
"roles": len(model.get("roles") or []),
|
|
"checks": len(own),
|
|
"checks_not_verifiable": sum(
|
|
1 for v in own if v.get("verdict") == "not_verifiable"
|
|
),
|
|
"checks_waiting_on_findings": sum(
|
|
1 for v in own if v.get("verdict") == scenarios.WAITING
|
|
),
|
|
"proposed_additions": len(verdicts.get("proposed_additions") or []),
|
|
"scenarios": len(walked),
|
|
"findings": len(
|
|
_attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or []
|
|
),
|
|
"business_outcomes": len(spec.business_outcomes(scope)),
|
|
"measures": len(model.get("measures") or []),
|
|
"people": cost.get("people"),
|
|
"cost_per_month": cost.get("cost_per_month"),
|
|
"currency": cost.get("currency"),
|
|
}
|
|
return checks, revisions, figures
|