"""Checks applied to a finished Execution of the target operating model Workflow. Every check here is structural: it reads the Execution's own data and asks whether the chain holds and the result is what the Workflow promises. None judges whether the operating model is a good one; that needs a person, or a judge calibrated against one, and is not built. """ from __future__ import annotations import json from collections.abc import Mapping from dataclasses import asdict, dataclass, field from typing import Any from execution_engine.workflows.workflow import Workflow from app.operating_model import guide, scenarios, spec from app.operating_model.names import ( GUIDE, SCENARIOS_FILE, STAFFING_FILE, TARGET_OPERATING_MODEL, VALIDATION_RESULTS, VERDICTS_FILE, ) @dataclass(frozen=True) class Check: name: str passed: bool detail: str = "" @dataclass class CaseResult: case: str status: str checks: list[Check] = field(default_factory=list) minutes: float = 0.0 revisions: dict[str, int] = field(default_factory=dict) tokens: dict[str, dict[str, int]] = field(default_factory=dict) figures: dict[str, Any] = field(default_factory=dict) warnings: list[str] = field(default_factory=list) folder: str = "" @property def passed(self) -> bool: return all(check.passed for check in self.checks) def as_dict(self) -> dict[str, Any]: return {**asdict(self), "passed": self.passed} def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any: text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "") try: return json.loads(text) if text else None except ValueError: return None def score( workflow: Workflow, state: Mapping[str, Any], *, status: str, max_revisions: int = 2, ) -> tuple[list[Check], dict[str, int], dict[str, Any]]: """The checks on a finished Execution, its Revisions, and its key figures.""" deliverables: Mapping[str, Any] = state.get("deliverables") or {} ledger = (state.get("ledger") or {}).get("deliverables") or {} revisions = { kind: int(entry.get("revisions", 0)) for kind, entry in ledger.items() if entry.get("revisions") } checks = [ Check("completed", status == "completed", status), Check( "every Deliverable was submitted", set(deliverables) == set(workflow.deliverables), ", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all", ), ] # The chain, checked again on what was delivered: each Deliverable's data # against the ones it builds on, as the engine checked it when it was submitted. broken = [] for kind in workflow.deliverables: if kind not in deliverables or kind not in workflow.data_checks: continue name, check = workflow.data_checks[kind] attachments = deliverables[kind].get("attachments", {}) upstream = { k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables } broken += ( [f"{kind}: {fault}" for fault in check(attachments, upstream)] if name in attachments else [f"{kind}: {name} is missing"] ) checks.append(Check("the chain holds", not broken, "; ".join(broken[:5]))) verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {} given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)] own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)] failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"] checks.append(Check("no Check failed", not failed, ", ".join(failed))) walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or [] checks.append( Check( "every scenario has a path", bool(walked) and all(item.get("passes") for item in walked), f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}", ) ) content = str((deliverables.get(GUIDE) or {}).get("content", "")) headings = [ line[3:].strip() for line in content.splitlines() if line.startswith("## ") ] checks.append( Check( "the guide has its sections", headings == list(guide.SECTIONS), ", ".join(headings), ) ) summary = content.split(f"## {guide.SUMMARY}")[-1].split( f"### {guide.KEY_FIGURES}" )[0] checks.append( Check( "the packager wrote a summary", len(summary.split()) >= 30, f"{len(summary.split())} words", ) ) most = max(revisions.values(), default=0) checks.append( Check( f"no Deliverable was revised more than {max_revisions} times", most <= max_revisions, ", ".join(f"{kind} x {count}" for kind, count in revisions.items()) or "none", ) ) scope = spec.agreed(deliverables) cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {} model = ( _attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json") or {} ) figures = { "items": len(spec.items(scope)), "conditions": len(spec.conditions(scope)), "operational_requirements": len( _attachment( deliverables, "operational_requirements", "operational-requirements.json", ) or [] ), "capabilities": len( _attachment(deliverables, "capability_model", "capabilities.json") or [] ), "custom_capabilities": sum( 1 for c in _attachment(deliverables, "capability_model", "capabilities.json") or [] if c.get("catalogue") == "custom" ), "parties": len(model.get("parties") or []), "roles": len(model.get("roles") or []), "checks": len(own), "checks_not_verifiable": sum( 1 for v in own if v.get("verdict") == "not_verifiable" ), "checks_waiting_on_findings": sum( 1 for v in own if v.get("verdict") == scenarios.WAITING ), "proposed_additions": len(verdicts.get("proposed_additions") or []), "scenarios": len(walked), "findings": len( _attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or [] ), "business_outcomes": len(spec.business_outcomes(scope)), "measures": len(model.get("measures") or []), "people": cost.get("people"), "cost_per_month": cost.get("cost_per_month"), "currency": cost.get("currency"), } return checks, revisions, figures