Files
tom_analysis_agent/eval/scoring.py

199 lines
6.7 KiB
Python

"""Checks applied to a finished Execution of the target operating model Workflow.
Every check here is structural: it reads the Execution's own data and asks whether
the chain holds and the result is what the Workflow promises. None judges whether
the operating model is a good one; that needs a person, or a judge calibrated
against one, and is not built.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from dataclasses import asdict, dataclass, field
from typing import Any
from execution_engine.workflows.workflow import Workflow
from app.operating_model import guide, scenarios, spec
from app.operating_model.names import (
GUIDE,
SCENARIOS_FILE,
STAFFING_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass
class CaseResult:
case: str
status: str
checks: list[Check] = field(default_factory=list)
minutes: float = 0.0
revisions: dict[str, int] = field(default_factory=dict)
tokens: dict[str, dict[str, int]] = field(default_factory=dict)
figures: dict[str, Any] = field(default_factory=dict)
warnings: list[str] = field(default_factory=list)
folder: str = ""
@property
def passed(self) -> bool:
return all(check.passed for check in self.checks)
def as_dict(self) -> dict[str, Any]:
return {**asdict(self), "passed": self.passed}
def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "")
try:
return json.loads(text) if text else None
except ValueError:
return None
def score(
workflow: Workflow,
state: Mapping[str, Any],
*,
status: str,
max_revisions: int = 2,
) -> tuple[list[Check], dict[str, int], dict[str, Any]]:
"""The checks on a finished Execution, its Revisions, and its key figures."""
deliverables: Mapping[str, Any] = state.get("deliverables") or {}
ledger = (state.get("ledger") or {}).get("deliverables") or {}
revisions = {
kind: int(entry.get("revisions", 0))
for kind, entry in ledger.items()
if entry.get("revisions")
}
checks = [
Check("completed", status == "completed", status),
Check(
"every Deliverable was submitted",
set(deliverables) == set(workflow.deliverables),
", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all",
),
]
# The chain, checked again on what was delivered: each Deliverable's data
# against the ones it builds on, as the engine checked it when it was submitted.
broken = []
for kind in workflow.deliverables:
if kind not in deliverables or kind not in workflow.data_checks:
continue
name, check = workflow.data_checks[kind]
attachments = deliverables[kind].get("attachments", {})
upstream = {
k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables
}
broken += (
[f"{kind}: {fault}" for fault in check(attachments, upstream)]
if name in attachments
else [f"{kind}: {name} is missing"]
)
checks.append(Check("the chain holds", not broken, "; ".join(broken[:5])))
verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {}
given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)]
own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)]
failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"]
checks.append(Check("no Check failed", not failed, ", ".join(failed)))
walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or []
checks.append(
Check(
"every scenario has a path",
bool(walked) and all(item.get("passes") for item in walked),
f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}",
)
)
content = str((deliverables.get(GUIDE) or {}).get("content", ""))
headings = [
line[3:].strip() for line in content.splitlines() if line.startswith("## ")
]
checks.append(
Check(
"the guide has its sections",
headings == list(guide.SECTIONS),
", ".join(headings),
)
)
summary = content.split(f"## {guide.SUMMARY}")[-1].split(
f"### {guide.KEY_FIGURES}"
)[0]
checks.append(
Check(
"the packager wrote a summary",
len(summary.split()) >= 30,
f"{len(summary.split())} words",
)
)
most = max(revisions.values(), default=0)
checks.append(
Check(
f"no Deliverable was revised more than {max_revisions} times",
most <= max_revisions,
", ".join(f"{kind} x {count}" for kind, count in revisions.items())
or "none",
)
)
scope = spec.agreed(deliverables)
cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {}
model = (
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
or {}
)
figures = {
"items": len(spec.items(scope)),
"conditions": len(spec.conditions(scope)),
"operational_requirements": len(
_attachment(
deliverables,
"operational_requirements",
"operational-requirements.json",
)
or []
),
"capabilities": len(
_attachment(deliverables, "capability_model", "capabilities.json") or []
),
"custom_capabilities": sum(
1
for c in _attachment(deliverables, "capability_model", "capabilities.json")
or []
if c.get("catalogue") == "custom"
),
"parties": len(model.get("parties") or []),
"roles": len(model.get("roles") or []),
"checks": len(own),
"checks_not_verifiable": sum(
1 for v in own if v.get("verdict") == "not_verifiable"
),
"checks_waiting_on_findings": sum(
1 for v in own if v.get("verdict") == scenarios.WAITING
),
"proposed_additions": len(verdicts.get("proposed_additions") or []),
"scenarios": len(walked),
"findings": len(
_attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or []
),
"business_outcomes": len(spec.business_outcomes(scope)),
"measures": len(model.get("measures") or []),
"people": cost.get("people"),
"cost_per_month": cost.get("cost_per_month"),
"currency": cost.get("currency"),
}
return checks, revisions, figures