Files
tom_analysis_agent/eval/scoring.py
Jonathan Boniface 6fdac22876 feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:32:05 +01:00

235 lines
8.2 KiB
Python

"""Checks applied to a finished Execution of the target operating model Workflow.
Every check here is structural: it reads the Execution's own data and asks whether
the chain holds and the result is what the Workflow promises. None judges whether
the operating model is a good one; that needs a person, or a judge calibrated
against one, and is not built.
"""
from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from dataclasses import asdict, dataclass, field
from typing import Any
from execution_engine.workflows.workflow import Workflow
from app.operating_model import guide, scenarios, spec
from app.operating_model.names import (
GUIDE,
SCENARIOS_FILE,
STAFFING_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass
class CaseResult:
case: str
status: str
checks: list[Check] = field(default_factory=list)
minutes: float = 0.0
revisions: dict[str, int] = field(default_factory=dict)
tokens: dict[str, dict[str, int]] = field(default_factory=dict)
figures: dict[str, Any] = field(default_factory=dict)
warnings: list[str] = field(default_factory=list)
folder: str = ""
@property
def passed(self) -> bool:
return all(check.passed for check in self.checks)
def as_dict(self) -> dict[str, Any]:
return {**asdict(self), "passed": self.passed}
def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "")
try:
return json.loads(text) if text else None
except ValueError:
return None
def unverified(
requirements: Sequence[Mapping[str, Any]], model: Mapping[str, Any]
) -> list[str]:
"""The Operational requirements no Check verifies: none has the requirement, or
a Condition it comes from, as its source."""
carried = {str(check.get("source")) for check in model.get("checks") or []}
missing = []
for requirement in requirements:
named = requirement.get("source") or []
named = [named] if isinstance(named, str) else named
# A Check's source is a requirement or a Condition (the graph refuses
# anything else), so a source of the requirement that a Check carries is a
# Condition it comes from.
if str(requirement.get("id")) not in carried and not any(
str(source) in carried for source in named
):
missing.append(str(requirement.get("id")))
return missing
def score(
workflow: Workflow,
state: Mapping[str, Any],
*,
status: str,
max_revisions: int = 2,
) -> tuple[list[Check], dict[str, int], dict[str, Any]]:
"""The checks on a finished Execution, its Revisions, and its key figures."""
deliverables: Mapping[str, Any] = state.get("deliverables") or {}
ledger = (state.get("ledger") or {}).get("deliverables") or {}
revisions = {
kind: int(entry.get("revisions", 0))
for kind, entry in ledger.items()
if entry.get("revisions")
}
checks = [
Check("completed", status == "completed", status),
Check(
"every Deliverable was submitted",
set(deliverables) == set(workflow.deliverables),
", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all",
),
]
# The chain, checked again on what was delivered: each Deliverable's data
# against the ones it builds on, as the engine checked it when it was submitted.
broken = []
for kind in workflow.deliverables:
if kind not in deliverables or kind not in workflow.data_checks:
continue
name, check = workflow.data_checks[kind]
attachments = deliverables[kind].get("attachments", {})
upstream = {
k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables
}
broken += (
[f"{kind}: {fault}" for fault in check(attachments, upstream)]
if name in attachments
else [f"{kind}: {name} is missing"]
)
checks.append(Check("the chain holds", not broken, "; ".join(broken[:5])))
verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {}
given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)]
own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)]
failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"]
checks.append(Check("no Check failed", not failed, ", ".join(failed)))
walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or []
checks.append(
Check(
"every scenario has a path",
bool(walked) and all(item.get("passes") for item in walked),
f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}",
)
)
content = str((deliverables.get(GUIDE) or {}).get("content", ""))
headings = [
line[3:].strip() for line in content.splitlines() if line.startswith("## ")
]
checks.append(
Check(
"the guide has its sections",
headings == list(guide.SECTIONS),
", ".join(headings),
)
)
summary = content.split(f"## {guide.SUMMARY}")[-1].split(
f"### {guide.KEY_FIGURES}"
)[0]
checks.append(
Check(
"the packager wrote a summary",
len(summary.split()) >= 30,
f"{len(summary.split())} words",
)
)
most = max(revisions.values(), default=0)
checks.append(
Check(
f"no Deliverable was revised more than {max_revisions} times",
most <= max_revisions,
", ".join(f"{kind} x {count}" for kind, count in revisions.items())
or "none",
)
)
scope = spec.agreed(deliverables)
cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {}
model = (
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
or {}
)
requirements = (
_attachment(
deliverables, "operational_requirements", "operational-requirements.json"
)
or []
)
missing = unverified(requirements, model)
checks.append(
Check(
"every Operational requirement is verified by a Check",
bool(requirements) and not missing,
f"{len(requirements) - len(missing)} of {len(requirements)}"
+ (f"; not verified: {', '.join(missing)}" if missing else ""),
)
)
figures = {
"items": len(spec.items(scope)),
"conditions": len(spec.conditions(scope)),
"operational_requirements": len(
_attachment(
deliverables,
"operational_requirements",
"operational-requirements.json",
)
or []
),
"capabilities": len(
_attachment(deliverables, "capability_model", "capabilities.json") or []
),
"custom_capabilities": sum(
1
for c in _attachment(deliverables, "capability_model", "capabilities.json")
or []
if c.get("catalogue") == "custom"
),
"parties": len(model.get("parties") or []),
"roles": len(model.get("roles") or []),
"checks": len(own),
"requirements_unverified": len(missing),
"checks_not_verifiable": sum(
1 for v in own if v.get("verdict") == "not_verifiable"
),
"checks_waiting_on_findings": sum(
1 for v in own if v.get("verdict") == scenarios.WAITING
),
"proposed_additions": len(verdicts.get("proposed_additions") or []),
"scenarios": len(walked),
"findings": len(
_attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or []
),
"business_outcomes": len(spec.business_outcomes(scope)),
"measures": len(model.get("measures") or []),
"people": cost.get("people"),
"cost_per_month": cost.get("cost_per_month"),
"currency": cost.get("currency"),
}
return checks, revisions, figures