Files
tom_analysis_agent/tests/support.py
Jonathan Boniface 6fdac22876 feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:32:05 +01:00

307 lines
9.4 KiB
Python

"""One small, coherent operating model, for tests: a fictional landing zone run as a
shared service. Every name in it is invented."""
from __future__ import annotations
import copy
import json
from typing import Any
from execution_engine.providers import load_provider
from execution_engine.scope import read_scope
from app.operating_model import catalogue as capability_catalogue
from app.operating_model import staffing
from app.operating_model.names import (
CAPABILITIES_FILE,
CAPABILITY_MODEL,
FINDINGS_FILE,
GRAPH_FILE,
OPERATING_SCOPE,
OPERATIONAL_REQUIREMENTS,
REQUIREMENTS_FILE,
SCOPE_DATA_FILE,
TARGET_OPERATING_MODEL,
VERDICTS_FILE,
)
from app.operating_model.workflow import ROOT, build
BUNDLE = load_provider("gcp", ROOT)
CATALOGUE = capability_catalogue.load(ROOT / "skills" / capability_catalogue.SKILL)
CARD = staffing.load(ROOT / "skills" / staffing.SKILL)
WORKFLOW = build(ROOT)
SPEC = """\
metadata:
customer_case: northwind-landing-zone
engagement:
kind: platform_foundation
intent:
statement: |
We want the landing zone run as a shared service from go-live, with our own
platform team keeping the final say.
business_outcomes:
- {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}
- {id: BO-2, outcome: Regulators always get the evidence they ask for.}
requirements:
- {id: REQ-01, statement: Our platform team approves every change to the network.}
solution:
cloud_provider: gcp
items:
- {id: S-1, name: Hub network, kind: network, product: Shared VPC, source_ref: E-1}
- {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage, source_ref: E-2}
- {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build, source_ref: E-3}
conditions:
- {id: K-1, "on": S-1, condition: Flow logs are retained for 400 days., source_ref: CHK-12}
- {id: K-2, "on": S-2, condition: Audit logs cannot be deleted for seven years., source_ref: CHK-20}
operations:
support_hours: 24x7
availability_hours: 24x7
change_process: decide
sourcing:
model: shared
delivery_countries: [GB, PL, IN]
service_levels:
priority_1_response_minutes: 15
recovery_time_hours: 4
compliance:
frameworks: [DORA]
commercial:
currency: GBP
monthly_run_budget: 120000
run:
pre_approved: true
"""
def scope_data(text: str = SPEC) -> dict[str, Any]:
"""What every Specialist is given of an operating spec."""
scope = read_scope(
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
)
return scope.for_context()
REQUIREMENTS: list[dict[str, Any]] = [
{
"id": "OR-1",
"kind": "service_level",
"statement": "A fault on any Item is acknowledged within 15 minutes, at any hour.",
"source": [
"service_levels.priority_1_response_minutes",
"operations.support_hours",
],
},
{
"id": "OR-2",
"kind": "process",
"statement": "The customer's platform team approves every change to the network.",
"source": ["REQ-01", "sourcing.model"],
},
{
"id": "OR-3",
"kind": "compliance",
"statement": "Logs are kept for as long as each Condition says, and cannot be removed sooner.",
"source": ["K-1", "K-2"],
},
{
"id": "OR-4",
"kind": "service_level",
"statement": "Service is restored within 4 hours of a major failure.",
"source": ["service_levels.recovery_time_hours"],
},
{
"id": "OR-5",
"kind": "organisation",
"statement": "The solution is operated only from the countries the customer allows.",
"source": ["sourcing.delivery_countries"],
},
]
CAPABILITIES: list[dict[str, Any]] = [
{
"id": "C-1",
"catalogue": "incident-management",
"serves": ["OR-1", "OR-4"],
"items": ["S-1", "S-2", "S-3"],
"hours": "24x7",
"effort_hours_per_month": 60,
"basis": "Two serious incidents a month and daily triage.",
},
{
"id": "C-2",
"catalogue": "change-enablement",
"serves": ["OR-2"],
"items": ["S-1", "S-2", "S-3"],
"hours": "business",
"effort_hours_per_month": 40,
"basis": "About twenty changes a month at two hours each.",
},
{
"id": "C-3",
"catalogue": "regulatory-evidence",
"serves": ["OR-3", "OR-5"],
"items": ["S-1", "S-2"],
"hours": "business",
"effort_hours_per_month": 16,
"basis": "A monthly review of retention settings and evidence.",
},
]
GRAPH: dict[str, list[dict[str, Any]]] = {
"parties": [
{"id": "P-1", "name": "Northwind platform team", "kind": "customer"},
{"id": "P-2", "name": "Managed service provider", "kind": "provider"},
],
"roles": [
{
"id": "R-1",
"party": "P-1",
"name": "Platform owner",
"country": "GB",
"level": "lead",
"hours": "business",
},
{
"id": "R-2",
"party": "P-2",
"name": "Platform SRE",
"country": "PL",
"level": "senior",
"hours": "24x7",
},
{
"id": "R-3",
"party": "P-2",
"name": "Service manager",
"country": "GB",
"level": "lead",
"hours": "business",
},
],
"responsibilities": [
{"id": "RS-1", "capability": "C-1", "role": "R-3", "kind": "accountable"},
{"id": "RS-2", "capability": "C-1", "role": "R-2", "kind": "responsible"},
{"id": "RS-3", "capability": "C-1", "role": "R-1", "kind": "consulted"},
{"id": "RS-4", "capability": "C-2", "role": "R-1", "kind": "accountable"},
{"id": "RS-5", "capability": "C-2", "role": "R-2", "kind": "responsible"},
{"id": "RS-6", "capability": "C-3", "role": "R-3", "kind": "accountable"},
{"id": "RS-7", "capability": "C-3", "role": "R-3", "kind": "responsible"},
{"id": "RS-8", "capability": "C-3", "role": "R-1", "kind": "consulted"},
],
"measures": [
{
"id": "M-1",
"name": "Time to restore service",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-1",
"framework": "dora-metrics",
"target": "Under 4 hours",
},
{
"id": "M-2",
"name": "Evidence requests met on time",
"layer": "service_level",
"outcome": "BO-2",
"measured_by": "C-3",
},
],
"checks": [
{
"id": "OC-1",
"on": "C-1",
"source": "OR-1",
"condition": "A fault on the hub network is acknowledged within 15 minutes at any hour.",
},
{
"id": "OC-2",
"on": "C-3",
"source": "K-1",
"condition": "Flow log retention is reviewed monthly and is 400 days.",
},
{
"id": "OC-3",
"on": "C-3",
"source": "K-2",
"condition": "The audit log store's retention lock is reviewed monthly.",
},
{
"id": "OC-4",
"on": "RS-4",
"source": "OR-2",
"condition": "No change to the network is applied without the platform owner's approval.",
},
{
"id": "OC-5",
"on": "C-1",
"source": "OR-4",
"condition": "A recovery of the hub network is rehearsed each quarter within 4 hours.",
},
{
"id": "OC-6",
"on": "C-3",
"source": "OR-5",
"condition": "Every Role is placed in a country the customer allows.",
},
],
}
VERDICTS: dict[str, Any] = {
"checks": [
{
"check": check["id"],
"verdict": "pass",
"evidence": "The model provides for it.",
}
for check in GRAPH["checks"]
],
"proposed_additions": [],
}
def graph(**changes: Any) -> dict[str, list[dict[str, Any]]]:
"""A copy of the graph to change."""
copied = copy.deepcopy(GRAPH)
copied.update(changes)
return copied
def upstream(
*,
spec: str = SPEC,
requirements: list[dict[str, Any]] | None = None,
capabilities: list[dict[str, Any]] | None = None,
model: dict[str, Any] | None = None,
findings: list[dict[str, Any]] | None = None,
) -> dict[str, dict[str, Any]]:
"""The Deliverables a submission builds on, as the engine holds them."""
def deliverable(name: str, data: Any) -> dict[str, Any]:
return {"content": "# Prose\n", "attachments": {name: json.dumps(data)}}
built = {
OPERATING_SCOPE: deliverable(SCOPE_DATA_FILE, scope_data(spec)),
OPERATIONAL_REQUIREMENTS: deliverable(
REQUIREMENTS_FILE, REQUIREMENTS if requirements is None else requirements
),
CAPABILITY_MODEL: deliverable(
CAPABILITIES_FILE, CAPABILITIES if capabilities is None else capabilities
),
TARGET_OPERATING_MODEL: deliverable(
GRAPH_FILE, GRAPH if model is None else model
),
}
if findings is not None:
built[TARGET_OPERATING_MODEL]["attachments"][FINDINGS_FILE] = json.dumps(
findings
)
return built
def as_file(name: str, data: Any) -> dict[str, str]:
return {name: json.dumps(data)}
__all__ = ["VERDICTS_FILE"]