Files
tom_analysis_agent/tests/test_graph.py
Jonathan Boniface 6fdac22876 feat: every Operational requirement verified by a Check; eval spread and report
The operating model is refused while an Operational requirement is verified
by no Check, from it or from a Condition it comes from; the architect is
told so, and the operating-chain Skill lists it. Six of nine earlier runs
left a requirement unverified.

The evaluation checks it too, runs each case N times (--repeat), reports
the spread of people, cost, Roles, Checks and Capabilities (spread.json),
and writes report.md to publish. Result folders are relative to the run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:32:05 +01:00

254 lines
8.4 KiB
Python

"""The Target operating model graph: what is refused, and the views code writes."""
import json
import pytest
from app.operating_model import graph as model
from app.operating_model.names import (
FINDINGS_FILE,
GRAPH_FILE,
ORGANISATION_FILE,
STAFFING_FILE,
)
from tests.support import CARD, CATALOGUE, GRAPH, as_file, graph, upstream
check = model.check(CARD)
add_views = model.views(CATALOGUE, CARD)
FINDING = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
def faults(changed, findings=None) -> list[str]:
attachments = as_file(GRAPH_FILE, changed)
if findings is not None:
attachments[FINDINGS_FILE] = json.dumps(findings)
return check(attachments, upstream())
def test_a_graph_that_holds_together_is_accepted():
assert faults(GRAPH) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda g: g["parties"][0].update(kind="friend"),
"Party P-1: kind must be one of",
),
(
lambda g: g["roles"][0].update(party="P-9"),
"Role R-1 is in P-9, which is not a Party",
),
(
lambda g: g["roles"][1].update(country="FR"),
"Role R-2 sits in FR, which the rate card does not have",
),
(
lambda g: g["roles"][1].update(country="US"),
"Role R-2 sits in US; the customer allows only GB, PL, IN",
),
(
lambda g: g["roles"][1].update(level="wizard"),
"Role R-2 is at level wizard, which the rate card does not have",
),
(
lambda g: g["roles"][1].update(hours="nights"),
"Role R-2: hours must be one of",
),
(lambda g: g["roles"][1].pop("country"), "(R-2) needs country"),
(
lambda g: g["responsibilities"][0].update(kind="owner"),
"Responsibility RS-1: kind must be one of",
),
(
lambda g: g["responsibilities"][0].update(role="R-9"),
"RS-1 names R-9, which is not a Role",
),
(
lambda g: g["responsibilities"][0].update(capability="C-9"),
"RS-1 names C-9, which is not a Capability",
),
(
lambda g: g["responsibilities"].pop(0),
"Capability C-1 has 0 accountable Roles; it needs exactly one",
),
(
lambda g: g["responsibilities"][2].update(kind="accountable"),
"Capability C-1 has 2 accountable Roles; it needs exactly one",
),
(
lambda g: g["responsibilities"].pop(1),
"Capability C-1 has no responsible Role",
),
(
lambda g: g["roles"][1].update(hours="business"),
"Capability C-1 is needed 24x7, and no Role responsible for it works those hours",
),
(lambda g: g["checks"].pop(1), "Condition K-1 is carried by no Check"),
(
lambda g: g["checks"].pop(0),
"Operational requirement OR-1 is verified by no Check",
),
(
lambda g: g["checks"][0].update(on="C-9"),
"Check OC-1 is on C-9, which is not",
),
(
lambda g: g["checks"][0].update(source="OR-9"),
"Check OC-1: source OR-9 is not an Operational requirement",
),
(
lambda g: g["checks"][0].update(depends_on_finding="AF-1"),
"Check OC-1 depends on AF-1, which is not a finding",
),
(
lambda g: g["roles"].append(dict(g["roles"][0])),
"id R-1 is used more than once in roles",
),
],
)
def test_a_graph_that_does_not_hold_together_is_refused(change, problem):
changed = graph()
change(changed)
found = faults(changed)
assert any(problem in fault for fault in found), found
def test_a_requirement_is_verified_through_the_conditions_it_comes_from():
# OR-3 comes from K-1 and K-2: the Checks carrying them verify it.
changed = graph()
assert not any(c["source"] == "OR-3" for c in changed["checks"])
assert not any("OR-3" in fault for fault in faults(changed))
def test_a_file_that_is_not_a_graph_is_named():
assert "not valid JSON" in check({GRAPH_FILE: "{"}, upstream())[0]
assert "must be an object with" in check({GRAPH_FILE: "[]"}, upstream())[0]
def test_a_finding_is_recorded_and_a_check_may_wait_on_it():
changed = graph()
changed["checks"][2]["depends_on_finding"] = "AF-1"
assert faults(changed, [FINDING]) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(lambda f: f.update(lacks=""), "AF-1 needs lacks"),
(lambda f: f.update(proposed=""), "AF-1 needs proposed"),
(lambda f: f.update(needed_by=[]), "AF-1 must say what needs it"),
(lambda f: f.update(about=["S-9"]), "AF-1 is about S-9, which is not an Item"),
(
lambda f: f.update(about=[]),
"AF-1 names no Item of the solution under about",
),
(
lambda f: f.update(needed_by=["C-9"]),
"AF-1 is needed by C-9, which is not in the operating model",
),
],
)
def test_a_finding_that_does_not_hold_together_is_refused(change, problem):
finding = dict(FINDING)
change(finding)
found = faults(GRAPH, [finding])
assert any(problem in fault for fault in found), found
def viewed(changed=GRAPH, findings=None, spec=None):
attachments = as_file(GRAPH_FILE, changed)
if findings is not None:
attachments[FINDINGS_FILE] = json.dumps(findings)
built = upstream() if spec is None else upstream(spec=spec)
return add_views(
"# Target operating model\n\n## Design rationale\n\nWhy.\n", attachments, built
)
def test_the_matrix_is_written_from_the_graph():
content, _ = viewed()
assert "## Responsibility matrix" in content
assert "| Capability | Platform owner | Platform SRE | Service manager |" in content
assert "| Incident management (C-1) | C | R | A |" in content
assert "| Change enablement (C-2) | A | R | |" in content
# One Role may be answerable for a Capability and do it: both parts are shown.
assert "| Regulatory evidence and reporting (C-3) | C | | A/R |" in content
def test_staffing_and_cost_are_written_with_the_prices_they_came_from():
content, attachments = viewed()
assert (
"| Platform SRE | Managed service provider | PL | senior | 24x7 | 100.0 h | 5.0 | 41,640 GBP |"
in content
)
assert (
"| Platform owner | Northwind platform team | GB | lead | business | 0.0 h | 0.0 | not priced |"
in content
)
assert "Cost a month: 43,410 GBP." in content
assert "The monthly budget is 120,000 GBP: this is within it." in content
assert "version `draft-2026-10-13`" in content
assert "Its figures are placeholders" in content
assert "Converted from USD at 0.79, a rate set on 2026-10-13." in content
assert json.loads(attachments[STAFFING_FILE])["cost_per_month"] == 43410
def test_a_cost_over_the_budget_is_shown_as_over_not_refused():
from tests.support import SPEC
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
content, _ = viewed(spec=tight)
assert "this is over it by 3,410 GBP." in content
assert check(as_file(GRAPH_FILE, GRAPH), upstream(spec=tight)) == []
def test_the_organisation_chart_is_drawn_from_the_graph():
_, attachments = viewed()
chart = attachments[ORGANISATION_FILE]
assert chart.startswith("flowchart TB\n")
assert 'subgraph P_2["Managed service provider (provider)"]' in chart
assert 'R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
def test_findings_are_listed_for_whoever_owns_the_design():
content, attachments = viewed(findings=[FINDING])
assert "## Findings" in content
assert "**AF-1** (S-2): Nothing alerts when" in content
assert json.loads(attachments[FINDINGS_FILE]) == [FINDING]
def test_with_no_findings_the_file_is_empty_and_no_section_is_written():
content, attachments = viewed()
assert "## Findings" not in content
assert attachments[FINDINGS_FILE] == "[]"
def test_a_revision_replaces_the_views_and_never_stacks_them():
first, attachments = viewed()
second, _ = add_views(first, dict(attachments), upstream())
assert second == first
assert second.count("## Responsibility matrix") == 1