Files
tom_analysis_agent/tests/test_guide.py

167 lines
5.7 KiB
Python

"""The operating model guide: the packager writes the summary, code writes the rest."""
import json
from app.operating_model import guide, scenarios, spec, staffing
from app.operating_model.names import (
FINDINGS_FILE,
SCENARIOS_FILE,
STAFFING_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
from tests.support import (
CAPABILITIES,
CARD,
CATALOGUE,
GRAPH,
VERDICTS,
WORKFLOW,
upstream,
)
DRAFT = "# Northwind landing zone\n\n" + "".join(
f"## {section}\n\n"
+ (
"The provider runs the platform at every hour; Northwind approves changes.\n\n"
if section == guide.SUMMARY
else "It costs 9,999 GBP and needs 2 people.\n\n"
)
for section in guide.SECTIONS
)
def delivered(findings=None, model=GRAPH) -> dict:
"""Every Deliverable the guide is assembled from, with what code added to each."""
built = upstream(model=model, findings=findings)
scope = spec.agreed(built)
cost = staffing.compute(model, CAPABILITIES, CARD, "GBP")
built[TARGET_OPERATING_MODEL]["attachments"][STAFFING_FILE] = json.dumps(cost)
built[TARGET_OPERATING_MODEL]["attachments"].setdefault(FINDINGS_FILE, "[]")
walked = scenarios.walk(model, CAPABILITIES, scope, CATALOGUE, cost)
waiting = {
str(c["id"]): str(c["depends_on_finding"])
for c in model["checks"]
if c.get("depends_on_finding")
}
judged = json.loads(json.dumps(VERDICTS))
for verdict in judged["checks"]:
if verdict["check"] in waiting:
verdict.update(verdict="not_verifiable", evidence="Waits on its finding.")
built[VALIDATION_RESULTS] = {
"content": "# Validation\n",
"attachments": {
VERDICTS_FILE: scenarios.merge(json.dumps(judged), walked, waiting),
SCENARIOS_FILE: json.dumps(walked),
},
}
return built
def assembled(**options) -> str:
return guide.assemble(DRAFT, delivered(**options), CATALOGUE)
def section(text: str, heading: str) -> str:
return text.split(f"## {heading}\n")[1].split("\n## ")[0]
def test_the_guide_has_every_section_once_and_in_order():
text = assembled()
headings = [line[3:] for line in text.splitlines() if line.startswith("## ")]
assert headings == list(guide.SECTIONS)
assert text.startswith("# Northwind landing zone\n")
assert WORKFLOW.check_submission("operating_model_guide", text, {}) == ()
def test_the_packagers_summary_is_kept_and_the_key_figures_are_set_under_it():
summary = section(assembled(), guide.SUMMARY)
assert "The provider runs the platform at every hour" in summary
assert "### Key figures" in summary
assert "- Operated: 3 Items, with 2 Conditions to keep true." in summary
assert (
"- People: 5.1. Cost a month: 43,410 GBP, from rate card version "
"`draft-2026-10-13` (placeholder prices: an illustration)." in summary
)
assert "- Scenario walk-throughs: 10 of 10 pass." in summary
def test_a_figure_the_packager_wrote_under_another_heading_is_replaced():
text = assembled()
assert "9,999" not in text
assert "needs 2 people" not in text
assert "Cost a month: 43,410 GBP." in section(text, "Staffing and cost")
def test_each_section_is_written_from_the_data():
text = assembled()
assert "| S-2 | Audit log store | storage | Cloud Storage | E-2 |" in section(
text, "What is operated"
)
assert "| K-1 | S-1 | Flow logs are retained for 400 days. | CHK-12 |" in text
assert "| OR-2 | process |" in section(text, "Operational requirements")
assert (
"| C-1 | Incident management | OR-1, OR-4 | S-1, S-2, S-3 | 24x7 | 60 h |"
in section(text, "Capabilities")
)
who = section(text, "Who does what")
assert (
"| Platform SRE | Managed service provider (provider) | PL | senior | 24x7 |"
in who
)
assert "| Incident management (C-1) | C | R | A |" in who
checked = section(text, "Validation")
assert "| OC-1 | A fault on the hub network is acknowledged" in checked
assert "| Hub network (S-1) must be changed. | pass |" in checked
def test_sources_are_cited_and_said_to_be_unchecked():
cited = section(assembled(), "Sources")
assert "- ITIL 4: Incident management practice; Change enablement practice" in cited
assert "certifies nothing against them" in cited
assert "have not been checked against the frameworks by a person" in cited
def test_what_was_left_open_is_listed_and_what_nobody_asked_for_is_not():
left_open = section(assembled(), "Findings and open questions")
assert "No findings" in left_open
assert (
"`operations.change_process`: affects change enablement and release."
in left_open
)
assert "`service_management.tooling`: Which service management" in left_open
assert "metadata.author" not in left_open
def test_a_finding_and_the_check_that_waits_on_it_are_shown():
finding = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
waiting = json.loads(json.dumps(GRAPH))
waiting["checks"][2]["depends_on_finding"] = "AF-1"
text = assembled(findings=[finding], model=waiting)
assert "**AF-1** (S-2): Nothing alerts when" in section(
text, "Findings and open questions"
)
assert "| waits on finding (AF-1) |" in section(text, "Validation")
assert "- Findings for whoever owns the design: 1." in text
def test_assembling_a_guide_again_changes_nothing():
first = assembled()
assert guide.assemble(first, delivered(), CATALOGUE) == first