Files
tom_analysis_agent/tests/test_execution.py

412 lines
15 KiB
Python

"""Whole Executions of the target operating model Workflow, on the scripted runtime."""
import json
from pathlib import Path
from typing import Any
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.roles import load_roles
from execution_engine.adk.runtime import NoDecider
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from langgraph.checkpoint.memory import InMemorySaver
from app.operating_model.names import (
CAPABILITIES_FILE,
CAPABILITY_MODEL,
FINDINGS_FILE,
GRAPH_FILE,
GUIDE,
OPERATING_SCOPE,
OPERATIONAL_REQUIREMENTS,
REQUIREMENTS_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
from app.operating_model.workflow import ROOT
from tests.support import (
BUNDLE,
CAPABILITIES,
GRAPH,
REQUIREMENTS,
SPEC,
VERDICTS,
WORKFLOW,
graph,
)
ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW)
CASE = "northwind-landing-zone"
FINDING = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
def submit(kind: str, **files: Any) -> list[Call]:
"""One turn that submits a Deliverable with every section of its contract."""
spec = WORKFLOW.deliverables[kind]
content = f"# {kind}\n" + "".join(
f"\n## {section}\n\nDetails.\n" for section in spec.sections
)
return [
Call(
"submit_deliverable",
{
"content": content,
"attachments": {name: json.dumps(data) for name, data in files.items()},
},
)
]
def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]:
model_files = {GRAPH_FILE: model}
if findings is not None:
model_files[FINDINGS_FILE] = findings
return {
"scope-analyst": [submit(OPERATING_SCOPE)],
"operations-analyst": [
submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS})
],
"capability-modeller": [
submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES})
],
"operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)],
"validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})],
"packager": [submit(GUIDE)],
}
def make_engine(
tmp_path: Path, scripted: dict[str, list[Any]]
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(scripted)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
scope_catalogue=BUNDLE.scope_catalogue(),
scope_values=BUNDLE.scope_values,
scope_skill=BUNDLE.scope_skill,
)
return engine, runtime
def folder(tmp_path: Path, execution_id: str) -> Path:
return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path):
engine, runtime = make_engine(tmp_path, turns())
result = await engine.start("", scope=SPEC, execution_id="e1")
assert result.status == "completed", result.state.get("warnings")
assert [run.role for run in runtime.runs] == [
"scope-analyst",
"operations-analyst",
"capability-modeller",
"operating-model-architect",
"validator",
"packager",
]
out = folder(tmp_path, "e1")
written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
assert {
"execution.json",
"operating-model-guide.md",
"scope.yaml",
"scope.json",
"operational-requirements.json",
"capabilities.json",
"target-operating-model.json",
"findings.json",
"staffing.json",
"organisation.mmd",
"verdicts.json",
"scenarios.json",
} <= written
record = json.loads((out / "execution.json").read_text())
assert record["customer_case"] == CASE
assert record["final_document"] == "operating-model-guide.md"
assert record["staffing"] == {
"people": 5.1,
"cost_per_month": 43410,
"currency": "GBP",
"rate_card": {
"name": "rate-card",
"version": "draft-2026-10-13",
"draft": True,
},
}
assert record["findings"] == []
assert record["capability_sources_verified"] is False
async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path):
engine, _ = make_engine(tmp_path, turns())
result = await engine.start("", scope=SPEC)
model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
assert "| Incident management (C-1) | C | R | A |" in model
assert "Cost a month: 43,410 GBP." in model
validation = result.state["deliverables"][VALIDATION_RESULTS]
assert "## Scenario walk-throughs" in validation["content"]
verdicts = json.loads(validation["attachments"][VERDICTS_FILE])
assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass."
assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10
async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path):
engine, runtime = make_engine(tmp_path, turns())
await engine.start("", scope=SPEC)
by_role = {run.role: run.task.context for run in runtime.runs}
proposed = by_role["capability-modeller"]["capabilities_proposed"]
assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set(
proposed
)
assert by_role["operating-model-architect"]["rate_card"]["draft"] is True
assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"]
# The architect builds the graph with the engine's tools, in this Workflow's shape.
architect = next(
run for run in runtime.runs if run.role == "operating-model-architect"
)
assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools)
# Every Specialist is given the agreed operating spec, with the solution in it.
items = by_role["validator"]["scope"]["stated"]["solution"]["items"]
assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"]
async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused(
tmp_path,
):
import pytest
engine, runtime = make_engine(tmp_path, turns())
with pytest.raises(ValueError, match="commercial.currency"):
await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF"))
assert runtime.runs == []
async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path):
import pytest
engine, _ = make_engine(tmp_path, turns())
with pytest.raises(ValueError, match="commercial.currency"):
await engine.start("", scope=SPEC.replace(" currency: GBP\n", ""))
async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved(
tmp_path,
):
off_hours = graph()
off_hours["roles"][1]["hours"] = "extended"
scripted = turns()
# Refused, and put right in the same run: the customer never sees it.
scripted["operating-model-architect"] = [
[
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}),
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}),
]
]
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
refused, accepted = runtime.runs[3].results
assert (
"Capability C-1 is needed 24x7, and no Role responsible for it works "
"those hours" in refused[1]
)
assert accepted[1] == "Submitted target_operating_model."
assert (
result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0
)
async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path):
stranded = graph()
stranded["responsibilities"] = [
r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
]
scripted = turns()
scripted["operating-model-architect"] = [
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}),
lambda task: (
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH})
if any("SC-party-P-2 failed" in item for item in task.context["feedback"])
else []
),
]
scripted["validator"] = scripted["validator"] * 2
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1
ledger = result.state["ledger"]["deliverables"]
assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1
assert ledger[CAPABILITY_MODEL]["revisions"] == 0
assert [run.role for run in runtime.runs].count("validator") == 2
async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request(
tmp_path,
):
waiting = graph()
waiting["checks"][2]["depends_on_finding"] = "AF-1"
verdicts = json.loads(json.dumps(VERDICTS))
verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
digested = SPEC.replace(
" customer_case: northwind-landing-zone\n",
" customer_case: northwind-landing-zone\n source: architecture:exec-42\n",
)
engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING]))
result = await engine.start("", scope=digested, execution_id="e1")
assert result.status == "completed", result.state.get("warnings")
out = folder(tmp_path, "e1")
record = json.loads((out / "execution.json").read_text())
assert record["findings"] == ["AF-1"]
assert record["change_request"] == "architecture-change-request.md"
request = (out / "architecture-change-request.md").read_text()
assert "digested from `architecture:exec-42`" in request
assert (
"**What is asked for.** Add an alert on a change to the retention lock."
in request
)
assert (
"## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
)
# The Check that waits on the finding is reported with it, and nothing was
# revised for it: the model is not at fault, and neither are the Capabilities.
verdicts = json.loads((out / "verdicts.json").read_text())
waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"]
assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")]
ledger = result.state["ledger"]["deliverables"]
assert all(entry["revisions"] == 0 for entry in ledger.values())
# The solution itself is as it was described: nothing was added to it.
scope = json.loads((out / "scope.json").read_text())
assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [
"S-1",
"S-2",
"S-3",
]
# --- starting from sources ---------------------------------------------------------------
async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path):
from pathlib import Path as _Path
from app.operating_model import digest
fixture = (
_Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
)
engine, runtime = make_engine(
tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]}
)
result = await engine.start(
"",
sources=digest.read_folder(fixture),
# The one essential thing the architecture cannot say: who operates it.
scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n"
" - {id: BO-1, outcome: Payments are never down for long.}\n",
execution_id="e1",
)
assert result.status == "awaiting_user"
assert result.prompt["approval"]["phase"] == "scoping"
assert result.state["customer_case"] == "case-08-landing-zone"
assert "payment platform" in result.state["workflow_request"]
scope = json.loads(
result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"]
)
assert len(scope["stated"]["solution"]["items"]) == 9
assert len(scope["stated"]["solution"]["conditions"]) == 12
assert scope["stated"]["sourcing"] == {"model": "shared"}
# What was digested is shown to the customer as digested, to approve.
register = result.state["deliverables"][OPERATING_SCOPE]["content"]
assert "digested from the sources supplied" in register
# Every source is on record with its hash, and was read by code.
assert set(result.state["sources"]) == set(digest.FILES)
assert all(entry["digested"] for entry in result.state["sources"].values())
assert "source_documents" not in runtime.runs[0].task.context
async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path):
engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]})
document = "# Our platform\n\nWe run a booking service on three clusters.\n"
await engine.start(
"Design the operating model for our booking platform.",
customer_case="bookings",
sources={"design.md": document},
)
context = runtime.runs[0].task.context
assert context["source_documents"] == {"design.md": document}
assert runtime.runs[0].role == "scope-analyst"
async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path):
"""A gap in the operating model's own staffing is not a finding: the validator
fails the Check that waits on it, and the architect designs it instead."""
excuse = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["OC-3"],
"lacks": "The retention review is only staffed in business hours.",
"proposed": "Staff it at every hour.",
}
waiting = graph()
waiting["checks"][2]["depends_on_finding"] = "AF-1"
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(
verdict="fail",
evidence="Staffing is the operating model's to provide; this is not a finding.",
)
scripted = turns(waiting, judged, [excuse])
scripted["operating-model-architect"].append(
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []})
)
scripted["validator"].append(
submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS})
)
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
assert "revise_code" in [d["action"] for d in result.state["decisions"]]
second = [run for run in runtime.runs if run.role == "operating-model-architect"][1]
assert any("not a finding" in item for item in second.task.context["feedback"])
assert graph_findings(result) == []
def graph_findings(result) -> list:
attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"]
return json.loads(attachments.get(FINDINGS_FILE, "[]"))