412 lines
15 KiB
Python
412 lines
15 KiB
Python
"""Whole Executions of the target operating model Workflow, on the scripted runtime."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.roles import load_roles
|
|
from execution_engine.adk.runtime import NoDecider
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.nodes.router import Router, Thresholds
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.operating_model.names import (
|
|
CAPABILITIES_FILE,
|
|
CAPABILITY_MODEL,
|
|
FINDINGS_FILE,
|
|
GRAPH_FILE,
|
|
GUIDE,
|
|
OPERATING_SCOPE,
|
|
OPERATIONAL_REQUIREMENTS,
|
|
REQUIREMENTS_FILE,
|
|
TARGET_OPERATING_MODEL,
|
|
VALIDATION_RESULTS,
|
|
VERDICTS_FILE,
|
|
)
|
|
from app.operating_model.workflow import ROOT
|
|
from tests.support import (
|
|
BUNDLE,
|
|
CAPABILITIES,
|
|
GRAPH,
|
|
REQUIREMENTS,
|
|
SPEC,
|
|
VERDICTS,
|
|
WORKFLOW,
|
|
graph,
|
|
)
|
|
|
|
ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW)
|
|
CASE = "northwind-landing-zone"
|
|
FINDING = {
|
|
"id": "AF-1",
|
|
"about": ["S-2"],
|
|
"needed_by": ["C-3"],
|
|
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
|
|
"proposed": "Add an alert on a change to the retention lock.",
|
|
}
|
|
|
|
|
|
def submit(kind: str, **files: Any) -> list[Call]:
|
|
"""One turn that submits a Deliverable with every section of its contract."""
|
|
spec = WORKFLOW.deliverables[kind]
|
|
content = f"# {kind}\n" + "".join(
|
|
f"\n## {section}\n\nDetails.\n" for section in spec.sections
|
|
)
|
|
return [
|
|
Call(
|
|
"submit_deliverable",
|
|
{
|
|
"content": content,
|
|
"attachments": {name: json.dumps(data) for name, data in files.items()},
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]:
|
|
model_files = {GRAPH_FILE: model}
|
|
if findings is not None:
|
|
model_files[FINDINGS_FILE] = findings
|
|
return {
|
|
"scope-analyst": [submit(OPERATING_SCOPE)],
|
|
"operations-analyst": [
|
|
submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS})
|
|
],
|
|
"capability-modeller": [
|
|
submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES})
|
|
],
|
|
"operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)],
|
|
"validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})],
|
|
"packager": [submit(GUIDE)],
|
|
}
|
|
|
|
|
|
def make_engine(
|
|
tmp_path: Path, scripted: dict[str, list[Any]]
|
|
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
|
runtime = FakeAgentRuntime(scripted)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
scope_catalogue=BUNDLE.scope_catalogue(),
|
|
scope_values=BUNDLE.scope_values,
|
|
scope_skill=BUNDLE.scope_skill,
|
|
)
|
|
return engine, runtime
|
|
|
|
|
|
def folder(tmp_path: Path, execution_id: str) -> Path:
|
|
return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
|
|
|
|
|
|
async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path):
|
|
engine, runtime = make_engine(tmp_path, turns())
|
|
|
|
result = await engine.start("", scope=SPEC, execution_id="e1")
|
|
|
|
assert result.status == "completed", result.state.get("warnings")
|
|
assert [run.role for run in runtime.runs] == [
|
|
"scope-analyst",
|
|
"operations-analyst",
|
|
"capability-modeller",
|
|
"operating-model-architect",
|
|
"validator",
|
|
"packager",
|
|
]
|
|
out = folder(tmp_path, "e1")
|
|
written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
|
|
assert {
|
|
"execution.json",
|
|
"operating-model-guide.md",
|
|
"scope.yaml",
|
|
"scope.json",
|
|
"operational-requirements.json",
|
|
"capabilities.json",
|
|
"target-operating-model.json",
|
|
"findings.json",
|
|
"staffing.json",
|
|
"organisation.mmd",
|
|
"verdicts.json",
|
|
"scenarios.json",
|
|
} <= written
|
|
record = json.loads((out / "execution.json").read_text())
|
|
assert record["customer_case"] == CASE
|
|
assert record["final_document"] == "operating-model-guide.md"
|
|
assert record["staffing"] == {
|
|
"people": 5.1,
|
|
"cost_per_month": 43410,
|
|
"currency": "GBP",
|
|
"rate_card": {
|
|
"name": "rate-card",
|
|
"version": "draft-2026-10-13",
|
|
"draft": True,
|
|
},
|
|
}
|
|
assert record["findings"] == []
|
|
assert record["capability_sources_verified"] is False
|
|
|
|
|
|
async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path):
|
|
engine, _ = make_engine(tmp_path, turns())
|
|
|
|
result = await engine.start("", scope=SPEC)
|
|
|
|
model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
|
assert "| Incident management (C-1) | C | R | A |" in model
|
|
assert "Cost a month: 43,410 GBP." in model
|
|
validation = result.state["deliverables"][VALIDATION_RESULTS]
|
|
assert "## Scenario walk-throughs" in validation["content"]
|
|
verdicts = json.loads(validation["attachments"][VERDICTS_FILE])
|
|
assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass."
|
|
assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10
|
|
|
|
|
|
async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path):
|
|
engine, runtime = make_engine(tmp_path, turns())
|
|
|
|
await engine.start("", scope=SPEC)
|
|
|
|
by_role = {run.role: run.task.context for run in runtime.runs}
|
|
proposed = by_role["capability-modeller"]["capabilities_proposed"]
|
|
assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set(
|
|
proposed
|
|
)
|
|
assert by_role["operating-model-architect"]["rate_card"]["draft"] is True
|
|
assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"]
|
|
# The architect builds the graph with the engine's tools, in this Workflow's shape.
|
|
architect = next(
|
|
run for run in runtime.runs if run.role == "operating-model-architect"
|
|
)
|
|
assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools)
|
|
# Every Specialist is given the agreed operating spec, with the solution in it.
|
|
items = by_role["validator"]["scope"]["stated"]["solution"]["items"]
|
|
assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"]
|
|
|
|
|
|
async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused(
|
|
tmp_path,
|
|
):
|
|
import pytest
|
|
|
|
engine, runtime = make_engine(tmp_path, turns())
|
|
|
|
with pytest.raises(ValueError, match="commercial.currency"):
|
|
await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF"))
|
|
|
|
assert runtime.runs == []
|
|
|
|
|
|
async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path):
|
|
import pytest
|
|
|
|
engine, _ = make_engine(tmp_path, turns())
|
|
|
|
with pytest.raises(ValueError, match="commercial.currency"):
|
|
await engine.start("", scope=SPEC.replace(" currency: GBP\n", ""))
|
|
|
|
|
|
async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved(
|
|
tmp_path,
|
|
):
|
|
off_hours = graph()
|
|
off_hours["roles"][1]["hours"] = "extended"
|
|
scripted = turns()
|
|
# Refused, and put right in the same run: the customer never sees it.
|
|
scripted["operating-model-architect"] = [
|
|
[
|
|
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}),
|
|
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}),
|
|
]
|
|
]
|
|
engine, runtime = make_engine(tmp_path, scripted)
|
|
|
|
result = await engine.start("", scope=SPEC)
|
|
|
|
assert result.status == "completed", result.state.get("warnings")
|
|
refused, accepted = runtime.runs[3].results
|
|
assert (
|
|
"Capability C-1 is needed 24x7, and no Role responsible for it works "
|
|
"those hours" in refused[1]
|
|
)
|
|
assert accepted[1] == "Submitted target_operating_model."
|
|
assert (
|
|
result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0
|
|
)
|
|
|
|
|
|
async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path):
|
|
stranded = graph()
|
|
stranded["responsibilities"] = [
|
|
r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
|
|
]
|
|
scripted = turns()
|
|
scripted["operating-model-architect"] = [
|
|
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}),
|
|
lambda task: (
|
|
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH})
|
|
if any("SC-party-P-2 failed" in item for item in task.context["feedback"])
|
|
else []
|
|
),
|
|
]
|
|
scripted["validator"] = scripted["validator"] * 2
|
|
engine, runtime = make_engine(tmp_path, scripted)
|
|
|
|
result = await engine.start("", scope=SPEC)
|
|
|
|
assert result.status == "completed", result.state.get("warnings")
|
|
assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1
|
|
ledger = result.state["ledger"]["deliverables"]
|
|
assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1
|
|
assert ledger[CAPABILITY_MODEL]["revisions"] == 0
|
|
assert [run.role for run in runtime.runs].count("validator") == 2
|
|
|
|
|
|
async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request(
|
|
tmp_path,
|
|
):
|
|
waiting = graph()
|
|
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
|
verdicts = json.loads(json.dumps(VERDICTS))
|
|
verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
|
|
digested = SPEC.replace(
|
|
" customer_case: northwind-landing-zone\n",
|
|
" customer_case: northwind-landing-zone\n source: architecture:exec-42\n",
|
|
)
|
|
engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING]))
|
|
|
|
result = await engine.start("", scope=digested, execution_id="e1")
|
|
|
|
assert result.status == "completed", result.state.get("warnings")
|
|
out = folder(tmp_path, "e1")
|
|
record = json.loads((out / "execution.json").read_text())
|
|
assert record["findings"] == ["AF-1"]
|
|
assert record["change_request"] == "architecture-change-request.md"
|
|
request = (out / "architecture-change-request.md").read_text()
|
|
assert "digested from `architecture:exec-42`" in request
|
|
assert (
|
|
"**What is asked for.** Add an alert on a change to the retention lock."
|
|
in request
|
|
)
|
|
assert (
|
|
"## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
|
)
|
|
# The Check that waits on the finding is reported with it, and nothing was
|
|
# revised for it: the model is not at fault, and neither are the Capabilities.
|
|
verdicts = json.loads((out / "verdicts.json").read_text())
|
|
waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"]
|
|
assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")]
|
|
ledger = result.state["ledger"]["deliverables"]
|
|
assert all(entry["revisions"] == 0 for entry in ledger.values())
|
|
# The solution itself is as it was described: nothing was added to it.
|
|
scope = json.loads((out / "scope.json").read_text())
|
|
assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [
|
|
"S-1",
|
|
"S-2",
|
|
"S-3",
|
|
]
|
|
|
|
|
|
# --- starting from sources ---------------------------------------------------------------
|
|
|
|
|
|
async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path):
|
|
from pathlib import Path as _Path
|
|
|
|
from app.operating_model import digest
|
|
|
|
fixture = (
|
|
_Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
|
)
|
|
engine, runtime = make_engine(
|
|
tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]}
|
|
)
|
|
|
|
result = await engine.start(
|
|
"",
|
|
sources=digest.read_folder(fixture),
|
|
# The one essential thing the architecture cannot say: who operates it.
|
|
scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n"
|
|
" - {id: BO-1, outcome: Payments are never down for long.}\n",
|
|
execution_id="e1",
|
|
)
|
|
|
|
assert result.status == "awaiting_user"
|
|
assert result.prompt["approval"]["phase"] == "scoping"
|
|
assert result.state["customer_case"] == "case-08-landing-zone"
|
|
assert "payment platform" in result.state["workflow_request"]
|
|
scope = json.loads(
|
|
result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"]
|
|
)
|
|
assert len(scope["stated"]["solution"]["items"]) == 9
|
|
assert len(scope["stated"]["solution"]["conditions"]) == 12
|
|
assert scope["stated"]["sourcing"] == {"model": "shared"}
|
|
# What was digested is shown to the customer as digested, to approve.
|
|
register = result.state["deliverables"][OPERATING_SCOPE]["content"]
|
|
assert "digested from the sources supplied" in register
|
|
# Every source is on record with its hash, and was read by code.
|
|
assert set(result.state["sources"]) == set(digest.FILES)
|
|
assert all(entry["digested"] for entry in result.state["sources"].values())
|
|
assert "source_documents" not in runtime.runs[0].task.context
|
|
|
|
|
|
async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path):
|
|
engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]})
|
|
document = "# Our platform\n\nWe run a booking service on three clusters.\n"
|
|
|
|
await engine.start(
|
|
"Design the operating model for our booking platform.",
|
|
customer_case="bookings",
|
|
sources={"design.md": document},
|
|
)
|
|
|
|
context = runtime.runs[0].task.context
|
|
assert context["source_documents"] == {"design.md": document}
|
|
assert runtime.runs[0].role == "scope-analyst"
|
|
|
|
|
|
async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path):
|
|
"""A gap in the operating model's own staffing is not a finding: the validator
|
|
fails the Check that waits on it, and the architect designs it instead."""
|
|
excuse = {
|
|
"id": "AF-1",
|
|
"about": ["S-2"],
|
|
"needed_by": ["OC-3"],
|
|
"lacks": "The retention review is only staffed in business hours.",
|
|
"proposed": "Staff it at every hour.",
|
|
}
|
|
waiting = graph()
|
|
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
|
judged = json.loads(json.dumps(VERDICTS))
|
|
judged["checks"][2].update(
|
|
verdict="fail",
|
|
evidence="Staffing is the operating model's to provide; this is not a finding.",
|
|
)
|
|
scripted = turns(waiting, judged, [excuse])
|
|
scripted["operating-model-architect"].append(
|
|
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []})
|
|
)
|
|
scripted["validator"].append(
|
|
submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS})
|
|
)
|
|
engine, runtime = make_engine(tmp_path, scripted)
|
|
|
|
result = await engine.start("", scope=SPEC)
|
|
|
|
assert result.status == "completed", result.state.get("warnings")
|
|
assert "revise_code" in [d["action"] for d in result.state["decisions"]]
|
|
second = [run for run in runtime.runs if run.role == "operating-model-architect"][1]
|
|
assert any("not a finding" in item for item in second.task.context["feedback"])
|
|
assert graph_findings(result) == []
|
|
|
|
|
|
def graph_findings(result) -> list:
|
|
attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"]
|
|
return json.loads(attachments.get(FINDINGS_FILE, "[]"))
|