"""Whole Executions of the target operating model Workflow, on the scripted runtime.""" import json from pathlib import Path from typing import Any from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime from execution_engine.adk.roles import load_roles from execution_engine.adk.runtime import NoDecider from execution_engine.agent import ExecutionEngine from execution_engine.nodes.router import Router, Thresholds from execution_engine.recorder import InMemoryRecorder from langgraph.checkpoint.memory import InMemorySaver from app.operating_model.names import ( CAPABILITIES_FILE, CAPABILITY_MODEL, FINDINGS_FILE, GRAPH_FILE, GUIDE, OPERATING_SCOPE, OPERATIONAL_REQUIREMENTS, REQUIREMENTS_FILE, TARGET_OPERATING_MODEL, VALIDATION_RESULTS, VERDICTS_FILE, ) from app.operating_model.workflow import ROOT from tests.support import ( BUNDLE, CAPABILITIES, GRAPH, REQUIREMENTS, SPEC, VERDICTS, WORKFLOW, graph, ) ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW) CASE = "northwind-landing-zone" FINDING = { "id": "AF-1", "about": ["S-2"], "needed_by": ["C-3"], "lacks": "Nothing alerts when the audit log store's retention lock is removed.", "proposed": "Add an alert on a change to the retention lock.", } def submit(kind: str, **files: Any) -> list[Call]: """One turn that submits a Deliverable with every section of its contract.""" spec = WORKFLOW.deliverables[kind] content = f"# {kind}\n" + "".join( f"\n## {section}\n\nDetails.\n" for section in spec.sections ) return [ Call( "submit_deliverable", { "content": content, "attachments": {name: json.dumps(data) for name, data in files.items()}, }, ) ] def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]: model_files = {GRAPH_FILE: model} if findings is not None: model_files[FINDINGS_FILE] = findings return { "scope-analyst": [submit(OPERATING_SCOPE)], "operations-analyst": [ submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS}) ], "capability-modeller": [ submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES}) ], "operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)], "validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})], "packager": [submit(GUIDE)], } def make_engine( tmp_path: Path, scripted: dict[str, list[Any]] ) -> tuple[ExecutionEngine, FakeAgentRuntime]: runtime = FakeAgentRuntime(scripted) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), scope_catalogue=BUNDLE.scope_catalogue(), scope_values=BUNDLE.scope_values, scope_skill=BUNDLE.scope_skill, ) return engine, runtime def folder(tmp_path: Path, execution_id: str) -> Path: return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path): engine, runtime = make_engine(tmp_path, turns()) result = await engine.start("", scope=SPEC, execution_id="e1") assert result.status == "completed", result.state.get("warnings") assert [run.role for run in runtime.runs] == [ "scope-analyst", "operations-analyst", "capability-modeller", "operating-model-architect", "validator", "packager", ] out = folder(tmp_path, "e1") written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()} assert { "execution.json", "operating-model-guide.md", "scope.yaml", "scope.json", "operational-requirements.json", "capabilities.json", "target-operating-model.json", "findings.json", "staffing.json", "organisation.mmd", "verdicts.json", "scenarios.json", } <= written record = json.loads((out / "execution.json").read_text()) assert record["customer_case"] == CASE assert record["final_document"] == "operating-model-guide.md" assert record["staffing"] == { "people": 5.1, "cost_per_month": 43410, "currency": "GBP", "rate_card": { "name": "rate-card", "version": "draft-2026-10-13", "draft": True, }, } assert record["findings"] == [] assert record["capability_sources_verified"] is False async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path): engine, _ = make_engine(tmp_path, turns()) result = await engine.start("", scope=SPEC) model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"] assert "| Incident management (C-1) | C | R | A |" in model assert "Cost a month: 43,410 GBP." in model validation = result.state["deliverables"][VALIDATION_RESULTS] assert "## Scenario walk-throughs" in validation["content"] verdicts = json.loads(validation["attachments"][VERDICTS_FILE]) assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass." assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10 async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path): engine, runtime = make_engine(tmp_path, turns()) await engine.start("", scope=SPEC) by_role = {run.role: run.task.context for run in runtime.runs} proposed = by_role["capability-modeller"]["capabilities_proposed"] assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set( proposed ) assert by_role["operating-model-architect"]["rate_card"]["draft"] is True assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"] # The architect builds the graph with the engine's tools, in this Workflow's shape. architect = next( run for run in runtime.runs if run.role == "operating-model-architect" ) assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools) # Every Specialist is given the agreed operating spec, with the solution in it. items = by_role["validator"]["scope"]["stated"]["solution"]["items"] assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"] async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused( tmp_path, ): import pytest engine, runtime = make_engine(tmp_path, turns()) with pytest.raises(ValueError, match="commercial.currency"): await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF")) assert runtime.runs == [] async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path): import pytest engine, _ = make_engine(tmp_path, turns()) with pytest.raises(ValueError, match="commercial.currency"): await engine.start("", scope=SPEC.replace(" currency: GBP\n", "")) async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved( tmp_path, ): off_hours = graph() off_hours["roles"][1]["hours"] = "extended" scripted = turns() # Refused, and put right in the same run: the customer never sees it. scripted["operating-model-architect"] = [ [ *submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}), *submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}), ] ] engine, runtime = make_engine(tmp_path, scripted) result = await engine.start("", scope=SPEC) assert result.status == "completed", result.state.get("warnings") refused, accepted = runtime.runs[3].results assert ( "Capability C-1 is needed 24x7, and no Role responsible for it works " "those hours" in refused[1] ) assert accepted[1] == "Submitted target_operating_model." assert ( result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0 ) async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path): stranded = graph() stranded["responsibilities"] = [ r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8") ] scripted = turns() scripted["operating-model-architect"] = [ submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}), lambda task: ( submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}) if any("SC-party-P-2 failed" in item for item in task.context["feedback"]) else [] ), ] scripted["validator"] = scripted["validator"] * 2 engine, runtime = make_engine(tmp_path, scripted) result = await engine.start("", scope=SPEC) assert result.status == "completed", result.state.get("warnings") assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1 ledger = result.state["ledger"]["deliverables"] assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1 assert ledger[CAPABILITY_MODEL]["revisions"] == 0 assert [run.role for run in runtime.runs].count("validator") == 2 async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request( tmp_path, ): waiting = graph() waiting["checks"][2]["depends_on_finding"] = "AF-1" verdicts = json.loads(json.dumps(VERDICTS)) verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.") digested = SPEC.replace( " customer_case: northwind-landing-zone\n", " customer_case: northwind-landing-zone\n source: architecture:exec-42\n", ) engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING])) result = await engine.start("", scope=digested, execution_id="e1") assert result.status == "completed", result.state.get("warnings") out = folder(tmp_path, "e1") record = json.loads((out / "execution.json").read_text()) assert record["findings"] == ["AF-1"] assert record["change_request"] == "architecture-change-request.md" request = (out / "architecture-change-request.md").read_text() assert "digested from `architecture:exec-42`" in request assert ( "**What is asked for.** Add an alert on a change to the retention lock." in request ) assert ( "## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"] ) # The Check that waits on the finding is reported with it, and nothing was # revised for it: the model is not at fault, and neither are the Capabilities. verdicts = json.loads((out / "verdicts.json").read_text()) waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"] assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")] ledger = result.state["ledger"]["deliverables"] assert all(entry["revisions"] == 0 for entry in ledger.values()) # The solution itself is as it was described: nothing was added to it. scope = json.loads((out / "scope.json").read_text()) assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [ "S-1", "S-2", "S-3", ] # --- starting from sources --------------------------------------------------------------- async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path): from pathlib import Path as _Path from app.operating_model import digest fixture = ( _Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone" ) engine, runtime = make_engine( tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]} ) result = await engine.start( "", sources=digest.read_folder(fixture), # The one essential thing the architecture cannot say: who operates it. scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n" " - {id: BO-1, outcome: Payments are never down for long.}\n", execution_id="e1", ) assert result.status == "awaiting_user" assert result.prompt["approval"]["phase"] == "scoping" assert result.state["customer_case"] == "case-08-landing-zone" assert "payment platform" in result.state["workflow_request"] scope = json.loads( result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"] ) assert len(scope["stated"]["solution"]["items"]) == 9 assert len(scope["stated"]["solution"]["conditions"]) == 12 assert scope["stated"]["sourcing"] == {"model": "shared"} # What was digested is shown to the customer as digested, to approve. register = result.state["deliverables"][OPERATING_SCOPE]["content"] assert "digested from the sources supplied" in register # Every source is on record with its hash, and was read by code. assert set(result.state["sources"]) == set(digest.FILES) assert all(entry["digested"] for entry in result.state["sources"].values()) assert "source_documents" not in runtime.runs[0].task.context async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path): engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]}) document = "# Our platform\n\nWe run a booking service on three clusters.\n" await engine.start( "Design the operating model for our booking platform.", customer_case="bookings", sources={"design.md": document}, ) context = runtime.runs[0].task.context assert context["source_documents"] == {"design.md": document} assert runtime.runs[0].role == "scope-analyst" async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path): """A gap in the operating model's own staffing is not a finding: the validator fails the Check that waits on it, and the architect designs it instead.""" excuse = { "id": "AF-1", "about": ["S-2"], "needed_by": ["OC-3"], "lacks": "The retention review is only staffed in business hours.", "proposed": "Staff it at every hour.", } waiting = graph() waiting["checks"][2]["depends_on_finding"] = "AF-1" judged = json.loads(json.dumps(VERDICTS)) judged["checks"][2].update( verdict="fail", evidence="Staffing is the operating model's to provide; this is not a finding.", ) scripted = turns(waiting, judged, [excuse]) scripted["operating-model-architect"].append( submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []}) ) scripted["validator"].append( submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS}) ) engine, runtime = make_engine(tmp_path, scripted) result = await engine.start("", scope=SPEC) assert result.status == "completed", result.state.get("warnings") assert "revise_code" in [d["action"] for d in result.state["decisions"]] second = [run for run in runtime.runs if run.role == "operating-model-architect"][1] assert any("not a finding" in item for item in second.task.context["feedback"]) assert graph_findings(result) == [] def graph_findings(result) -> list: attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"] return json.loads(attachments.get(FINDINGS_FILE, "[]"))