feat: working TOM agent
This commit is contained in:
411
tests/test_execution.py
Normal file
411
tests/test_execution.py
Normal file
@@ -0,0 +1,411 @@
|
||||
"""Whole Executions of the target operating model Workflow, on the scripted runtime."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.roles import load_roles
|
||||
from execution_engine.adk.runtime import NoDecider
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.operating_model.names import (
|
||||
CAPABILITIES_FILE,
|
||||
CAPABILITY_MODEL,
|
||||
FINDINGS_FILE,
|
||||
GRAPH_FILE,
|
||||
GUIDE,
|
||||
OPERATING_SCOPE,
|
||||
OPERATIONAL_REQUIREMENTS,
|
||||
REQUIREMENTS_FILE,
|
||||
TARGET_OPERATING_MODEL,
|
||||
VALIDATION_RESULTS,
|
||||
VERDICTS_FILE,
|
||||
)
|
||||
from app.operating_model.workflow import ROOT
|
||||
from tests.support import (
|
||||
BUNDLE,
|
||||
CAPABILITIES,
|
||||
GRAPH,
|
||||
REQUIREMENTS,
|
||||
SPEC,
|
||||
VERDICTS,
|
||||
WORKFLOW,
|
||||
graph,
|
||||
)
|
||||
|
||||
ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW)
|
||||
CASE = "northwind-landing-zone"
|
||||
FINDING = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["C-3"],
|
||||
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
|
||||
"proposed": "Add an alert on a change to the retention lock.",
|
||||
}
|
||||
|
||||
|
||||
def submit(kind: str, **files: Any) -> list[Call]:
|
||||
"""One turn that submits a Deliverable with every section of its contract."""
|
||||
spec = WORKFLOW.deliverables[kind]
|
||||
content = f"# {kind}\n" + "".join(
|
||||
f"\n## {section}\n\nDetails.\n" for section in spec.sections
|
||||
)
|
||||
return [
|
||||
Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": content,
|
||||
"attachments": {name: json.dumps(data) for name, data in files.items()},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]:
|
||||
model_files = {GRAPH_FILE: model}
|
||||
if findings is not None:
|
||||
model_files[FINDINGS_FILE] = findings
|
||||
return {
|
||||
"scope-analyst": [submit(OPERATING_SCOPE)],
|
||||
"operations-analyst": [
|
||||
submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS})
|
||||
],
|
||||
"capability-modeller": [
|
||||
submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES})
|
||||
],
|
||||
"operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)],
|
||||
"validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})],
|
||||
"packager": [submit(GUIDE)],
|
||||
}
|
||||
|
||||
|
||||
def make_engine(
|
||||
tmp_path: Path, scripted: dict[str, list[Any]]
|
||||
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
||||
runtime = FakeAgentRuntime(scripted)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
scope_catalogue=BUNDLE.scope_catalogue(),
|
||||
scope_values=BUNDLE.scope_values,
|
||||
scope_skill=BUNDLE.scope_skill,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
def folder(tmp_path: Path, execution_id: str) -> Path:
|
||||
return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
|
||||
|
||||
|
||||
async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
result = await engine.start("", scope=SPEC, execution_id="e1")
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"scope-analyst",
|
||||
"operations-analyst",
|
||||
"capability-modeller",
|
||||
"operating-model-architect",
|
||||
"validator",
|
||||
"packager",
|
||||
]
|
||||
out = folder(tmp_path, "e1")
|
||||
written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
|
||||
assert {
|
||||
"execution.json",
|
||||
"operating-model-guide.md",
|
||||
"scope.yaml",
|
||||
"scope.json",
|
||||
"operational-requirements.json",
|
||||
"capabilities.json",
|
||||
"target-operating-model.json",
|
||||
"findings.json",
|
||||
"staffing.json",
|
||||
"organisation.mmd",
|
||||
"verdicts.json",
|
||||
"scenarios.json",
|
||||
} <= written
|
||||
record = json.loads((out / "execution.json").read_text())
|
||||
assert record["customer_case"] == CASE
|
||||
assert record["final_document"] == "operating-model-guide.md"
|
||||
assert record["staffing"] == {
|
||||
"people": 5.1,
|
||||
"cost_per_month": 43410,
|
||||
"currency": "GBP",
|
||||
"rate_card": {
|
||||
"name": "rate-card",
|
||||
"version": "draft-2026-10-13",
|
||||
"draft": True,
|
||||
},
|
||||
}
|
||||
assert record["findings"] == []
|
||||
assert record["capability_sources_verified"] is False
|
||||
|
||||
|
||||
async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, turns())
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
||||
assert "| Incident management (C-1) | C | R | A |" in model
|
||||
assert "Cost a month: 43,410 GBP." in model
|
||||
validation = result.state["deliverables"][VALIDATION_RESULTS]
|
||||
assert "## Scenario walk-throughs" in validation["content"]
|
||||
verdicts = json.loads(validation["attachments"][VERDICTS_FILE])
|
||||
assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass."
|
||||
assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10
|
||||
|
||||
|
||||
async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
await engine.start("", scope=SPEC)
|
||||
|
||||
by_role = {run.role: run.task.context for run in runtime.runs}
|
||||
proposed = by_role["capability-modeller"]["capabilities_proposed"]
|
||||
assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set(
|
||||
proposed
|
||||
)
|
||||
assert by_role["operating-model-architect"]["rate_card"]["draft"] is True
|
||||
assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"]
|
||||
# The architect builds the graph with the engine's tools, in this Workflow's shape.
|
||||
architect = next(
|
||||
run for run in runtime.runs if run.role == "operating-model-architect"
|
||||
)
|
||||
assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools)
|
||||
# Every Specialist is given the agreed operating spec, with the solution in it.
|
||||
items = by_role["validator"]["scope"]["stated"]["solution"]["items"]
|
||||
assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"]
|
||||
|
||||
|
||||
async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused(
|
||||
tmp_path,
|
||||
):
|
||||
import pytest
|
||||
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
with pytest.raises(ValueError, match="commercial.currency"):
|
||||
await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF"))
|
||||
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path):
|
||||
import pytest
|
||||
|
||||
engine, _ = make_engine(tmp_path, turns())
|
||||
|
||||
with pytest.raises(ValueError, match="commercial.currency"):
|
||||
await engine.start("", scope=SPEC.replace(" currency: GBP\n", ""))
|
||||
|
||||
|
||||
async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved(
|
||||
tmp_path,
|
||||
):
|
||||
off_hours = graph()
|
||||
off_hours["roles"][1]["hours"] = "extended"
|
||||
scripted = turns()
|
||||
# Refused, and put right in the same run: the customer never sees it.
|
||||
scripted["operating-model-architect"] = [
|
||||
[
|
||||
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}),
|
||||
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}),
|
||||
]
|
||||
]
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
refused, accepted = runtime.runs[3].results
|
||||
assert (
|
||||
"Capability C-1 is needed 24x7, and no Role responsible for it works "
|
||||
"those hours" in refused[1]
|
||||
)
|
||||
assert accepted[1] == "Submitted target_operating_model."
|
||||
assert (
|
||||
result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0
|
||||
)
|
||||
|
||||
|
||||
async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path):
|
||||
stranded = graph()
|
||||
stranded["responsibilities"] = [
|
||||
r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
|
||||
]
|
||||
scripted = turns()
|
||||
scripted["operating-model-architect"] = [
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}),
|
||||
lambda task: (
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH})
|
||||
if any("SC-party-P-2 failed" in item for item in task.context["feedback"])
|
||||
else []
|
||||
),
|
||||
]
|
||||
scripted["validator"] = scripted["validator"] * 2
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1
|
||||
assert ledger[CAPABILITY_MODEL]["revisions"] == 0
|
||||
assert [run.role for run in runtime.runs].count("validator") == 2
|
||||
|
||||
|
||||
async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request(
|
||||
tmp_path,
|
||||
):
|
||||
waiting = graph()
|
||||
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
verdicts = json.loads(json.dumps(VERDICTS))
|
||||
verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
|
||||
digested = SPEC.replace(
|
||||
" customer_case: northwind-landing-zone\n",
|
||||
" customer_case: northwind-landing-zone\n source: architecture:exec-42\n",
|
||||
)
|
||||
engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING]))
|
||||
|
||||
result = await engine.start("", scope=digested, execution_id="e1")
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
out = folder(tmp_path, "e1")
|
||||
record = json.loads((out / "execution.json").read_text())
|
||||
assert record["findings"] == ["AF-1"]
|
||||
assert record["change_request"] == "architecture-change-request.md"
|
||||
request = (out / "architecture-change-request.md").read_text()
|
||||
assert "digested from `architecture:exec-42`" in request
|
||||
assert (
|
||||
"**What is asked for.** Add an alert on a change to the retention lock."
|
||||
in request
|
||||
)
|
||||
assert (
|
||||
"## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
||||
)
|
||||
# The Check that waits on the finding is reported with it, and nothing was
|
||||
# revised for it: the model is not at fault, and neither are the Capabilities.
|
||||
verdicts = json.loads((out / "verdicts.json").read_text())
|
||||
waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"]
|
||||
assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")]
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert all(entry["revisions"] == 0 for entry in ledger.values())
|
||||
# The solution itself is as it was described: nothing was added to it.
|
||||
scope = json.loads((out / "scope.json").read_text())
|
||||
assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [
|
||||
"S-1",
|
||||
"S-2",
|
||||
"S-3",
|
||||
]
|
||||
|
||||
|
||||
# --- starting from sources ---------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path):
|
||||
from pathlib import Path as _Path
|
||||
|
||||
from app.operating_model import digest
|
||||
|
||||
fixture = (
|
||||
_Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
||||
)
|
||||
engine, runtime = make_engine(
|
||||
tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]}
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"",
|
||||
sources=digest.read_folder(fixture),
|
||||
# The one essential thing the architecture cannot say: who operates it.
|
||||
scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n"
|
||||
" - {id: BO-1, outcome: Payments are never down for long.}\n",
|
||||
execution_id="e1",
|
||||
)
|
||||
|
||||
assert result.status == "awaiting_user"
|
||||
assert result.prompt["approval"]["phase"] == "scoping"
|
||||
assert result.state["customer_case"] == "case-08-landing-zone"
|
||||
assert "payment platform" in result.state["workflow_request"]
|
||||
scope = json.loads(
|
||||
result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"]
|
||||
)
|
||||
assert len(scope["stated"]["solution"]["items"]) == 9
|
||||
assert len(scope["stated"]["solution"]["conditions"]) == 12
|
||||
assert scope["stated"]["sourcing"] == {"model": "shared"}
|
||||
# What was digested is shown to the customer as digested, to approve.
|
||||
register = result.state["deliverables"][OPERATING_SCOPE]["content"]
|
||||
assert "digested from the sources supplied" in register
|
||||
# Every source is on record with its hash, and was read by code.
|
||||
assert set(result.state["sources"]) == set(digest.FILES)
|
||||
assert all(entry["digested"] for entry in result.state["sources"].values())
|
||||
assert "source_documents" not in runtime.runs[0].task.context
|
||||
|
||||
|
||||
async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]})
|
||||
document = "# Our platform\n\nWe run a booking service on three clusters.\n"
|
||||
|
||||
await engine.start(
|
||||
"Design the operating model for our booking platform.",
|
||||
customer_case="bookings",
|
||||
sources={"design.md": document},
|
||||
)
|
||||
|
||||
context = runtime.runs[0].task.context
|
||||
assert context["source_documents"] == {"design.md": document}
|
||||
assert runtime.runs[0].role == "scope-analyst"
|
||||
|
||||
|
||||
async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path):
|
||||
"""A gap in the operating model's own staffing is not a finding: the validator
|
||||
fails the Check that waits on it, and the architect designs it instead."""
|
||||
excuse = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["OC-3"],
|
||||
"lacks": "The retention review is only staffed in business hours.",
|
||||
"proposed": "Staff it at every hour.",
|
||||
}
|
||||
waiting = graph()
|
||||
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
judged = json.loads(json.dumps(VERDICTS))
|
||||
judged["checks"][2].update(
|
||||
verdict="fail",
|
||||
evidence="Staffing is the operating model's to provide; this is not a finding.",
|
||||
)
|
||||
scripted = turns(waiting, judged, [excuse])
|
||||
scripted["operating-model-architect"].append(
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []})
|
||||
)
|
||||
scripted["validator"].append(
|
||||
submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS})
|
||||
)
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert "revise_code" in [d["action"] for d in result.state["decisions"]]
|
||||
second = [run for run in runtime.runs if run.role == "operating-model-architect"][1]
|
||||
assert any("not a finding" in item for item in second.task.context["feedback"])
|
||||
assert graph_findings(result) == []
|
||||
|
||||
|
||||
def graph_findings(result) -> list:
|
||||
attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"]
|
||||
return json.loads(attachments.get(FINDINGS_FILE, "[]"))
|
||||
Reference in New Issue
Block a user