Files
orchestration_engine/tests/execution/test_ledger.py

398 lines
13 KiB
Python

"""The Ledger: Dependency acceptance, Approvals and Revision rules."""
import pytest
from execution_engine.states.ledger import (
DeliverableStatus,
Ledger,
LedgerRefusal,
PhaseStatus,
)
from tests.reference_workflow.workflow import WORKFLOW
from tests.reference_workflow.workflow import DeliverableKind as K
from tests.reference_workflow.workflow import PhaseId as P
def produce(ledger: Ledger, *kinds: K) -> None:
"""Delegate and submit each kind, in order."""
for kind in kinds:
attempt = ledger.start([kind], actor="orchestrator")[kind]
ledger.submit(kind, attempt, actor=f"specialist:{kind}")
def approve(ledger: Ledger, phase: P) -> None:
ledger.request_approval(phase, actor="orchestrator")
if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL:
ledger.approve(phase, actor="user")
def scoped(**options) -> Ledger:
"""A Ledger whose scope is agreed: where requirements discovery begins."""
ledger = Ledger(WORKFLOW, **options)
produce(ledger, K.SCOPE)
approve(ledger, P.SCOPING)
return ledger
def through_phase_one(ledger: Ledger) -> None:
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
approve(ledger, P.REQUIREMENTS_DISCOVERY)
def through_phase_two(ledger: Ledger) -> None:
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
approve(ledger, P.SOLUTION_ARCHITECTURE)
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
approve(ledger, P.SOLUTION_IMPLEMENTATION)
def through_phase_three(ledger: Ledger) -> None:
through_phase_two(ledger)
produce(ledger, K.VALIDATION_RESULTS)
approve(ledger, P.SOLUTION_VALIDATION)
# --- ready work and the two acceptance thresholds -------------------------------
def test_a_new_execution_has_only_the_scope_ready():
ledger = Ledger(WORKFLOW)
assert ledger.ready_work() == (K.SCOPE,)
assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P)
def test_once_the_scope_is_approved_requirements_are_ready():
ledger = scoped()
assert ledger.ready_work() == (K.REQUIREMENTS,)
assert ledger.phase_status(P.SCOPING) is PhaseStatus.APPROVED
def test_same_phase_dependency_is_met_once_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,)
def test_cross_phase_dependency_needs_the_earlier_phase_approved():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert K.PRODUCT_SELECTION not in ledger.ready_work()
approve(ledger, P.REQUIREMENTS_DISCOVERY)
assert ledger.ready_work() == (K.PRODUCT_SELECTION,)
def test_independent_work_becomes_ready_together():
ledger = scoped()
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
# Nothing in implementation is ready until the design is approved.
assert ledger.ready_work() == ()
approve(ledger, P.SOLUTION_ARCHITECTURE)
assert set(ledger.ready_work()) == {
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
}
def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.phases_ready_for_approval() == ()
produce(ledger, K.TECHNICAL_DECOMPOSITION)
assert ledger.phases_ready_for_approval() == (P.REQUIREMENTS_DISCOVERY,)
# --- delegation and submission ---------------------------------------------------
def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
def test_start_moves_ready_work_in_progress_and_returns_attempts():
ledger = scoped()
attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.IN_PROGRESS
assert attempts == {K.REQUIREMENTS: 1}
assert ledger.ready_work() == ()
def test_submit_requires_the_current_attempt():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt + 1, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
ledger = scoped()
first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, first, reason="needs input")
second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, first, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, second, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_submit_refuses_work_not_in_progress():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="in_progress"):
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
def test_release_returns_work_to_pending_without_a_revision():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, attempt, reason="needs input")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
assert ledger.revisions(K.REQUIREMENTS) == 0
assert ledger.ready_work() == (K.REQUIREMENTS,)
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
# --- approvals ---------------------------------------------------------------------
def test_request_approval_refuses_until_every_deliverable_is_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
def test_approval_moves_through_awaiting_to_approved_by_the_user():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert (
ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.AWAITING_APPROVAL
)
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert ledger.events[-1].type == "approved"
assert ledger.events[-1].actor == "user"
def test_approve_refuses_a_phase_not_awaiting_approval():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="awaiting_approval"):
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
def test_pre_approved_ledger_grants_approval_immediately():
ledger = scoped(pre_approved=True)
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert [e.type for e in ledger.events[-2:]] == ["approval_requested", "approved"]
assert ledger.events[-1].actor == "pre-approved"
def test_request_changes_reopens_the_phase_for_further_work():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
ledger.request_changes(P.REQUIREMENTS_DISCOVERY, "Add the RTO target", actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
assert ledger.events[-1].type == "changes_requested"
assert ledger.events[-1].detail == "Add the RTO target"
def test_finish_is_allowed_only_once_packaging_is_approved():
ledger = scoped()
through_phase_three(ledger)
assert not ledger.can_finish()
produce(ledger, K.WORK_BREAKDOWN, K.SOLUTION_GUIDE)
approve(ledger, P.PACKAGING)
assert ledger.can_finish()
# --- revisions ---------------------------------------------------------------------
def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.DEPLOYMENT_GUIDANCE, "terraform plan failed", actor="orchestrator"
)
assert result.escalation is None
# solution_guide was still pending, so its status doesn't change and it isn't reset.
assert set(result.reset) == {K.DEPLOYMENT_GUIDANCE, K.VALIDATION_RESULTS}
assert ledger.status(K.DEPLOYMENT_GUIDANCE) is DeliverableStatus.PENDING
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
# Revising the code leaves the approved design standing.
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
assert set(result.withdrawn_approvals) == {
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
assert ledger.revisions(K.VALIDATION_RESULTS) == 0
assert ledger.feedback(K.DEPLOYMENT_GUIDANCE) == ("terraform plan failed",)
assert ledger.feedback(K.VALIDATION_RESULTS) == (
"Dependency deployment_guidance was revised: terraform plan failed",
)
assert ledger.ready_work() == (K.DEPLOYMENT_GUIDANCE,)
def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
ledger = scoped()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
K.DESIGN_RECOMMENDATIONS
]
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
def test_reopen_refuses_deliverables_that_were_never_submitted():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="submitted"):
ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator")
def test_reopen_escalates_at_the_revision_cap_without_changing_state():
ledger = scoped(max_revisions=2)
produce(ledger, K.REQUIREMENTS)
for _ in range(2):
ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(K.REQUIREMENTS, "one more", actor="orchestrator")
assert result.reset == ()
assert result.escalation is not None
assert "requirements" in result.escalation
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
assert ledger.revisions(K.REQUIREMENTS) == 2
assert ledger.events[-1].type == "escalated"
def test_user_can_override_the_revision_cap():
ledger = scoped(max_revisions=1)
produce(ledger, K.REQUIREMENTS)
ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(
K.REQUIREMENTS, "user insists", actor="user", override_cap=True
)
assert result.escalation is None
assert ledger.revisions(K.REQUIREMENTS) == 2
def test_default_revision_cap_is_three():
assert scoped().max_revisions == 3
# --- records and persistence -------------------------------------------------------
def test_transitions_are_recorded_as_ordered_events():
ledger = Ledger(WORKFLOW)
produce(ledger, K.SCOPE)
assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [
(1, "started", "scope"),
(2, "submitted", "scope"),
]
def test_round_trips_through_plain_data():
ledger = scoped(max_revisions=5)
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user")
restored = Ledger.from_dict(WORKFLOW, ledger.to_dict())
assert restored.to_dict() == ledger.to_dict()
assert restored.ready_work() == ledger.ready_work()
assert restored.max_revisions == 5
assert restored.events == ledger.events
def test_revising_the_target_architecture_returns_the_code_to_draft():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
)
assert set(result.reset) == {
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
K.VALIDATION_RESULTS,
}
assert set(result.withdrawn_approvals) == {
P.SOLUTION_ARCHITECTURE,
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
# The code cannot be rewritten until the revised design is approved again.
produce(ledger, K.TARGET_ARCHITECTURE)
assert ledger.ready_work() == ()