diff --git a/.scratch/solution-architect-agent/spec.md b/.scratch/solution-architect-agent/spec.md index ed3f012..d5e67f7 100644 --- a/.scratch/solution-architect-agent/spec.md +++ b/.scratch/solution-architect-agent/spec.md @@ -530,3 +530,42 @@ Slowest tests: eval benchmark 7.3 s, then 3โ€“4.5 s each for the orchestrator, e **Still to check** - The default model ids (`gemini/gemini-2.5-pro`, `gemini/gemini-2.5-flash`) come from the spec and haven't been checked against models currently available. Confirm them in increment 5, when LiteLLM is wired. + +### 2026-09-15 ยท Increment 3: Deliverables and Ledger + +**Done (test-first)** +- `app/execution/deliverables.py`: `PhaseId`, `PHASE_ORDER`, `DeliverableKind`, `DeliverableSpec`, `DEPENDENCY_GRAPH` (the table above), `deliverables_in`, and `check_submission(kind, content, attachments) -> missing`. +- `app/execution/ledger.py`: a pure `Ledger`. + - Queries: `ready_work`, `phases_ready_for_approval`, `dependencies_met`, `can_finish`. + - Work: `start`, `submit`, `release`. + - Approvals: `request_approval`, `approve`, `request_changes`. + - Revisions: `reopen`. + - Persistence: `to_dict` / `from_dict`. +- Tests: + - `tests/execution/test_deliverables.py`: 12 tests. + - `tests/execution/test_ledger.py`: 25 tests. + - Full suite: **102 passed**. `ruff` and `mypy` are clean. +- **Checking the tests bite.** I broke each of these in the code on purpose, and in each case at least one test failed: + - cross-Phase dependencies accepting *submitted* + - no cascade on reopen + - the revision cap off by one + - pre-approved mode not granting approval + + A fifth mutation, `release` not bumping the attempt, survived. It showed the extra increments were dead code, so I removed them. Attempts now change only in `start`, and a test covers a late submission from a superseded attempt. + +**Added to or tightened from the spec text** +- **Attempts.** `start` returns an attempt number per Deliverable, and `submit`/`release` must present the current one. This protects against late results from Specialists running in parallel after their work was released or reset by a Revision. The spec had no such token. +- **More event types.** Beyond the spec's list: `started`, `released`, `reset` (a dependent that went back to pending) and `approval_withdrawn`. +- **Return values.** `reopen` returns `ReopenResult(reset, withdrawn_approvals, escalation)`. `reset` lists only Deliverables whose status actually changed. Invalid requests raise `LedgerRefusal` and change nothing, and at the cap `reopen` records `escalated` and changes nothing. `override_cap=True` covers the user deciding to revise past the cap. +- **Changes on approval.** `request_changes(phase, feedback)` returns the Phase to `open` and records the feedback, but doesn't reopen Deliverables itself. The Orchestrator reopens the specific ones with `reopen`. +- **Dependent feedback.** A dependent that is reset gets a feedback note (`Dependency X was revised: `), so its next Specialist run knows why. +- **Single source for progress.** The Ledger owns each Deliverable's status, attempts, revisions and feedback. The Execution state (increment 4) holds only content and attachments, not the `feedback[]`/`revisions` fields the spec's state sketch showed. +- **Contracts,** matched against headings after stripping numbering, case and trailing punctuation, by exact match or a space-separated suffix. That way "4.1. Google Cloud products and features mapping" satisfies "Products and features mapping", but "Non-functional requirements" doesn't satisfy "Functional requirements". + - `requirements`: functional and non-functional requirements, current state, dependencies. + - `product_selection`: products and features mapping. + - `design_recommendations`: the 6 pillar headings. + - `deployment_guidance`: prerequisites and step-by-step instructions, plus attachments `terraform/main.tf` and `terraform/variables.tf`. + - `validation_results`: deployment dry-run, architecture and policy analysis. + - `solution_guide`: the template's 8 top-level sections. + - `architecture_diagram`: attachment `architecture.mmd`. + - `technical_decomposition` and `architecture_description`: non-empty content only. diff --git a/app/execution/__init__.py b/app/execution/__init__.py new file mode 100644 index 0000000..6e70f10 --- /dev/null +++ b/app/execution/__init__.py @@ -0,0 +1 @@ +"""Execution of the solution-architecture workflow: Deliverables, the Ledger and the graph.""" diff --git a/app/execution/deliverables.py b/app/execution/deliverables.py new file mode 100644 index 0000000..0dc3aa0 --- /dev/null +++ b/app/execution/deliverables.py @@ -0,0 +1,174 @@ +"""The dependency graph of Deliverables, grouped into Phases. + +This is the only place Phases, Deliverables and Dependencies are declared. Section +names are provider-neutral; they match the numbered headings of a Cloud provider's +solution-guide template by suffix (e.g. "Products and features mapping" matches +"4.1. Google Cloud products and features mapping"). +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum + + +class PhaseId(StrEnum): + REQUIREMENTS_DISCOVERY = "requirements_discovery" + SOLUTION_ARCHITECTURE = "solution_architecture" + SOLUTION_VALIDATION = "solution_validation" + PACKAGING = "packaging" + + +PHASE_ORDER: tuple[PhaseId, ...] = tuple(PhaseId) + + +class DeliverableKind(StrEnum): + REQUIREMENTS = "requirements" + TECHNICAL_DECOMPOSITION = "technical_decomposition" + PRODUCT_SELECTION = "product_selection" + ARCHITECTURE_DIAGRAM = "architecture_diagram" + ARCHITECTURE_DESCRIPTION = "architecture_description" + DESIGN_RECOMMENDATIONS = "design_recommendations" + DEPLOYMENT_GUIDANCE = "deployment_guidance" + VALIDATION_RESULTS = "validation_results" + SOLUTION_GUIDE = "solution_guide" + + +@dataclass(frozen=True) +class DeliverableSpec: + kind: DeliverableKind + phase: PhaseId + specialist: str + depends_on: tuple[DeliverableKind, ...] = () + sections: tuple[str, ...] = () + attachments: tuple[str, ...] = () + + +_K = DeliverableKind + +DEPENDENCY_GRAPH: Mapping[DeliverableKind, DeliverableSpec] = { + spec.kind: spec + for spec in ( + DeliverableSpec( + _K.REQUIREMENTS, + PhaseId.REQUIREMENTS_DISCOVERY, + "requirements-analyst", + sections=( + "Functional requirements", + "Non-functional requirements", + "Current state", + "Dependencies", + ), + ), + DeliverableSpec( + _K.TECHNICAL_DECOMPOSITION, + PhaseId.REQUIREMENTS_DISCOVERY, + "decomposer", + depends_on=(_K.REQUIREMENTS,), + ), + DeliverableSpec( + _K.PRODUCT_SELECTION, + PhaseId.SOLUTION_ARCHITECTURE, + "product-selector", + depends_on=(_K.TECHNICAL_DECOMPOSITION,), + sections=("Products and features mapping",), + ), + DeliverableSpec( + _K.ARCHITECTURE_DIAGRAM, + PhaseId.SOLUTION_ARCHITECTURE, + "diagrammer", + depends_on=(_K.PRODUCT_SELECTION,), + attachments=("architecture.mmd",), + ), + DeliverableSpec( + _K.ARCHITECTURE_DESCRIPTION, + PhaseId.SOLUTION_ARCHITECTURE, + "describer", + depends_on=(_K.ARCHITECTURE_DIAGRAM, _K.PRODUCT_SELECTION), + ), + DeliverableSpec( + _K.DESIGN_RECOMMENDATIONS, + PhaseId.SOLUTION_ARCHITECTURE, + "design-advisor", + depends_on=(_K.PRODUCT_SELECTION, _K.REQUIREMENTS), + sections=( + "Security, privacy, and compliance", + "Reliability", + "Operational excellence", + "Cost optimization", + "Performance efficiency", + "Sustainability", + ), + ), + DeliverableSpec( + _K.DEPLOYMENT_GUIDANCE, + PhaseId.SOLUTION_ARCHITECTURE, + "iac-author", + depends_on=(_K.PRODUCT_SELECTION, _K.DESIGN_RECOMMENDATIONS), + sections=( + "Deployment prerequisites", + "Step-by-step deployment instructions", + ), + attachments=("terraform/main.tf", "terraform/variables.tf"), + ), + DeliverableSpec( + _K.VALIDATION_RESULTS, + PhaseId.SOLUTION_VALIDATION, + "validator", + depends_on=(_K.DEPLOYMENT_GUIDANCE,), + sections=("Deployment dry-run", "Architecture and policy analysis"), + ), + DeliverableSpec( + _K.SOLUTION_GUIDE, + PhaseId.PACKAGING, + "packager", + depends_on=(_K.VALIDATION_RESULTS,), + sections=( + "Executive summary and workload overview", + "Requirements and current state", + "Technical decomposition of the workload", + "Proposed solution architecture", + "Design and configuration recommendations", + "Deployment guidance", + "Validation plan", + "References", + ), + ), + ) +} + + +def deliverables_in(phase: PhaseId) -> tuple[DeliverableKind, ...]: + return tuple(kind for kind, spec in DEPENDENCY_GRAPH.items() if spec.phase is phase) + + +_HEADING = re.compile(r"^ {0,3}#{1,6}\s+(.+?)\s*#*\s*$", re.MULTILINE) +_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+") +_TRAILING = re.compile(r"[\s:.\-โ€“โ€”]+$") + + +def _normalise(text: str) -> str: + text = _NUMBERING.sub("", text.strip().lower()) + return " ".join(_TRAILING.sub("", text).split()) + + +def check_submission( + kind: DeliverableKind, content: str, attachments: Mapping[str, str] +) -> tuple[str, ...]: + """Return what a submission lacks against its contract; empty means it's complete.""" + if not content.strip(): + return ("content: must not be empty",) + + spec = DEPENDENCY_GRAPH[kind] + headings = [_normalise(match) for match in _HEADING.findall(content)] + missing = [] + for section in spec.sections: + required = _normalise(section) + if not any(h == required or h.endswith(" " + required) for h in headings): + missing.append(f"section: {section}") + for name in spec.attachments: + if name not in attachments: + missing.append(f"attachment: {name}") + return tuple(missing) diff --git a/app/execution/ledger.py b/app/execution/ledger.py new file mode 100644 index 0000000..657c700 --- /dev/null +++ b/app/execution/ledger.py @@ -0,0 +1,317 @@ +"""The Ledger: every rule code owns about Deliverable progress, Approvals and Revisions. + +Pure and synchronous: no I/O, no framework types. Invalid requests raise +LedgerRefusal and leave the Ledger unchanged; tools turn refusals into messages +the agent can act on. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import asdict, dataclass, field +from enum import StrEnum +from typing import Any + +from app.execution.deliverables import ( + DEPENDENCY_GRAPH, + PHASE_ORDER, + DeliverableKind, + PhaseId, + deliverables_in, +) + +PRE_APPROVED_ACTOR = "pre-approved" + + +class DeliverableStatus(StrEnum): + PENDING = "pending" + IN_PROGRESS = "in_progress" + SUBMITTED = "submitted" + + +class PhaseStatus(StrEnum): + OPEN = "open" + AWAITING_APPROVAL = "awaiting_approval" + APPROVED = "approved" + + +class LedgerRefusal(Exception): + """A request the Ledger's rules don't allow. Nothing was changed.""" + + +@dataclass(frozen=True) +class LedgerEvent: + sequence: int + type: str # started | released | submitted | approval_requested | approved | + # changes_requested | reopened | reset | approval_withdrawn | escalated + subject: str # a DeliverableKind or PhaseId value + actor: str + detail: str = "" + + +@dataclass(frozen=True) +class ReopenResult: + reset: tuple[DeliverableKind, ...] + withdrawn_approvals: tuple[PhaseId, ...] + escalation: str | None = None + + +@dataclass +class _DeliverableRecord: + status: DeliverableStatus = DeliverableStatus.PENDING + attempt: int = 0 + revisions: int = 0 + feedback: list[str] = field(default_factory=list) + + +class Ledger: + def __init__(self, pre_approved: bool = False, max_revisions: int = 3) -> None: + self.pre_approved = pre_approved + self.max_revisions = max_revisions + self._deliverables = {kind: _DeliverableRecord() for kind in DEPENDENCY_GRAPH} + self._phases = {phase: PhaseStatus.OPEN for phase in PHASE_ORDER} + self._events: list[LedgerEvent] = [] + + # --- queries ----------------------------------------------------------------- + + @property + def events(self) -> tuple[LedgerEvent, ...]: + return tuple(self._events) + + def status(self, kind: DeliverableKind) -> DeliverableStatus: + return self._deliverables[kind].status + + def phase_status(self, phase: PhaseId) -> PhaseStatus: + return self._phases[phase] + + def revisions(self, kind: DeliverableKind) -> int: + return self._deliverables[kind].revisions + + def feedback(self, kind: DeliverableKind) -> tuple[str, ...]: + return tuple(self._deliverables[kind].feedback) + + def dependencies_met(self, kind: DeliverableKind) -> bool: + spec = DEPENDENCY_GRAPH[kind] + for dependency in spec.depends_on: + dependency_phase = DEPENDENCY_GRAPH[dependency].phase + if dependency_phase is spec.phase: + if self.status(dependency) is not DeliverableStatus.SUBMITTED: + return False + elif self._phases[dependency_phase] is not PhaseStatus.APPROVED: + return False + return True + + def ready_work(self) -> tuple[DeliverableKind, ...]: + return tuple( + kind + for kind in DEPENDENCY_GRAPH + if self.status(kind) is DeliverableStatus.PENDING + and self.dependencies_met(kind) + ) + + def phases_ready_for_approval(self) -> tuple[PhaseId, ...]: + return tuple( + phase + for phase in PHASE_ORDER + if self._phases[phase] is PhaseStatus.OPEN and not self._unsubmitted(phase) + ) + + def can_finish(self) -> bool: + return self._phases[PHASE_ORDER[-1]] is PhaseStatus.APPROVED + + # --- delegation and submission ----------------------------------------------- + + def start( + self, kinds: Iterable[DeliverableKind], actor: str + ) -> dict[DeliverableKind, int]: + """Move ready Deliverables in progress; returns the attempt number for each.""" + kinds = list(dict.fromkeys(kinds)) + ready = set(self.ready_work()) + not_ready = [kind for kind in kinds if kind not in ready] + if not_ready: + raise LedgerRefusal( + "Not ready to start: " + + ", ".join(self._describe_unready(k) for k in not_ready) + ) + attempts = {} + for kind in kinds: + record = self._deliverables[kind] + record.status = DeliverableStatus.IN_PROGRESS + record.attempt += 1 + attempts[kind] = record.attempt + self._record("started", kind, actor, f"attempt {record.attempt}") + return attempts + + def submit(self, kind: DeliverableKind, attempt: int, actor: str) -> None: + """Accept the current attempt. Any later start, release or reopen supersedes it.""" + self._require_current_attempt(kind, attempt) + self._deliverables[kind].status = DeliverableStatus.SUBMITTED + self._record("submitted", kind, actor, f"attempt {attempt}") + + def release(self, kind: DeliverableKind, attempt: int, reason: str) -> None: + """Return in-progress work to pending (e.g. the Specialist needs input).""" + self._require_current_attempt(kind, attempt) + record = self._deliverables[kind] + record.status = DeliverableStatus.PENDING + self._record("released", kind, DEPENDENCY_GRAPH[kind].specialist, reason) + + # --- approvals --------------------------------------------------------------- + + def request_approval(self, phase: PhaseId, actor: str) -> None: + if self._phases[phase] is not PhaseStatus.OPEN: + raise LedgerRefusal(f"Phase {phase} is {self._phases[phase]}, not open") + unsubmitted = self._unsubmitted(phase) + if unsubmitted: + raise LedgerRefusal( + f"Phase {phase} has unsubmitted Deliverables: {', '.join(unsubmitted)}" + ) + self._phases[phase] = PhaseStatus.AWAITING_APPROVAL + self._record("approval_requested", phase, actor) + if self.pre_approved: + self.approve(phase, actor=PRE_APPROVED_ACTOR) + + def approve(self, phase: PhaseId, actor: str) -> None: + self._require_awaiting(phase) + self._phases[phase] = PhaseStatus.APPROVED + self._record("approved", phase, actor) + + def request_changes(self, phase: PhaseId, feedback: str, actor: str) -> None: + self._require_awaiting(phase) + self._phases[phase] = PhaseStatus.OPEN + self._record("changes_requested", phase, actor, feedback) + + # --- revisions --------------------------------------------------------------- + + def reopen( + self, + kind: DeliverableKind, + reason: str, + actor: str, + override_cap: bool = False, + ) -> ReopenResult: + record = self._deliverables[kind] + if record.status is not DeliverableStatus.SUBMITTED: + raise LedgerRefusal( + f"Only submitted Deliverables can be reopened; {kind} is {record.status}" + ) + if record.revisions >= self.max_revisions and not override_cap: + escalation = ( + f"{kind} has reached its revision limit ({self.max_revisions}). " + f"Requested change: {reason}. Ask the user whether to revise it again " + "or accept it as it stands." + ) + self._record("escalated", kind, actor, reason) + return ReopenResult(reset=(), withdrawn_approvals=(), escalation=escalation) + + record.status = DeliverableStatus.PENDING + record.revisions += 1 + record.feedback.append(reason) + self._record("reopened", kind, actor, reason) + reset = [kind] + + for dependent in self._dependents_of(kind): + dependent_record = self._deliverables[dependent] + if dependent_record.status is DeliverableStatus.PENDING: + continue + dependent_record.status = DeliverableStatus.PENDING + dependent_record.feedback.append(f"Dependency {kind} was revised: {reason}") + self._record("reset", dependent, actor, f"dependency {kind} reopened") + reset.append(dependent) + + withdrawn = [] + for phase in PHASE_ORDER: + affected = any(DEPENDENCY_GRAPH[k].phase is phase for k in reset) + if affected and self._phases[phase] is not PhaseStatus.OPEN: + self._phases[phase] = PhaseStatus.OPEN + self._record("approval_withdrawn", phase, actor, f"{kind} reopened") + withdrawn.append(phase) + + return ReopenResult(reset=tuple(reset), withdrawn_approvals=tuple(withdrawn)) + + # --- persistence ------------------------------------------------------------- + + def to_dict(self) -> dict[str, Any]: + return { + "pre_approved": self.pre_approved, + "max_revisions": self.max_revisions, + "deliverables": { + kind.value: { + "status": record.status.value, + "attempt": record.attempt, + "revisions": record.revisions, + "feedback": list(record.feedback), + } + for kind, record in self._deliverables.items() + }, + "phases": { + phase.value: status.value for phase, status in self._phases.items() + }, + "events": [asdict(event) for event in self._events], + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Ledger: + ledger = cls( + pre_approved=data["pre_approved"], max_revisions=data["max_revisions"] + ) + for kind_value, record in data["deliverables"].items(): + ledger._deliverables[DeliverableKind(kind_value)] = _DeliverableRecord( + status=DeliverableStatus(record["status"]), + attempt=record["attempt"], + revisions=record["revisions"], + feedback=list(record["feedback"]), + ) + for phase_value, status in data["phases"].items(): + ledger._phases[PhaseId(phase_value)] = PhaseStatus(status) + ledger._events = [LedgerEvent(**event) for event in data["events"]] + return ledger + + # --- internals --------------------------------------------------------------- + + def _record(self, type_: str, subject: str, actor: str, detail: str = "") -> None: + self._events.append( + LedgerEvent(len(self._events) + 1, type_, str(subject), actor, detail) + ) + + def _unsubmitted(self, phase: PhaseId) -> list[DeliverableKind]: + return [ + kind + for kind in deliverables_in(phase) + if self.status(kind) is not DeliverableStatus.SUBMITTED + ] + + def _dependents_of(self, kind: DeliverableKind) -> list[DeliverableKind]: + """Every Deliverable that depends on `kind`, directly or not, in graph order.""" + dependents: set[DeliverableKind] = set() + frontier = {kind} + while frontier: + frontier = { + candidate + for candidate, spec in DEPENDENCY_GRAPH.items() + if candidate not in dependents and frontier & set(spec.depends_on) + } + dependents |= frontier + return [candidate for candidate in DEPENDENCY_GRAPH if candidate in dependents] + + def _describe_unready(self, kind: DeliverableKind) -> str: + status = self.status(kind) + if status is not DeliverableStatus.PENDING: + return f"{kind} ({status})" + return f"{kind} (dependencies not met)" + + def _require_current_attempt(self, kind: DeliverableKind, attempt: int) -> None: + record = self._deliverables[kind] + if ( + record.status is not DeliverableStatus.IN_PROGRESS + or attempt != record.attempt + ): + raise LedgerRefusal( + f"{kind} is {record.status}, not in_progress with attempt {attempt}; " + "this attempt is not current" + ) + + def _require_awaiting(self, phase: PhaseId) -> None: + if self._phases[phase] is not PhaseStatus.AWAITING_APPROVAL: + raise LedgerRefusal( + f"Phase {phase} is {self._phases[phase]}, not awaiting_approval" + ) diff --git a/tests/execution/__init__.py b/tests/execution/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/execution/test_deliverables.py b/tests/execution/test_deliverables.py new file mode 100644 index 0000000..721580b --- /dev/null +++ b/tests/execution/test_deliverables.py @@ -0,0 +1,155 @@ +"""The dependency graph and Deliverable contracts.""" + +import pytest + +from app.execution.deliverables import ( + DEPENDENCY_GRAPH, + PHASE_ORDER, + DeliverableKind, + PhaseId, + check_submission, + deliverables_in, +) + + +def test_every_kind_is_declared_once_with_known_dependencies(): + assert set(DEPENDENCY_GRAPH) == set(DeliverableKind) + for kind, spec in DEPENDENCY_GRAPH.items(): + assert spec.kind is kind + assert set(spec.depends_on) <= set(DeliverableKind), kind + + +def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle(): + position = {phase: index for index, phase in enumerate(PHASE_ORDER)} + for spec in DEPENDENCY_GRAPH.values(): + for dependency in spec.depends_on: + assert position[DEPENDENCY_GRAPH[dependency].phase] <= position[spec.phase] + + visiting: set[DeliverableKind] = set() + done: set[DeliverableKind] = set() + + def visit(kind: DeliverableKind) -> None: + assert kind not in visiting, f"cycle through {kind}" + if kind in done: + return + visiting.add(kind) + for dependency in DEPENDENCY_GRAPH[kind].depends_on: + visit(dependency) + visiting.discard(kind) + done.add(kind) + + for kind in DeliverableKind: + visit(kind) + + +def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist(): + assert PHASE_ORDER == ( + PhaseId.REQUIREMENTS_DISCOVERY, + PhaseId.SOLUTION_ARCHITECTURE, + PhaseId.SOLUTION_VALIDATION, + PhaseId.PACKAGING, + ) + for phase in PHASE_ORDER: + assert deliverables_in(phase), phase + specialists = [spec.specialist for spec in DEPENDENCY_GRAPH.values()] + assert len(specialists) == len(set(specialists)) + + +def test_graph_matches_the_upstream_skill_dependencies(): + depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()} + K = DeliverableKind + + assert depends[K.REQUIREMENTS] == set() + assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS} + assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION} + assert depends[K.ARCHITECTURE_DIAGRAM] == {K.PRODUCT_SELECTION} + assert depends[K.ARCHITECTURE_DESCRIPTION] == { + K.ARCHITECTURE_DIAGRAM, + K.PRODUCT_SELECTION, + } + assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS} + assert depends[K.DEPLOYMENT_GUIDANCE] == { + K.PRODUCT_SELECTION, + K.DESIGN_RECOMMENDATIONS, + } + assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE} + assert depends[K.SOLUTION_GUIDE] == set( + deliverables_in(PhaseId.SOLUTION_VALIDATION) + ) + + +def test_submission_meeting_the_contract_has_nothing_missing(): + content = ( + "# Deployment guidance\n" + "## 6.1. Deployment prerequisites\nEnable APIs.\n" + "## 6.2. Step-by-step deployment instructions\nRun terraform.\n" + ) + attachments = {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""} + + assert ( + check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments) + == () + ) + + +def test_submission_reports_missing_sections_and_attachments(): + missing = check_submission( + DeliverableKind.DEPLOYMENT_GUIDANCE, + "## Deployment prerequisites\n", + {"terraform/main.tf": "resource {}"}, + ) + + assert missing == ( + "section: Step-by-step deployment instructions", + "attachment: terraform/variables.tf", + ) + + +@pytest.mark.parametrize( + "heading", + [ + "### 5.1. Security, privacy, and compliance", + "## security, privacy, and compliance", + "# 5.1 Security, Privacy, and Compliance:", + ], +) +def test_section_matching_ignores_numbering_case_and_trailing_punctuation(heading): + content = ( + f"{heading}\n" + "## Reliability\n" + "## Operational excellence\n" + "## Cost optimization\n" + "## Performance efficiency\n" + "## Sustainability\n" + ) + + assert check_submission(DeliverableKind.DESIGN_RECOMMENDATIONS, content, {}) == () + + +def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word(): + content = ( + "### 4.1. Google Cloud products and features mapping\n" + "## Non-functional requirements\n" + ) + + assert check_submission(DeliverableKind.PRODUCT_SELECTION, content, {}) == () + assert "section: Functional requirements" in check_submission( + DeliverableKind.REQUIREMENTS, content, {} + ) + + +def test_section_names_in_body_text_do_not_count_as_headings(): + missing = check_submission( + DeliverableKind.REQUIREMENTS, + "Functional requirements, non-functional requirements, current state and " + "dependencies are all covered below.\n", + {}, + ) + + assert "section: Functional requirements" in missing + + +def test_empty_content_is_refused(): + assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == ( + "content: must not be empty", + ) diff --git a/tests/execution/test_ledger.py b/tests/execution/test_ledger.py new file mode 100644 index 0000000..b5f5181 --- /dev/null +++ b/tests/execution/test_ledger.py @@ -0,0 +1,358 @@ +"""The Ledger: Dependency acceptance, Approvals and Revision rules.""" + +import pytest + +from app.execution.deliverables import DeliverableKind as K +from app.execution.deliverables import PhaseId as P +from app.execution.ledger import ( + DeliverableStatus, + Ledger, + LedgerRefusal, + PhaseStatus, +) + + +def produce(ledger: Ledger, *kinds: K) -> None: + """Delegate and submit each kind, in order.""" + for kind in kinds: + attempt = ledger.start([kind], actor="orchestrator")[kind] + ledger.submit(kind, attempt, actor=f"specialist:{kind}") + + +def approve(ledger: Ledger, phase: P) -> None: + ledger.request_approval(phase, actor="orchestrator") + if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL: + ledger.approve(phase, actor="user") + + +def through_phase_one(ledger: Ledger) -> None: + produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) + approve(ledger, P.REQUIREMENTS_DISCOVERY) + + +def through_phase_two(ledger: Ledger) -> None: + through_phase_one(ledger) + produce( + ledger, + K.PRODUCT_SELECTION, + K.ARCHITECTURE_DIAGRAM, + K.ARCHITECTURE_DESCRIPTION, + K.DESIGN_RECOMMENDATIONS, + K.DEPLOYMENT_GUIDANCE, + ) + approve(ledger, P.SOLUTION_ARCHITECTURE) + + +def through_phase_three(ledger: Ledger) -> None: + through_phase_two(ledger) + produce(ledger, K.VALIDATION_RESULTS) + approve(ledger, P.SOLUTION_VALIDATION) + + +# --- ready work and the two acceptance thresholds ------------------------------- + + +def test_new_execution_has_only_requirements_ready(): + ledger = Ledger() + + assert ledger.ready_work() == (K.REQUIREMENTS,) + assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P) + + +def test_same_phase_dependency_is_met_once_submitted(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS) + + assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,) + + +def test_cross_phase_dependency_needs_the_earlier_phase_approved(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) + + assert K.PRODUCT_SELECTION not in ledger.ready_work() + + approve(ledger, P.REQUIREMENTS_DISCOVERY) + + assert ledger.ready_work() == (K.PRODUCT_SELECTION,) + + +def test_independent_work_becomes_ready_together(): + ledger = Ledger() + through_phase_one(ledger) + produce(ledger, K.PRODUCT_SELECTION) + + assert set(ledger.ready_work()) == { + K.ARCHITECTURE_DIAGRAM, + K.DESIGN_RECOMMENDATIONS, + } + + produce(ledger, K.ARCHITECTURE_DIAGRAM) + + assert set(ledger.ready_work()) == { + K.ARCHITECTURE_DESCRIPTION, + K.DESIGN_RECOMMENDATIONS, + } + + +def test_ready_phases_lists_open_phases_with_every_deliverable_submitted(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS) + assert ledger.phases_ready_for_approval() == () + + produce(ledger, K.TECHNICAL_DECOMPOSITION) + + assert ledger.phases_ready_for_approval() == (P.REQUIREMENTS_DISCOVERY,) + + +# --- delegation and submission --------------------------------------------------- + + +def test_start_refuses_work_that_is_not_ready_and_changes_nothing(): + ledger = Ledger() + + with pytest.raises(LedgerRefusal, match="technical_decomposition"): + ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator") + + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING + + +def test_start_moves_ready_work_in_progress_and_returns_attempts(): + ledger = Ledger() + + attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator") + + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.IN_PROGRESS + assert attempts == {K.REQUIREMENTS: 1} + assert ledger.ready_work() == () + + +def test_submit_requires_the_current_attempt(): + ledger = Ledger() + attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS] + + with pytest.raises(LedgerRefusal, match="attempt"): + ledger.submit(K.REQUIREMENTS, attempt + 1, actor="requirements-analyst") + + ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst") + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED + + +def test_late_submission_from_a_superseded_attempt_is_refused_after_restart(): + ledger = Ledger() + first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS] + ledger.release(K.REQUIREMENTS, first, reason="needs input") + second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS] + + with pytest.raises(LedgerRefusal, match="attempt"): + ledger.submit(K.REQUIREMENTS, first, actor="requirements-analyst") + + ledger.submit(K.REQUIREMENTS, second, actor="requirements-analyst") + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED + + +def test_submit_refuses_work_not_in_progress(): + ledger = Ledger() + + with pytest.raises(LedgerRefusal, match="in_progress"): + ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst") + + +def test_release_returns_work_to_pending_without_a_revision(): + ledger = Ledger() + attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS] + + ledger.release(K.REQUIREMENTS, attempt, reason="needs input") + + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING + assert ledger.revisions(K.REQUIREMENTS) == 0 + assert ledger.ready_work() == (K.REQUIREMENTS,) + with pytest.raises(LedgerRefusal, match="attempt"): + ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst") + + +# --- approvals --------------------------------------------------------------------- + + +def test_request_approval_refuses_until_every_deliverable_is_submitted(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS) + + with pytest.raises(LedgerRefusal, match="technical_decomposition"): + ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator") + + assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN + + +def test_approval_moves_through_awaiting_to_approved_by_the_user(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) + + ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator") + assert ( + ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.AWAITING_APPROVAL + ) + + ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user") + + assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED + assert ledger.events[-1].type == "approved" + assert ledger.events[-1].actor == "user" + + +def test_approve_refuses_a_phase_not_awaiting_approval(): + ledger = Ledger() + + with pytest.raises(LedgerRefusal, match="awaiting_approval"): + ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user") + + +def test_pre_approved_ledger_grants_approval_immediately(): + ledger = Ledger(pre_approved=True) + produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) + + ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator") + + assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED + assert [e.type for e in ledger.events[-2:]] == ["approval_requested", "approved"] + assert ledger.events[-1].actor == "pre-approved" + + +def test_request_changes_reopens_the_phase_for_further_work(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) + ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator") + + ledger.request_changes(P.REQUIREMENTS_DISCOVERY, "Add the RTO target", actor="user") + + assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN + assert ledger.events[-1].type == "changes_requested" + assert ledger.events[-1].detail == "Add the RTO target" + + +def test_finish_is_allowed_only_once_packaging_is_approved(): + ledger = Ledger() + through_phase_three(ledger) + assert not ledger.can_finish() + + produce(ledger, K.SOLUTION_GUIDE) + approve(ledger, P.PACKAGING) + + assert ledger.can_finish() + + +# --- revisions --------------------------------------------------------------------- + + +def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals(): + ledger = Ledger() + through_phase_three(ledger) + + result = ledger.reopen( + K.DEPLOYMENT_GUIDANCE, "terraform plan failed", actor="orchestrator" + ) + + assert result.escalation is None + # solution_guide was still pending, so its status doesn't change and it isn't reset. + assert set(result.reset) == {K.DEPLOYMENT_GUIDANCE, K.VALIDATION_RESULTS} + assert ledger.status(K.DEPLOYMENT_GUIDANCE) is DeliverableStatus.PENDING + assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING + assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED + assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED + assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.OPEN + assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN + assert set(result.withdrawn_approvals) == { + P.SOLUTION_ARCHITECTURE, + P.SOLUTION_VALIDATION, + } + assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1 + assert ledger.revisions(K.VALIDATION_RESULTS) == 0 + assert ledger.feedback(K.DEPLOYMENT_GUIDANCE) == ("terraform plan failed",) + assert ledger.feedback(K.VALIDATION_RESULTS) == ( + "Dependency deployment_guidance was revised: terraform plan failed", + ) + assert ledger.ready_work() == (K.DEPLOYMENT_GUIDANCE,) + + +def test_reopen_invalidates_attempts_of_dependents_already_in_progress(): + ledger = Ledger() + through_phase_one(ledger) + produce(ledger, K.PRODUCT_SELECTION) + attempt = ledger.start([K.ARCHITECTURE_DIAGRAM], actor="orchestrator")[ + K.ARCHITECTURE_DIAGRAM + ] + + ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator") + + assert ledger.status(K.ARCHITECTURE_DIAGRAM) is DeliverableStatus.PENDING + with pytest.raises(LedgerRefusal, match="attempt"): + ledger.submit(K.ARCHITECTURE_DIAGRAM, attempt, actor="diagrammer") + + +def test_reopen_refuses_deliverables_that_were_never_submitted(): + ledger = Ledger() + + with pytest.raises(LedgerRefusal, match="submitted"): + ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator") + + +def test_reopen_escalates_at_the_revision_cap_without_changing_state(): + ledger = Ledger(max_revisions=2) + produce(ledger, K.REQUIREMENTS) + for _ in range(2): + ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator") + produce(ledger, K.REQUIREMENTS) + + result = ledger.reopen(K.REQUIREMENTS, "one more", actor="orchestrator") + + assert result.reset == () + assert result.escalation is not None + assert "requirements" in result.escalation + assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED + assert ledger.revisions(K.REQUIREMENTS) == 2 + assert ledger.events[-1].type == "escalated" + + +def test_user_can_override_the_revision_cap(): + ledger = Ledger(max_revisions=1) + produce(ledger, K.REQUIREMENTS) + ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator") + produce(ledger, K.REQUIREMENTS) + + result = ledger.reopen( + K.REQUIREMENTS, "user insists", actor="user", override_cap=True + ) + + assert result.escalation is None + assert ledger.revisions(K.REQUIREMENTS) == 2 + + +def test_default_revision_cap_is_three(): + assert Ledger().max_revisions == 3 + + +# --- records and persistence ------------------------------------------------------- + + +def test_transitions_are_recorded_as_ordered_events(): + ledger = Ledger() + produce(ledger, K.REQUIREMENTS) + + assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [ + (1, "started", "requirements"), + (2, "submitted", "requirements"), + ] + + +def test_round_trips_through_plain_data(): + ledger = Ledger(max_revisions=5) + through_phase_one(ledger) + produce(ledger, K.PRODUCT_SELECTION) + ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user") + + restored = Ledger.from_dict(ledger.to_dict()) + + assert restored.to_dict() == ledger.to_dict() + assert restored.ready_work() == ledger.ready_work() + assert restored.max_revisions == 5 + assert restored.events == ledger.events