Solution architecture outcome pattern: agent and its MCP servers
Some checks failed
check agents / check (push) Failing after 13s

The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate
stream from the individual and sub-agent patterns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-05 22:13:25 +00:00
parent 8bb62a4a42
commit bc84ee4df6
267 changed files with 44287 additions and 45 deletions

View File

@@ -0,0 +1,157 @@
"""Sample submissions for the core's tests, from the source's fake runtime."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any
from app.core.deliverables import DEPENDENCY_GRAPH, DeliverableKind, ProviderContract
CONTRACT = ProviderContract(
("Security, privacy, and compliance", "Reliability", "Operational excellence",
"Cost optimization", "Performance efficiency", "Sustainability"),
("terraform/main.tf", "terraform/variables.tf"),
)
@dataclass(frozen=True)
class Call:
tool: str
args: Mapping[str, Any] = field(default_factory=dict)
SAMPLE_DATA: dict[str, str] = {
"scope.yaml": "engagement:\n kind: workload\nintent:\n statement: Accept events.\n",
"scope.json": json.dumps(
{
"kind": "workload",
"stated": {"intent": {"statement": "Accept events."}},
"to_decide": [],
"to_ask": [],
"not_applicable": [],
}
),
"requirements.json": json.dumps(
[
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
{
"id": "NFR-1",
"kind": "non_functional",
"text": "99.9% availability.",
"conditions": ["The service stays available when one instance fails."],
},
]
),
"components.json": json.dumps(
[{"id": "C-1", "name": "Ingress", "requirements": ["FR-1", "NFR-1"]}]
),
"products.json": json.dumps([{"component": "C-1", "product": "Example Run"}]),
"recommendations.json": json.dumps(
[
{
"id": "DR-1",
"pillar": "security",
"text": "Use a dedicated identity.",
"components": ["C-1"],
"conditions": ["The service runs as an identity nothing else uses."],
}
]
),
"target-architecture.json": json.dumps(
{
"elements": [
{
"id": "E-1",
"name": "Ingress service",
"component": "C-1",
"product": "Example Run",
"resource_type": "example_run_service",
},
{
"id": "E-2",
"name": "Ingress identity",
"product": "Example IAM",
"resource_type": "example_service_account",
},
],
"relationships": [
{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "runs as"}
],
"checks": [
{
"id": "K-1",
"applies_to": "E-1",
"statement": "The service runs at least two instances.",
"source": "NFR-1",
},
{
"id": "K-2",
"applies_to": "E-2",
"statement": "The identity is used by this service only.",
"source": "DR-1",
},
],
}
),
"implementation.json": json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "example.run"},
{"check": "K-2", "file": "terraform/main.tf", "address": "example.sa"},
]
),
"work-breakdown.json": json.dumps(
{
"work_packages": [
{
"id": "WP-1",
"name": "Ingress",
"parent": None,
"elements": ["E-1", "E-2"],
"scope": "The service and its identity.",
"produces": ["a running service"],
"skills": ["platform engineering"],
}
],
"activities": [
{
"id": "A-1",
"work_package": "WP-1",
"name": "Create the identity",
"days": 1,
"after": [],
},
{
"id": "A-2",
"work_package": "WP-1",
"name": "Deploy the service",
"days": 2,
"after": ["A-1"],
},
],
}
),
"verdicts.json": json.dumps(
{
"dry_run": {"verdict": "pass", "evidence": "terraform validate passed."},
"checks": [
{"check": "K-1", "verdict": "pass", "evidence": "min instances is 2."},
{"check": "K-2", "verdict": "pass", "evidence": "One binding."},
],
"proposed_additions": [],
}
),
}
def sample_submission(kind: DeliverableKind, note: str = "") -> Call:
"""A `submit_deliverable` call that meets the Deliverable's contract."""
spec = DEPENDENCY_GRAPH[kind]
parts = [f"# {kind}", note or "Summary."]
parts += [f"## {section}\n{note or 'Details.'}" for section in CONTRACT.sections(kind)]
attachments = {
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in CONTRACT.attachments(kind)
}
return Call(
"submit_deliverable",
{"content": "\n".join(parts) + "\n", "attachments": attachments},
)

View File

@@ -0,0 +1,325 @@
"""The dependency graph and Deliverable contracts."""
import pytest
from tests.core.samples import CONTRACT
from app.core.deliverables import (
ProviderContract,
DEPENDENCY_GRAPH,
PHASE_ORDER,
DeliverableKind,
PhaseId,
check_submission,
deliverables_in,
inputs_of,
)
def test_every_kind_is_declared_once_with_known_dependencies():
assert set(DEPENDENCY_GRAPH) == set(DeliverableKind)
for kind, spec in DEPENDENCY_GRAPH.items():
assert spec.kind is kind
assert set(spec.depends_on) <= set(DeliverableKind), kind
def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle():
position = {phase: index for index, phase in enumerate(PHASE_ORDER)}
for spec in DEPENDENCY_GRAPH.values():
for dependency in spec.depends_on:
assert position[DEPENDENCY_GRAPH[dependency].phase] <= position[spec.phase]
visiting: set[DeliverableKind] = set()
done: set[DeliverableKind] = set()
def visit(kind: DeliverableKind) -> None:
assert kind not in visiting, f"cycle through {kind}"
if kind in done:
return
visiting.add(kind)
for dependency in DEPENDENCY_GRAPH[kind].depends_on:
visit(dependency)
visiting.discard(kind)
done.add(kind)
for kind in DeliverableKind:
visit(kind)
def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist():
assert PHASE_ORDER == (
PhaseId.SCOPING,
PhaseId.REQUIREMENTS_DISCOVERY,
PhaseId.SOLUTION_ARCHITECTURE,
PhaseId.SOLUTION_IMPLEMENTATION,
PhaseId.SOLUTION_VALIDATION,
PhaseId.PACKAGING,
)
for phase in PHASE_ORDER:
assert deliverables_in(phase), phase
specialists = [spec.specialist for spec in DEPENDENCY_GRAPH.values()]
assert len(specialists) == len(set(specialists))
def test_the_target_architecture_sits_between_the_design_and_the_code():
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
K = DeliverableKind
assert depends[K.SCOPE] == set()
assert depends[K.REQUIREMENTS] == {K.SCOPE}
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
assert depends[K.TARGET_ARCHITECTURE] == {
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
}
# Everything after the graph is derived from it.
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
assert depends[K.SOLUTION_GUIDE] == set(
deliverables_in(PhaseId.SOLUTION_VALIDATION)
)
def test_the_design_is_approved_before_code_is_written():
design = PhaseId.SOLUTION_ARCHITECTURE
implementation = PhaseId.SOLUTION_IMPLEMENTATION
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
design = (
DeliverableKind.SCOPE,
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.DESIGN_RECOMMENDATIONS,
DeliverableKind.TARGET_ARCHITECTURE,
)
assert inputs_of(DeliverableKind.SCOPE) == ()
assert inputs_of(DeliverableKind.REQUIREMENTS) == (DeliverableKind.SCOPE,)
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:5]
# The plan is produced beside the guide, not before it: neither reads the other.
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
DeliverableKind.SOLUTION_GUIDE,
DeliverableKind.WORK_BREAKDOWN,
}
assert DeliverableKind.SOLUTION_GUIDE not in inputs_of(
DeliverableKind.WORK_BREAKDOWN
)
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
def test_submission_meeting_the_contract_has_nothing_missing():
content = (
"# Deployment guidance\n"
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
)
attachments = {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "variable {}",
"implementation.json": "[]",
}
assert (
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
== ()
)
def test_submission_reports_missing_sections_and_attachments():
missing = check_submission(
DeliverableKind.DEPLOYMENT_GUIDANCE,
"## Deployment prerequisites\n",
{"terraform/main.tf": "resource {}"},
contract=CONTRACT,
)
assert missing == (
"section: Step-by-step deployment instructions",
"attachment: terraform/variables.tf",
"attachment: implementation.json",
)
@pytest.mark.parametrize(
"heading",
[
"### 5.1. Security, privacy, and compliance",
"## security, privacy, and compliance",
"# 5.1 Security, Privacy, and Compliance:",
],
)
def test_section_matching_ignores_numbering_case_and_trailing_punctuation(heading):
content = (
f"{heading}\n"
"## Reliability\n"
"## Operational excellence\n"
"## Cost optimization\n"
"## Performance efficiency\n"
"## Sustainability\n"
)
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
content = (
"### 4.1. Google Cloud products and features mapping\n"
"## Non-functional requirements\n"
)
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
assert "section: Functional requirements" in check_submission(
DeliverableKind.REQUIREMENTS, content, {}
)
def test_section_names_in_body_text_do_not_count_as_headings():
missing = check_submission(
DeliverableKind.REQUIREMENTS,
"Functional requirements, non-functional requirements, current state and "
"dependencies are all covered below.\n",
{},
)
assert "section: Functional requirements" in missing
def test_empty_content_is_refused():
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
"content: must not be empty",
)
# --- contracts beyond headings ---------------------------------------------------------
def test_architecture_description_needs_component_and_flow_sections():
missing = check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
)
assert missing == ("section: Components", "section: Flow")
assert (
check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION,
"## Architecture components\nA.\n## Data flow\nB.\n",
{},
)
== ()
)
def test_a_required_attachment_must_have_content():
missing = check_submission(
DeliverableKind.VALIDATION_RESULTS,
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
{"verdicts.json": " \n"},
)
assert missing == ("attachment: verdicts.json must not be empty",)
# --- what a Specialist is shown of what it is given --------------------------------------
def given_for(kind: DeliverableKind) -> dict:
from tests.core.samples import sample_submission
from app.core.deliverables import dependencies_for
submitted = {}
for each in DeliverableKind:
call = sample_submission(each)
submitted[each.value] = {
"content": call.args["content"],
"attachments": call.args["attachments"],
}
review = "\n## Architecture framework checklist review\n| a | b |\n"
submitted["design_recommendations"]["content"] += review
return dependencies_for(kind, submitted)
def test_a_specialist_is_shown_only_what_it_uses():
from app.core.deliverables import shown
author = shown(
DeliverableKind.DEPLOYMENT_GUIDANCE,
given_for(DeliverableKind.DEPLOYMENT_GUIDANCE),
)
validator = shown(
DeliverableKind.VALIDATION_RESULTS,
given_for(DeliverableKind.VALIDATION_RESULTS),
)
# The graph is all the code is written from.
assert list(author) == ["target_architecture"]
# The validator judges code against Checks, and reads requirements and
# recommendations for coverage; the prose describing the design is not judged.
assert "architecture_description" not in validator
assert {"requirements", "design_recommendations", "deployment_guidance"} <= set(
validator
)
def test_nothing_is_taken_from_what_a_submission_is_checked_against():
"""Showing less changes what the model reads, never what the code checks."""
given = given_for(DeliverableKind.DEPLOYMENT_GUIDANCE)
assert {"requirements", "design_recommendations", "scope"} <= set(given)
def test_the_scope_is_shown_once_and_only_its_prose_to_who_works_from_it():
from app.core.deliverables import shown
analyst = shown(
DeliverableKind.REQUIREMENTS, given_for(DeliverableKind.REQUIREMENTS)
)
architect = shown(
DeliverableKind.TARGET_ARCHITECTURE,
given_for(DeliverableKind.TARGET_ARCHITECTURE),
)
assert list(analyst["scope"]) == ["content"]
assert "scope" not in architect
def test_a_section_written_for_a_reader_is_left_off_except_for_the_guide():
REVIEW_HEADING = "Architecture framework checklist review"
from app.core.deliverables import FOR_READERS, shown
assert FOR_READERS[DeliverableKind.DESIGN_RECOMMENDATIONS] == REVIEW_HEADING
architect = shown(
DeliverableKind.TARGET_ARCHITECTURE,
given_for(DeliverableKind.TARGET_ARCHITECTURE),
)
packager = shown(
DeliverableKind.SOLUTION_GUIDE, given_for(DeliverableKind.SOLUTION_GUIDE)
)
assert REVIEW_HEADING not in architect["design_recommendations"]["content"]
assert "## Design" in architect["design_recommendations"]["content"] or architect[
"design_recommendations"
]["content"].startswith("# design_recommendations")
assert REVIEW_HEADING in packager["design_recommendations"]["content"]
def test_a_provider_brings_its_own_pillars_and_code_files():
five = ProviderContract(("Security", "Reliability", "Cost Optimization", "Operational Excellence", "Performance Efficiency"), ("infra/main.tf",))
content = "# d\n## Security\nx\n"
missing = check_submission(DeliverableKind.DESIGN_RECOMMENDATIONS, content, {}, contract=five)
assert "section: Reliability" in missing and "section: Sustainability" not in missing
assert five.attachments(DeliverableKind.DEPLOYMENT_GUIDANCE)[0] == "infra/main.tf"

View File

@@ -0,0 +1,396 @@
"""The Ledger: Dependency acceptance, Approvals and Revision rules."""
import pytest
from app.core.ledger import (
DeliverableStatus,
Ledger,
LedgerRefusal,
PhaseStatus,
)
from app.core.deliverables import DeliverableKind as K
from app.core.deliverables import PhaseId as P
def produce(ledger: Ledger, *kinds: K) -> None:
"""Delegate and submit each kind, in order."""
for kind in kinds:
attempt = ledger.start([kind], actor="orchestrator")[kind]
ledger.submit(kind, attempt, actor=f"specialist:{kind}")
def approve(ledger: Ledger, phase: P) -> None:
ledger.request_approval(phase, actor="orchestrator")
if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL:
ledger.approve(phase, actor="user")
def scoped(**options) -> Ledger:
"""A Ledger whose scope is agreed: where requirements discovery begins."""
ledger = Ledger(**options)
produce(ledger, K.SCOPE)
approve(ledger, P.SCOPING)
return ledger
def through_phase_one(ledger: Ledger) -> None:
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
approve(ledger, P.REQUIREMENTS_DISCOVERY)
def through_phase_two(ledger: Ledger) -> None:
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
approve(ledger, P.SOLUTION_ARCHITECTURE)
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
approve(ledger, P.SOLUTION_IMPLEMENTATION)
def through_phase_three(ledger: Ledger) -> None:
through_phase_two(ledger)
produce(ledger, K.VALIDATION_RESULTS)
approve(ledger, P.SOLUTION_VALIDATION)
# --- ready work and the two acceptance thresholds -------------------------------
def test_a_new_execution_has_only_the_scope_ready():
ledger = Ledger()
assert ledger.ready_work() == (K.SCOPE,)
assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P)
def test_once_the_scope_is_approved_requirements_are_ready():
ledger = scoped()
assert ledger.ready_work() == (K.REQUIREMENTS,)
assert ledger.phase_status(P.SCOPING) is PhaseStatus.APPROVED
def test_same_phase_dependency_is_met_once_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,)
def test_cross_phase_dependency_needs_the_earlier_phase_approved():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert K.PRODUCT_SELECTION not in ledger.ready_work()
approve(ledger, P.REQUIREMENTS_DISCOVERY)
assert ledger.ready_work() == (K.PRODUCT_SELECTION,)
def test_independent_work_becomes_ready_together():
ledger = scoped()
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
# Nothing in implementation is ready until the design is approved.
assert ledger.ready_work() == ()
approve(ledger, P.SOLUTION_ARCHITECTURE)
assert set(ledger.ready_work()) == {
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
}
def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.phases_ready_for_approval() == ()
produce(ledger, K.TECHNICAL_DECOMPOSITION)
assert ledger.phases_ready_for_approval() == (P.REQUIREMENTS_DISCOVERY,)
# --- delegation and submission ---------------------------------------------------
def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
def test_start_moves_ready_work_in_progress_and_returns_attempts():
ledger = scoped()
attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.IN_PROGRESS
assert attempts == {K.REQUIREMENTS: 1}
assert ledger.ready_work() == ()
def test_submit_requires_the_current_attempt():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt + 1, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
ledger = scoped()
first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, first, reason="needs input")
second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, first, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, second, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_submit_refuses_work_not_in_progress():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="in_progress"):
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
def test_release_returns_work_to_pending_without_a_revision():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, attempt, reason="needs input")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
assert ledger.revisions(K.REQUIREMENTS) == 0
assert ledger.ready_work() == (K.REQUIREMENTS,)
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
# --- approvals ---------------------------------------------------------------------
def test_request_approval_refuses_until_every_deliverable_is_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
def test_approval_moves_through_awaiting_to_approved_by_the_user():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert (
ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.AWAITING_APPROVAL
)
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert ledger.events[-1].type == "approved"
assert ledger.events[-1].actor == "user"
def test_approve_refuses_a_phase_not_awaiting_approval():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="awaiting_approval"):
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
def test_pre_approved_ledger_grants_approval_immediately():
ledger = scoped(pre_approved=True)
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert [e.type for e in ledger.events[-2:]] == ["approval_requested", "approved"]
assert ledger.events[-1].actor == "pre-approved"
def test_request_changes_reopens_the_phase_for_further_work():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
ledger.request_changes(P.REQUIREMENTS_DISCOVERY, "Add the RTO target", actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
assert ledger.events[-1].type == "changes_requested"
assert ledger.events[-1].detail == "Add the RTO target"
def test_finish_is_allowed_only_once_packaging_is_approved():
ledger = scoped()
through_phase_three(ledger)
assert not ledger.can_finish()
produce(ledger, K.WORK_BREAKDOWN, K.SOLUTION_GUIDE)
approve(ledger, P.PACKAGING)
assert ledger.can_finish()
# --- revisions ---------------------------------------------------------------------
def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.DEPLOYMENT_GUIDANCE, "terraform plan failed", actor="orchestrator"
)
assert result.escalation is None
# solution_guide was still pending, so its status doesn't change and it isn't reset.
assert set(result.reset) == {K.DEPLOYMENT_GUIDANCE, K.VALIDATION_RESULTS}
assert ledger.status(K.DEPLOYMENT_GUIDANCE) is DeliverableStatus.PENDING
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
# Revising the code leaves the approved design standing.
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
assert set(result.withdrawn_approvals) == {
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
assert ledger.revisions(K.VALIDATION_RESULTS) == 0
assert ledger.feedback(K.DEPLOYMENT_GUIDANCE) == ("terraform plan failed",)
assert ledger.feedback(K.VALIDATION_RESULTS) == (
"Dependency deployment_guidance was revised: terraform plan failed",
)
assert ledger.ready_work() == (K.DEPLOYMENT_GUIDANCE,)
def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
ledger = scoped()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
K.DESIGN_RECOMMENDATIONS
]
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
def test_reopen_refuses_deliverables_that_were_never_submitted():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="submitted"):
ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator")
def test_reopen_escalates_at_the_revision_cap_without_changing_state():
ledger = scoped(max_revisions=2)
produce(ledger, K.REQUIREMENTS)
for _ in range(2):
ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(K.REQUIREMENTS, "one more", actor="orchestrator")
assert result.reset == ()
assert result.escalation is not None
assert "requirements" in result.escalation
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
assert ledger.revisions(K.REQUIREMENTS) == 2
assert ledger.events[-1].type == "escalated"
def test_user_can_override_the_revision_cap():
ledger = scoped(max_revisions=1)
produce(ledger, K.REQUIREMENTS)
ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(
K.REQUIREMENTS, "user insists", actor="user", override_cap=True
)
assert result.escalation is None
assert ledger.revisions(K.REQUIREMENTS) == 2
def test_default_revision_cap_is_three():
assert scoped().max_revisions == 3
# --- records and persistence -------------------------------------------------------
def test_transitions_are_recorded_as_ordered_events():
ledger = Ledger()
produce(ledger, K.SCOPE)
assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [
(1, "started", "scope"),
(2, "submitted", "scope"),
]
def test_round_trips_through_plain_data():
ledger = scoped(max_revisions=5)
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user")
restored = Ledger.from_dict(ledger.to_dict())
assert restored.to_dict() == ledger.to_dict()
assert restored.ready_work() == ledger.ready_work()
assert restored.max_revisions == 5
assert restored.events == ledger.events
def test_revising_the_target_architecture_returns_the_code_to_draft():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
)
assert set(result.reset) == {
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
K.VALIDATION_RESULTS,
}
assert set(result.withdrawn_approvals) == {
P.SOLUTION_ARCHITECTURE,
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
# The code cannot be rewritten until the revised design is approved again.
produce(ledger, K.TARGET_ARCHITECTURE)
assert ledger.ready_work() == ()

View File

@@ -0,0 +1,290 @@
"""The scope spec (ADR-0008): read against the catalogue, with no model involved."""
from pathlib import Path
import pytest
import yaml
from app.core.providers import load_provider
from app.core.scope import (
APPROVED,
ASK,
DRAFT,
ESSENTIAL,
UNDER_REVIEW,
Catalogue,
FieldSpec,
ScopeError,
read_scope,
render_reading,
render_template,
slug,
)
CATALOGUE = Catalogue(
schema_version=1,
kinds=("workload", "platform_foundation"),
fields={
spec.path: spec
for spec in (
FieldSpec(
"metadata.customer_case",
tier=ESSENTIAL,
aliases=("metadata.engagement_id",),
),
FieldSpec(
"engagement.kind",
type="enum",
values=("workload", "platform_foundation"),
tier=ESSENTIAL,
),
FieldSpec("intent.statement", type="text", tier=ESSENTIAL),
FieldSpec(
"intent.requirements",
type="records",
keys=("id", "statement"),
optional_keys=("category",),
),
FieldSpec(
"users.count",
type="integer",
tier={"workload": ESSENTIAL},
question="How many users?",
),
FieldSpec(
"users.peak",
type="integer",
tier="important",
question="How many at once?",
default="a tenth of the users",
affects="scaling limits",
),
FieldSpec("estate.region", tier="important", question="Which region?"),
FieldSpec("network.topology", type="enum", provider_values=True),
FieldSpec("delivery.start", type="date"),
FieldSpec("run.pre_approved", type="boolean"),
)
},
)
COMPLETE = """
metadata: {customer_case: acme-bookings}
engagement: {kind: workload}
intent:
statement: Staff book rooms.
requirements:
- {id: REQ-01, statement: Rooms can be booked.}
users: {count: 400, peak: 40}
estate: {region: europe-west2}
"""
def read(text: str, **options):
return read_scope(text, CATALOGUE, **options)
def test_a_complete_spec_has_nothing_to_ask():
scope = read(COMPLETE)
assert scope.kind == "workload" and scope.customer_case == "acme-bookings"
assert scope.statement == "Staff book rooms."
assert scope.gaps(ESSENTIAL, "important") == []
assert scope.values["users.count"] == 400
assert (
scope.status() == UNDER_REVIEW and scope.status(pre_approved=True) == APPROVED
)
def test_a_field_left_out_or_marked_ask_is_a_gap_with_its_question():
scope = read(COMPLETE.replace("count: 400, peak: 40", "peak: ask"))
assert scope.gaps() == [
{"field": "users.count", "prompt": "How many users?", "severity": "HIGH"},
{"field": "users.peak", "prompt": "How many at once?", "severity": "MEDIUM"},
# Optional fields nobody filled are gaps too, the least pressing.
*[gap for gap in scope.gaps() if gap["severity"] == "LOW"],
]
assert scope.status(pre_approved=True) == DRAFT
def test_decide_is_an_assumption_and_not_applicable_is_neither():
scope = read(
COMPLETE.replace("peak: 40", "peak: decide").replace(
"estate: {region: europe-west2}", "estate: not_applicable"
)
)
(assumed,) = scope.assumptions()
assert assumed["id"] == "ASSUMP-01" and assumed["field"] == "users.peak"
assert "a tenth of the users" in assumed["statement"]
assert assumed["impact_if_invalid"] == "Changes scaling limits."
# A whole section marked at once covers every field in it.
assert scope.states["estate.region"] == "not_applicable"
assert all(gap["field"] != "estate.region" for gap in scope.gaps())
def test_which_fields_are_essential_depends_on_the_kind_of_engagement():
foundation = read(
COMPLETE.replace("workload", "platform_foundation").replace(
"users: {count: 400, peak: 40}", ""
)
)
assert foundation.gaps(ESSENTIAL) == []
assert foundation.tier("users.count") == "optional"
def test_an_unknown_field_is_named_never_ignored():
with pytest.raises(ScopeError) as refused:
read(COMPLETE + "userz: {count: 1}\nestate: {regoin: x}\n")
assert "userz is not a field of the scope spec" in refused.value.problems
# The second estate key replaced the first; its typo is reported too.
assert any("estate.regoin" in problem for problem in refused.value.problems)
@pytest.mark.parametrize(
("change", "fault"),
[
(("count: 400", "count: many"), "users.count: must be a whole number"),
(("kind: workload", "kind: holiday"), "must be one of workload"),
(
("{id: REQ-01, statement: Rooms can be booked.}", "{id: REQ-01}"),
"needs statement",
),
(("estate: {region: europe-west2}", "estate: europe"), "estate is a section"),
],
)
def test_a_value_of_the_wrong_kind_is_refused_with_the_reason(change, fault):
with pytest.raises(ScopeError) as refused:
read(COMPLETE.replace(*change))
assert any(fault in problem for problem in refused.value.problems)
def test_text_that_is_not_yaml_is_refused():
with pytest.raises(ScopeError, match="not valid YAML"):
read("intent: [unclosed")
with pytest.raises(ScopeError, match="must be a mapping"):
read("- just\n- a list\n")
def test_a_provider_supplies_the_values_the_catalogue_leaves_to_it():
spec = COMPLETE + "network: {topology: hub_and_spoke}\n"
with pytest.raises(ScopeError):
read(spec, provider_values={"network.topology": ["flat"]})
allowed = read(spec, provider_values={"network.topology": ["hub_and_spoke"]})
assert allowed.values["network.topology"] == "hub_and_spoke"
def test_an_engagement_reference_and_a_customer_case_name_the_same_thing():
by_engagement = read(
COMPLETE.replace(
"customer_case: acme-bookings", "engagement_id: ENG-2026-UK-084"
)
)
assert by_engagement.customer_case == "eng-2026-uk-084"
assert slug("ENG 2026/UK 084") == "eng-2026-uk-084"
with pytest.raises(ScopeError, match="name the same thing and differ"):
read(
COMPLETE.replace(
"{customer_case: acme-bookings}", "{customer_case: a, engagement_id: b}"
)
)
def test_the_completed_copy_holds_every_field_and_the_agents_sections():
scope = read(
COMPLETE.replace("peak: 40", "peak: decide") + "delivery: {start: 2026-11-01}\n"
)
completed = scope.completed(DRAFT)
assert completed["metadata"]["status"] == DRAFT
assert completed["users"] == {"count": 400, "peak": "decide"}
assert completed["delivery"]["start"] == "2026-11-01" # plain data, not a date
assert completed["network"]["topology"] == ASK
assert completed["provenance"]["users.count"] == "supplied"
assert completed["provenance"]["users.peak"] == "assumed"
assert completed["provenance"]["network.topology"] == "unknown"
assert completed["readiness"]["supplied"] == len(scope.values)
assert completed["inferred_assumptions"][0]["field"] == "users.peak"
assert yaml.safe_load(yaml.safe_dump(completed)) == completed
def test_a_completed_copy_can_be_read_back_as_a_spec():
"""The agent's own sections are replaced, not rejected, on the way back in."""
scope = read(COMPLETE.replace("peak: 40", "peak: decide"))
again = read(yaml.safe_dump(scope.completed(DRAFT)))
assert again.values == scope.values and again.states == scope.states
def test_a_specialist_is_shown_what_is_stated_and_what_is_open():
scope = read(
COMPLETE.replace("peak: 40", "peak: decide") + "run: {pre_approved: true}\n"
)
context = scope.for_context()
assert context["kind"] == "workload"
assert context["stated"]["users"] == {"count": 400}
assert context["stated"]["intent"]["requirements"][0]["id"] == "REQ-01"
assert context["to_decide"][0]["field"] == "users.peak"
# How to run the agent, and the case's name, are not the Specialists' business.
assert "run" not in context["stated"] and "metadata" not in context["stated"]
def test_a_template_and_a_reading_are_written_for_a_person():
template = yaml.safe_load(render_template(CATALOGUE, "workload"))
reading = render_reading(read(COMPLETE.replace("count: 400, peak: 40", "peak: 40")))
assert template["engagement"]["kind"] == "workload"
assert template["users"] == {"count": ASK, "peak": ASK}
assert "delivery" not in template # optional fields are left out of a template
assert "Essential, needed before work can start:" in reading
assert "users.count: How many users?" in reading
# --- the real catalogue and the benchmark spec ------------------------------------------
def test_the_gcp_bundle_reads_the_benchmark_landing_zone_spec():
bundle = load_provider("gcp")
text = (
Path(__file__).resolve().parent.parent.parent
/ "eval"
/ "datasets"
/ "specs"
/ "landing-zone.yaml"
).read_text()
scope = read_scope(
text, bundle.scope_catalogue(), provider_values=bundle.scope_values
)
assert scope.kind == "platform_foundation"
assert scope.customer_case == "case-08-landing-zone"
# Complete enough to start with nobody to ask.
assert scope.gaps(ESSENTIAL) == []
assert scope.values["networking.topology"] == "shared_vpc_hub_and_spoke"
assert scope.states["workload.users.count"] == "not_applicable"
assert [item["id"] for item in scope.values["intent.requirements"]] == [
"REQ-01",
"REQ-02",
"REQ-03",
"REQ-04",
]
def test_every_essential_field_of_the_catalogue_has_a_question_to_ask():
catalogue = load_provider("gcp").scope_catalogue()
for kind in catalogue.kinds:
for path, spec in catalogue.fields.items():
if spec.tier_for(kind) != "optional":
assert spec.question, (
f"{path} is {spec.tier_for(kind)} and has no question"
)

View File

@@ -0,0 +1,232 @@
"""The work breakdown structure: a plan checked, scheduled and drawn by code."""
import json
from tests.core.samples import SAMPLE_DATA
from app.core.work_breakdown import (
check_work_breakdown,
mermaid,
render,
schedule,
summary,
)
GRAPH = SAMPLE_DATA["target-architecture.json"] # Elements E-1 and E-2
def upstream(**delivery) -> dict:
scope = {"stated": {"delivery": delivery}, "assumed": []}
return {
"target_architecture": {"attachments": {"target-architecture.json": GRAPH}},
"scope": {"attachments": {"scope.json": json.dumps(scope)}},
}
def plan(packages=None, activities=None) -> dict[str, str]:
data = json.loads(SAMPLE_DATA["work-breakdown.json"])
if packages is not None:
data["work_packages"] = packages
if activities is not None:
data["activities"] = activities
return {"work-breakdown.json": json.dumps(data)}
def activity(ident: str, days: float, *after: str, **more) -> dict:
return {
"id": ident,
"work_package": "WP-1",
"name": f"Do {ident}",
"days": days,
"after": list(after),
**more,
}
PACKAGE = json.loads(SAMPLE_DATA["work-breakdown.json"])["work_packages"][0]
# --- the plan is checked against the design ---------------------------------------------
def test_a_plan_that_covers_the_design_is_accepted():
assert check_work_breakdown(plan(), upstream()) == []
def test_every_element_of_the_design_belongs_to_a_work_package():
only_one = [{**PACKAGE, "elements": ["E-1"]}]
invented = [{**PACKAGE, "elements": ["E-1", "E-2", "E-9"]}]
(left_out,) = check_work_breakdown(plan(only_one), upstream())
(unknown,) = check_work_breakdown(plan(invented), upstream())
assert "no work package delivers E-2" in left_out
assert "WP-1 delivers E-9, which is not an Element" in unknown
def test_a_plan_that_cannot_be_scheduled_is_refused_with_the_reason():
def problems(*activities):
return check_work_breakdown(plan(activities=list(activities)), upstream())
assert any(
"depend on each other in a circle" in p
for p in problems(activity("A-1", 1, "A-2"), activity("A-2", 1, "A-1"))
)
assert any(
"comes after A-9, which is not an activity" in p
for p in problems(activity("A-1", 1, "A-9"))
)
assert any(
"needs `days`, a number above 0" in p for p in problems(activity("A-1", 0))
)
assert any("split anything over" in p for p in problems(activity("A-1", 400)))
assert any(
"owner must be one of" in p
for p in problems(activity("A-1", 1, owner="vendor"))
)
assert any(
"is in WP-7, not a work package" in p
for p in problems({**activity("A-1", 1), "work_package": "WP-7"})
)
def test_a_work_package_holds_work_or_other_work_packages():
parent = {**PACKAGE, "id": "WP-0", "elements": []}
child = {**PACKAGE, "parent": "WP-0"}
idle = {**PACKAGE, "id": "WP-2", "elements": []}
assert check_work_breakdown(plan([parent, child]), upstream()) == []
assert any(
"WP-2 has no activity and no work package beneath it" in p
for p in check_work_breakdown(
plan([child | {"parent": None}, idle]), upstream()
)
)
assert any(
"has parent WP-9, which does not exist" in p
for p in check_work_breakdown(plan([{**PACKAGE, "parent": "WP-9"}]), upstream())
)
# --- the schedule is computed -----------------------------------------------------------
def test_an_activity_starts_when_what_it_follows_has_finished():
placed = schedule(
[activity("A-1", 2), activity("A-2", 3, "A-1"), activity("A-3", 1, "A-2")]
)
assert [(p.start, p.finish) for p in placed.values()] == [(0, 2), (2, 5), (5, 6)]
assert all(p.critical for p in placed.values())
def test_independent_work_runs_at_once_up_to_the_size_of_the_team():
work = [activity("A-1", 4), activity("A-2", 4), activity("A-3", 4)]
two = schedule(work, team_size=2)
three = schedule(work, team_size=3)
assert max(p.finish for p in two.values()) == 8
assert sorted(p.start for p in two.values()) == [0, 0, 4]
assert max(p.finish for p in three.values()) == 4
def test_what_the_customer_owns_takes_nobody_from_the_team():
work = [
activity("A-1", 10, owner="customer"),
activity("A-2", 3),
activity("A-3", 2, "A-1", "A-2"),
]
placed = schedule(work, team_size=1)
# The lead time and the team's work run together; the end waits on the longer.
assert placed["A-2"].start == 0 and placed["A-3"].start == 10
assert placed["A-1"].critical and placed["A-3"].critical
assert not placed["A-2"].critical
def test_when_people_are_short_the_longest_chain_goes_first():
work = [activity("A-1", 1), activity("A-2", 1), activity("A-3", 5, "A-2")]
placed = schedule(work, team_size=1)
assert placed["A-2"].start == 0 and placed["A-1"].start > 0
# --- the views are written from the plan ------------------------------------------------
def test_the_three_views_are_written_from_the_data():
text = render(plan(), upstream(team_size=2))
assert "## WBS dictionary" in text
# Acceptance is the Checks on the work package's Elements, read from the design.
assert (
"| WP-1 | Ingress | The service and its identity. | a running service | E-1, E-2 | K-1, K-2 |"
in text
)
assert (
"| A-2 | Deploy the service | WP-1 | 2 | A-1 | Day 1 | Day 3 | team | yes |"
in text
)
assert "## Timeline" in text and "Day 0" in text
assert "3 working days, about 0.6 weeks, with a team of 2 and a 5-day week." in text
assert "indicative" in text
def test_a_plan_over_the_delivery_window_says_so_and_names_the_critical_path():
long = plan(activities=[activity("A-1", 20), activity("A-2", 15, "A-1")])
over = render(long, upstream(team_size=2, window_weeks=4))
inside = render(long, upstream(team_size=2, window_weeks=10))
assert "**That is 3.0 weeks over the 4-week delivery window.**" in over
assert "The critical path is A-1 -> A-2" in over
assert "inside the 10-week delivery window, with 3.0 weeks to spare" in inside
assert summary(long, upstream(window_weeks=4))["within_window"] is False
def test_an_assumed_team_size_is_said_to_be_assumed():
assert "(assumed; the scope gives no team size)" in render(plan(), upstream())
def test_a_customers_lead_time_is_shown_as_theirs():
waiting = plan(
activities=[
activity(
"A-1", 30, owner="customer", assumption="The carrier takes six weeks."
),
activity("A-2", 2, "A-1"),
]
)
text = render(waiting, upstream(team_size=2))
assert "The customer owns these, and the plan waits on them:" in text
assert "- A-1 Do A-1: 30 days. The carrier takes six weeks." in text
assert "░" in text and "█" in text
def test_calendar_dates_and_a_chart_appear_only_with_a_start_date():
dated = upstream(team_size=2, target_start_date="2026-11-02") # a Monday
assert mermaid(plan(), upstream(team_size=2)) == ""
assert "Dates" not in render(plan(), upstream(team_size=2))
chart = mermaid(plan(), dated)
assert chart.startswith("gantt\n dateFormat YYYY-MM-DD")
assert "Deploy the service :crit, A-2, 2026-11-03, 2d" in chart
assert "| 2026-11-03 to 2026-11-04 |" in render(plan(), dated)
def test_a_week_of_working_days_skips_the_weekend():
week = plan(activities=[activity("A-1", 5), activity("A-2", 1, "A-1")])
text = render(week, upstream(team_size=1, target_start_date="2026-11-02"))
# Five working days from Monday end on Friday; the next starts on Monday.
assert "| 2026-11-02 to 2026-11-06 |" in text
assert "| 2026-11-09 to 2026-11-09 |" in text
# --- in an Execution --------------------------------------------------------------------