feat: working orhcestration engine
This commit is contained in:
310
tests/execution/test_deliverables.py
Normal file
310
tests/execution/test_deliverables.py
Normal file
@@ -0,0 +1,310 @@
|
||||
"""The dependency graph and Deliverable contracts."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
check_submission,
|
||||
deliverables_in,
|
||||
inputs_of,
|
||||
)
|
||||
|
||||
|
||||
def test_every_kind_is_declared_once_with_known_dependencies():
|
||||
assert set(DEPENDENCY_GRAPH) == set(DeliverableKind)
|
||||
for kind, spec in DEPENDENCY_GRAPH.items():
|
||||
assert spec.kind is kind
|
||||
assert set(spec.depends_on) <= set(DeliverableKind), kind
|
||||
|
||||
|
||||
def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle():
|
||||
position = {phase: index for index, phase in enumerate(PHASE_ORDER)}
|
||||
for spec in DEPENDENCY_GRAPH.values():
|
||||
for dependency in spec.depends_on:
|
||||
assert position[DEPENDENCY_GRAPH[dependency].phase] <= position[spec.phase]
|
||||
|
||||
visiting: set[str] = set()
|
||||
done: set[str] = set()
|
||||
|
||||
def visit(kind: str) -> None:
|
||||
assert kind not in visiting, f"cycle through {kind}"
|
||||
if kind in done:
|
||||
return
|
||||
visiting.add(kind)
|
||||
for dependency in DEPENDENCY_GRAPH[kind].depends_on:
|
||||
visit(dependency)
|
||||
visiting.discard(kind)
|
||||
done.add(kind)
|
||||
|
||||
for kind in DeliverableKind:
|
||||
visit(kind)
|
||||
|
||||
|
||||
def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist():
|
||||
assert PHASE_ORDER == (
|
||||
PhaseId.SCOPING,
|
||||
PhaseId.REQUIREMENTS_DISCOVERY,
|
||||
PhaseId.SOLUTION_ARCHITECTURE,
|
||||
PhaseId.SOLUTION_IMPLEMENTATION,
|
||||
PhaseId.SOLUTION_VALIDATION,
|
||||
PhaseId.PACKAGING,
|
||||
)
|
||||
for phase in PHASE_ORDER:
|
||||
assert deliverables_in(phase), phase
|
||||
specialists = [spec.specialist for spec in DEPENDENCY_GRAPH.values()]
|
||||
assert len(specialists) == len(set(specialists))
|
||||
|
||||
|
||||
def test_the_target_architecture_sits_between_the_design_and_the_code():
|
||||
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
|
||||
K = DeliverableKind
|
||||
|
||||
assert depends[K.SCOPE] == set()
|
||||
assert depends[K.REQUIREMENTS] == {K.SCOPE}
|
||||
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
|
||||
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
|
||||
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
|
||||
assert depends[K.TARGET_ARCHITECTURE] == {
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
}
|
||||
# Everything after the graph is derived from it.
|
||||
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
|
||||
assert depends[K.SOLUTION_GUIDE] == set(
|
||||
deliverables_in(PhaseId.SOLUTION_VALIDATION)
|
||||
)
|
||||
|
||||
|
||||
def test_the_design_is_approved_before_code_is_written():
|
||||
design = PhaseId.SOLUTION_ARCHITECTURE
|
||||
implementation = PhaseId.SOLUTION_IMPLEMENTATION
|
||||
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
|
||||
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
|
||||
|
||||
|
||||
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
|
||||
design = (
|
||||
DeliverableKind.SCOPE,
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.DESIGN_RECOMMENDATIONS,
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
)
|
||||
assert inputs_of(DeliverableKind.SCOPE) == ()
|
||||
assert inputs_of(DeliverableKind.REQUIREMENTS) == (DeliverableKind.SCOPE,)
|
||||
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
|
||||
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
|
||||
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:5]
|
||||
# The plan is produced beside the guide, not before it: neither reads the other.
|
||||
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
|
||||
DeliverableKind.SOLUTION_GUIDE,
|
||||
DeliverableKind.WORK_BREAKDOWN,
|
||||
}
|
||||
assert DeliverableKind.SOLUTION_GUIDE not in inputs_of(
|
||||
DeliverableKind.WORK_BREAKDOWN
|
||||
)
|
||||
|
||||
|
||||
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
|
||||
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
|
||||
|
||||
|
||||
def test_submission_meeting_the_contract_has_nothing_missing():
|
||||
content = (
|
||||
"# Deployment guidance\n"
|
||||
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
|
||||
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
|
||||
)
|
||||
attachments = {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "variable {}",
|
||||
"implementation.json": "[]",
|
||||
}
|
||||
|
||||
assert (
|
||||
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_submission_reports_missing_sections_and_attachments():
|
||||
missing = check_submission(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE,
|
||||
"## Deployment prerequisites\n",
|
||||
{"terraform/main.tf": "resource {}"},
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"section: Step-by-step deployment instructions",
|
||||
"attachment: terraform/variables.tf",
|
||||
"attachment: implementation.json",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"heading",
|
||||
[
|
||||
"### 5.1. Security, privacy, and compliance",
|
||||
"## security, privacy, and compliance",
|
||||
"# 5.1 Security, Privacy, and Compliance:",
|
||||
],
|
||||
)
|
||||
def test_section_matching_ignores_numbering_case_and_trailing_punctuation(heading):
|
||||
content = (
|
||||
f"{heading}\n"
|
||||
"## Reliability\n"
|
||||
"## Operational excellence\n"
|
||||
"## Cost optimization\n"
|
||||
"## Performance efficiency\n"
|
||||
"## Sustainability\n"
|
||||
)
|
||||
|
||||
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
|
||||
|
||||
|
||||
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
|
||||
content = (
|
||||
"### 4.1. Google Cloud products and features mapping\n"
|
||||
"## Non-functional requirements\n"
|
||||
)
|
||||
|
||||
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
|
||||
assert "section: Functional requirements" in check_submission(
|
||||
DeliverableKind.REQUIREMENTS, content, {}
|
||||
)
|
||||
|
||||
|
||||
def test_section_names_in_body_text_do_not_count_as_headings():
|
||||
missing = check_submission(
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
"Functional requirements, non-functional requirements, current state and "
|
||||
"dependencies are all covered below.\n",
|
||||
{},
|
||||
)
|
||||
|
||||
assert "section: Functional requirements" in missing
|
||||
|
||||
|
||||
def test_empty_content_is_refused():
|
||||
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
|
||||
"content: must not be empty",
|
||||
)
|
||||
|
||||
|
||||
# --- contracts beyond headings ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_architecture_description_needs_component_and_flow_sections():
|
||||
missing = check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
|
||||
)
|
||||
|
||||
assert missing == ("section: Components", "section: Flow")
|
||||
assert (
|
||||
check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION,
|
||||
"## Architecture components\nA.\n## Data flow\nB.\n",
|
||||
{},
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_a_required_attachment_must_have_content():
|
||||
missing = check_submission(
|
||||
DeliverableKind.VALIDATION_RESULTS,
|
||||
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
|
||||
{"verdicts.json": " \n"},
|
||||
)
|
||||
|
||||
assert missing == ("attachment: verdicts.json must not be empty",)
|
||||
|
||||
|
||||
# --- what a Specialist is shown of what it is given --------------------------------------
|
||||
|
||||
|
||||
def given_for(kind: DeliverableKind) -> dict:
|
||||
from tests.reference_workflow.workflow import dependencies_for, sample_submission
|
||||
|
||||
submitted = {}
|
||||
for each in DeliverableKind:
|
||||
call = sample_submission(each)
|
||||
submitted[each.value] = {
|
||||
"content": call.args["content"],
|
||||
"attachments": call.args["attachments"],
|
||||
}
|
||||
review = "\n## Architecture framework checklist review\n| a | b |\n"
|
||||
submitted["design_recommendations"]["content"] += review
|
||||
return dependencies_for(kind, submitted)
|
||||
|
||||
|
||||
def test_a_specialist_is_shown_only_what_it_uses():
|
||||
from tests.reference_workflow.workflow import shown
|
||||
|
||||
author = shown(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE,
|
||||
given_for(DeliverableKind.DEPLOYMENT_GUIDANCE),
|
||||
)
|
||||
validator = shown(
|
||||
DeliverableKind.VALIDATION_RESULTS,
|
||||
given_for(DeliverableKind.VALIDATION_RESULTS),
|
||||
)
|
||||
|
||||
# The graph is all the code is written from.
|
||||
assert list(author) == ["target_architecture"]
|
||||
# The validator judges code against Checks, and reads requirements and
|
||||
# recommendations for coverage; the prose describing the design is not judged.
|
||||
assert "architecture_description" not in validator
|
||||
assert {"requirements", "design_recommendations", "deployment_guidance"} <= set(
|
||||
validator
|
||||
)
|
||||
|
||||
|
||||
def test_nothing_is_taken_from_what_a_submission_is_checked_against():
|
||||
"""Showing less changes what the model reads, never what the code checks."""
|
||||
given = given_for(DeliverableKind.DEPLOYMENT_GUIDANCE)
|
||||
|
||||
assert {"requirements", "design_recommendations", "scope"} <= set(given)
|
||||
|
||||
|
||||
def test_the_scope_is_shown_once_and_only_its_prose_to_who_works_from_it():
|
||||
from tests.reference_workflow.workflow import shown
|
||||
|
||||
analyst = shown(
|
||||
DeliverableKind.REQUIREMENTS, given_for(DeliverableKind.REQUIREMENTS)
|
||||
)
|
||||
architect = shown(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
given_for(DeliverableKind.TARGET_ARCHITECTURE),
|
||||
)
|
||||
|
||||
assert list(analyst["scope"]) == ["content"]
|
||||
assert "scope" not in architect
|
||||
|
||||
|
||||
def test_a_section_written_for_a_reader_is_left_off_except_for_the_guide():
|
||||
from tests.reference_workflow.pillars import REVIEW_HEADING
|
||||
from tests.reference_workflow.workflow import FOR_READERS, shown
|
||||
|
||||
assert FOR_READERS[DeliverableKind.DESIGN_RECOMMENDATIONS] == REVIEW_HEADING
|
||||
architect = shown(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
given_for(DeliverableKind.TARGET_ARCHITECTURE),
|
||||
)
|
||||
packager = shown(
|
||||
DeliverableKind.SOLUTION_GUIDE, given_for(DeliverableKind.SOLUTION_GUIDE)
|
||||
)
|
||||
|
||||
assert REVIEW_HEADING not in architect["design_recommendations"]["content"]
|
||||
assert "## Design" in architect["design_recommendations"]["content"] or architect[
|
||||
"design_recommendations"
|
||||
]["content"].startswith("# design_recommendations")
|
||||
assert REVIEW_HEADING in packager["design_recommendations"]["content"]
|
||||
Reference in New Issue
Block a user